lancelot 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d1c9e53cfc1948e847b195b8d8860c6dba8a85813714b60a7d7747966533de3
-  data.tar.gz: 7c130c04c01a8eebce16755a1da082f4db6ec1f85ac2857d84943272c695eb89
+  metadata.gz: 59a61f845bead9178dc6b7ca831b0cd1f3577969c189d723b085a706527bb9a9
+  data.tar.gz: f6941d534cc770393803c152f1718dad15639086b4ae78e84da93f8f8ab6f43f
 SHA512:
-  metadata.gz: fb1a939d21c8935cec9fb9eb0ba1ddbef7ab9a8edd380149e0026db00ce67167609c99ed98597ccd692baa59f046dcd041fc76f653a5a635c152b82c343310d4
-  data.tar.gz: d35819fbfc3c2dec2b2cf65236b2bbf74eaef9f501d32f2a0a0ca7724ffc4bc1bdef56222cb50a3390d59d91e6e492a1170114b2762cebac38bcf4529bb27925
+  metadata.gz: 04f63038fe699b2441c22618daac20ef710df82a5d95c6c8258317a2600b23ea2844ab5c2f1bc01245fb4ff90d531f3ec09f7ddcb28876e758b983a0bcb1fdc2
+  data.tar.gz: 20b8c06914dc993b6868b7264cfcfba0eeb0a0a0c70b0473ac6e3b9e163c8ed3855c2eeae1ad02ab3f19125f80c844a0aebd2c26a91aafc005af0dfebd8d37d6

data/README.md

@@ -2,11 +2,52 @@
 
 Ruby bindings for [Lance](https://github.com/lancedb/lance), a modern columnar data format for ML. Lancelot provides a Ruby-native interface to Lance, enabling efficient storage and search of multimodal data including text, vectors, and more.
 
+## Quickstart
+
+```ruby
+require 'lancelot'
+require 'red-candle'
+
+strings = [
+  "apple",
+  "orange",
+  "google"
+]
+
+model = Candle::EmbeddingModel.from_pretrained
+
+dataset = Lancelot::Dataset.open_or_create("words", schema: {
+  text: :string,
+  embedding: { type: "vector", dimension: 768 }
+})
+
+records = strings.collect do |string|
+  embedding = model.embedding(string).first.to_a
+  { text: string, embedding: embedding }
+end
+
+dataset.add_documents(records)
+
+dataset.create_vector_index("embedding")
+dataset.create_text_index("text")
+
+
+query = "fruit"
+query_embedding = model.embedding(query).first.to_a
+dataset.vector_search(query_embedding, column: "embedding", limit: 5).each { |r| puts r[:text] }; nil
+
+dataset.text_search("apple", column: "text", limit: 5).each { |r| puts r[:text] }; nil
+
+query = "tech company"
+query_embedding = model.embedding(query).first.to_a
+dataset.vector_search(query_embedding, column: "embedding", limit: 5).each { |r| puts r[:text] }; nil
+```
+
 ## Features
 
 ### Implemented
 - **Dataset Creation**: Create Lance datasets with schemas
-- **Data Storage**: Add documents to datasets  
+- **Data Storage**: Add documents to datasets
 - **Document Retrieval**: Read documents from datasets with enumerable support
 - **Vector Search**: Create vector indices and perform similarity search
 - **Full-Text Search**: Built-in full-text search with inverted indices
@@ -14,12 +55,6 @@ Ruby bindings for [Lance](https://github.com/lancedb/lance), a modern columnar d
 - **Schema Support**: Define schemas with string, float32, and vector types
 - **Row Counting**: Get the number of rows in a dataset
 
-### Planned
-
-- **Multimodal Support**: Store and search across different data types beyond text and vectors
-- **Schema Evolution**: Add new columns to existing datasets without rewriting data
-- **Additional Fusion Methods**: Support for other fusion algorithms beyond RRF
-
 ## Installation
 
 Install the gem and add to the application's Gemfile by executing:

data/examples/idempotent_create.rb

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+# Demonstrates idempotent dataset creation with open_or_create
+
+require 'bundler/setup'
+require 'lancelot'
+require 'fileutils'
+
+dataset_path = "words"
+
+puts "="*60
+puts "Idempotent Dataset Creation Demo"
+puts "="*60
+
+schema = {
+  text: :string,
+  embedding: { type: "vector", dimension: 768 }
+}
+
+# First call - will CREATE the dataset
+puts "\n1. First call to open_or_create (should create)..."
+dataset = Lancelot::Dataset.open_or_create(dataset_path, schema: schema)
+puts "   Dataset opened/created. Current count: #{dataset.count}"
+
+# Add some data
+dataset.add_documents([
+  { text: "hello", embedding: Array.new(768) { rand } },
+  { text: "world", embedding: Array.new(768) { rand } }
+])
+puts "   Added 2 documents. New count: #{dataset.count}"
+
+# Second call - will OPEN the existing dataset
+puts "\n2. Second call to open_or_create (should open existing)..."
+dataset2 = Lancelot::Dataset.open_or_create(dataset_path, schema: schema)
+puts "   Dataset opened. Current count: #{dataset2.count}"
+puts "   ✓ Data persisted from previous session!"
+
+# Third call - still idempotent
+puts "\n3. Third call - still works..."
+dataset3 = Lancelot::Dataset.open_or_create(dataset_path, schema: schema)
+dataset3.add_documents([
+  { text: "more", embedding: Array.new(768) { rand } }
+])
+puts "   Added 1 more document. New count: #{dataset3.count}"
+
+# Demonstrate the OLD way that would fail
+puts "\n4. Compare with non-idempotent create (would fail)..."
+begin
+  # This will fail because dataset already exists
+  failing_dataset = Lancelot::Dataset.create(dataset_path, schema: schema)
+  puts "   ✗ This shouldn't happen!"
+rescue => e
+  puts "   ✓ Dataset.create correctly failed: #{e.class}"
+  puts "   Message: #{e.message[0..50]}..."
+end
+
+# Clean up
+FileUtils.rm_rf(dataset_path)
+
+puts "\n" + "="*60
+puts "Summary: Use open_or_create for idempotent operations!"
+puts "="*60
+puts "\nInstead of:"
+puts '  dataset = Lancelot::Dataset.create("words", schema: {...})'
+puts "\nUse:"
+puts '  dataset = Lancelot::Dataset.open_or_create("words", schema: {...})'
+puts "\nThis way your code works whether the dataset exists or not!"

data/examples/optional_fields_demo.rb

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+# This example demonstrates optional field support in lancelot
+# After the fix in conversion.rs, documents can have missing fields
+
+require 'bundler/setup'
+require 'lancelot'
+require 'fileutils'
+
+dataset_path = "example_optional_fields"
+FileUtils.rm_rf(dataset_path)
+
+puts "="*60
+puts "Lancelot Optional Fields Demo"
+puts "="*60
+
+# Step 1: Create dataset with initial schema
+puts "\n1. Creating dataset with 3 fields (id, text, score)..."
+schema = {
+  id: :string,
+  text: :string,
+  score: :float32
+}
+dataset = Lancelot::Dataset.create(dataset_path, schema: schema)
+
+# Add initial documents
+initial_docs = [
+  { id: "1", text: "First document", score: 0.9 },
+  { id: "2", text: "Second document", score: 0.8 }
+]
+dataset.add_documents(initial_docs)
+puts "   Added #{dataset.count} documents"
+
+# Step 2: Simulate schema evolution (adding a new field)
+puts "\n2. Simulating schema evolution (adding 'category' field)..."
+
+# Get existing data
+all_docs = dataset.to_a
+
+# Recreate with expanded schema
+FileUtils.rm_rf(dataset_path)
+expanded_schema = {
+  id: :string,
+  text: :string,
+  score: :float32,
+  category: :string  # NEW FIELD
+}
+dataset = Lancelot::Dataset.create(dataset_path, schema: expanded_schema)
+
+# Re-add existing docs with the new field
+docs_with_category = all_docs.map { |doc| doc.merge(category: "original") }
+dataset.add_documents(docs_with_category)
+puts "   Recreated dataset with expanded schema"
+
+# Step 3: Add new documents WITHOUT the new field
+puts "\n3. Adding new documents WITHOUT the 'category' field..."
+new_docs = [
+  { id: "3", text: "Third document", score: 0.7 },  # No category!
+  { id: "4", text: "Fourth document", score: 0.6 }  # No category!
+]
+
+begin
+  dataset.add_documents(new_docs)
+  puts "   ✅ SUCCESS! Added #{new_docs.size} documents with missing fields"
+rescue => e
+  puts "   ❌ FAILED: #{e.message}"
+  puts "   (This would have failed before the fix in conversion.rs)"
+end
+
+# Step 4: Verify the data
+puts "\n4. Verifying all documents..."
+dataset.to_a.each do |doc|
+  category = doc[:category] || "nil"
+  puts "   Doc #{doc[:id]}: category=#{category}"
+end
+
+puts "\nTotal documents: #{dataset.count}"
+
+# Cleanup
+FileUtils.rm_rf(dataset_path)
+
+puts "\n" + "="*60
+puts "Demo complete! Optional fields work correctly."
+puts "="*60

data/ext/lancelot/src/conversion.rs

@@ -39,11 +39,13 @@ pub fn build_record_batch(
         let item = RHash::try_convert(item)?;
         for field in schema.fields() {
             let key = Symbol::new(field.name());
-            let value: Value = item.fetch(key)
-                .or_else(|_| {
+            // Make fields optional - use get instead of fetch
+            let value: Value = item.get(key)
+                .or_else(|| {
                     // Try with string key  
-                    item.fetch(field.name().as_str())
-                })?;
+                    item.get(field.name().as_str())
+                })
+                .unwrap_or_else(|| Ruby::get().unwrap().qnil().as_value());
             
             match field.data_type() {
                 DataType::Utf8 => {

data/lib/lancelot/dataset.rb

@@ -15,6 +15,14 @@ module Lancelot
         dataset
       end
 
+      def open_or_create(path, schema:)
+        if File.exist?(path)
+          open(path)
+        else
+          create(path, schema: schema)
+        end
+      end
+
       private
 
       def normalize_schema(schema)

data/lib/lancelot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lancelot
-  VERSION = "0.2.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lancelot
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Chris Petersen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-07-27 00:00:00.000000000 Z
+date: 2025-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -113,6 +113,8 @@ files:
 - Rakefile
 - examples/basic_usage.rb
 - examples/full_text_search.rb
+- examples/idempotent_create.rb
+- examples/optional_fields_demo.rb
 - examples/red_candle_integration.rb
 - examples/vector_search.rb
 - ext/lancelot/.gitignore