lancelot 0.3.1 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59a61f845bead9178dc6b7ca831b0cd1f3577969c189d723b085a706527bb9a9
-  data.tar.gz: f6941d534cc770393803c152f1718dad15639086b4ae78e84da93f8f8ab6f43f
+  metadata.gz: b6da55f7ccdf6c0b08d1e317e63a627573e9599af9fe2c2f7c93ab8cdefc755d
+  data.tar.gz: fd75f76de200412842b7ad963a54dff99e691cf6a4a9c4e1df37ea65587c23a1
 SHA512:
-  metadata.gz: 04f63038fe699b2441c22618daac20ef710df82a5d95c6c8258317a2600b23ea2844ab5c2f1bc01245fb4ff90d531f3ec09f7ddcb28876e758b983a0bcb1fdc2
-  data.tar.gz: 20b8c06914dc993b6868b7264cfcfba0eeb0a0a0c70b0473ac6e3b9e163c8ed3855c2eeae1ad02ab3f19125f80c844a0aebd2c26a91aafc005af0dfebd8d37d6
+  metadata.gz: 44762f80ae8374f6ecf930360ca30b7e6987bbd978ae40d1fdb0c69093cc22d68877f5af2d18e25d088a13d386a2cfaba663c48d118c310e061ad130d5d86f1f
+  data.tar.gz: 976fee93527713f6404d9a3057e2feaa2d9b2f38601280a961c30653e9706a762aab2d1bf729dcee0b3954d4c07103f2db02e788736853ae70cd026bda0f8021

data/README.md

@@ -1,4 +1,4 @@
-# Lancelot
+<img src="/docs/assets/lancelot-wide.png" alt="lancelot" height="80px">
 
 Ruby bindings for [Lance](https://github.com/lancedb/lance), a modern columnar data format for ML. Lancelot provides a Ruby-native interface to Lance, enabling efficient storage and search of multimodal data including text, vectors, and more.
 
@@ -57,6 +57,25 @@ dataset.vector_search(query_embedding, column: "embedding", limit: 5).each { |r|
 
 ## Installation
 
+### System Requirements
+
+Lancelot requires the Protocol Buffers compiler (`protoc`) to build from source:
+
+```bash
+# macOS (via Homebrew)
+brew install protobuf
+
+# Ubuntu/Debian
+sudo apt-get install protobuf-compiler
+
+# Other systems
+# Download from https://github.com/protocolbuffers/protobuf/releases
+```
+
+**Note**: The `protoc` compiler is only needed when building the gem from source. Pre-built gems distributed through RubyGems.org do not require `protoc` to be installed.
+
+### Install the Gem
+
 Install the gem and add to the application's Gemfile by executing:
 
 ```bash
@@ -276,7 +295,7 @@ bundle exec rake compile
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/cpetersen/lancelot. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/cpetersen/lancelot/blob/main/CODE_OF_CONDUCT.md).
+Bug reports and pull requests are welcome on GitHub at https://github.com/scientist-labs/lancelot. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/scientist-labs/lancelot/blob/main/CODE_OF_CONDUCT.md).
 
 ## License
 
@@ -284,4 +303,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Lancelot project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/cpetersen/lancelot/blob/main/CODE_OF_CONDUCT.md).
+Everyone interacting in the Lancelot project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/scientist-labs/lancelot/blob/main/CODE_OF_CONDUCT.md).

data/ext/lancelot/src/conversion.rs

@@ -159,9 +159,9 @@ pub fn build_record_batch(
         .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))
 }
 
-pub fn convert_batch_to_ruby(batch: &RecordBatch) -> Result<Vec<RHash>, Error> {
+pub fn convert_batch_to_ruby(batch: &RecordBatch) -> Result<RArray, Error> {
     let ruby = Ruby::get().unwrap();
-    let mut documents = Vec::new();
+    let documents = ruby.ary_new();
     
     let num_rows = batch.num_rows();
     let schema = batch.schema();
@@ -173,6 +173,15 @@ pub fn convert_batch_to_ruby(batch: &RecordBatch) -> Result<Vec<RHash>, Error> {
             let column = batch.column(col_idx);
             let key = Symbol::new(field.name());
             
+            // CRITICAL: Add bounds checking for all array access
+            if row_idx >= column.len() {
+                return Err(Error::new(
+                    magnus::exception::runtime_error(), 
+                    format!("Row index {} out of bounds for column '{}' with length {}", 
+                            row_idx, field.name(), column.len())
+                ));
+            }
+            
             match field.data_type() {
                 DataType::Utf8 => {
                     let array = column.as_any().downcast_ref::<StringArray>()
@@ -225,9 +234,19 @@ pub fn convert_batch_to_ruby(batch: &RecordBatch) -> Result<Vec<RHash>, Error> {
                         let float_array = values.as_any().downcast_ref::<Float32Array>()
                             .ok_or_else(|| Error::new(magnus::exception::runtime_error(), "Failed to cast vector values to Float32Array"))?;
                         
+                        // CRITICAL: Verify the float_array has the expected size
+                        let expected_size = *list_size as usize;
+                        if float_array.len() != expected_size {
+                            return Err(Error::new(
+                                magnus::exception::runtime_error(),
+                                format!("Vector data corruption: expected {} elements but found {} for field '{}'",
+                                        expected_size, float_array.len(), field.name())
+                            ));
+                        }
+                        
                         let ruby_array = ruby.ary_new();
-                        for i in 0..*list_size {
-                            ruby_array.push(float_array.value(i as usize))?;
+                        for i in 0..expected_size {
+                            ruby_array.push(float_array.value(i))?;
                         }
                         doc.aset(key, ruby_array)?;
                     }
@@ -238,7 +257,7 @@ pub fn convert_batch_to_ruby(batch: &RecordBatch) -> Result<Vec<RHash>, Error> {
             }
         }
         
-        documents.push(doc);
+        documents.push(doc)?;
     }
     
     Ok(documents)

data/ext/lancelot/src/dataset.rs

@@ -1,4 +1,4 @@
-use magnus::{Error, Ruby, RHash, RArray, Symbol, TryConvert, function, method, RClass, Module, Object};
+use magnus::{Error, Ruby, RHash, RArray, Symbol, TryConvert, Value, function, method, RClass, Module, Object};
 use std::cell::RefCell;
 use std::sync::Arc;
 use tokio::runtime::Runtime;
@@ -11,6 +11,26 @@ use futures::stream::TryStreamExt;
 
 use crate::schema::build_arrow_schema;
 use crate::conversion::{build_record_batch, convert_batch_to_ruby};
+use arrow_schema::DataType;
+
+/// Convert Arrow DataType to Ruby-friendly string representation
+fn datatype_to_ruby_string(dtype: &DataType) -> &'static str {
+    match dtype {
+        DataType::Utf8 | DataType::LargeUtf8 => "string",
+        DataType::Boolean => "boolean",
+        DataType::Int8 | DataType::Int16 | DataType::Int32 => "int32",
+        DataType::Int64 => "int64",
+        DataType::UInt8 | DataType::UInt16 | DataType::UInt32 => "int32",
+        DataType::UInt64 => "int64",
+        DataType::Float32 => "float32",
+        DataType::Float64 => "float64",
+        DataType::Date32 => "date",
+        DataType::Date64 => "datetime",
+        DataType::Timestamp(_, _) => "datetime",
+        DataType::FixedSizeList(_, _) => "vector", // Will be handled specially
+        _ => "unknown"
+    }
+}
 
 #[magnus::wrap(class = "Lancelot::Dataset", free_immediately, size)]
 pub struct LancelotDataset {
@@ -121,16 +141,40 @@ impl LancelotDataset {
 
     pub fn schema(&self) -> Result<RHash, Error> {
         let dataset = self.dataset.borrow();
-        let _dataset = dataset.as_ref()
+        let dataset = dataset.as_ref()
             .ok_or_else(|| Error::new(magnus::exception::runtime_error(), "Dataset not opened"))?;
 
+        // Get the actual schema from the Lance dataset
+        let schema = self.runtime.borrow_mut().block_on(async {
+            dataset.schema()
+        });
+        
+        // Convert Lance schema to Arrow schema
+        let arrow_schema: arrow_schema::Schema = schema.into();
+        let arrow_schema = Arc::new(arrow_schema);
+
         let ruby = Ruby::get().unwrap();
         let hash = ruby.hash_new();
         
-        // TODO: Read actual schema from Lance dataset once we figure out the 0.31 API
-        // For now, return a hardcoded schema that matches what we support
-        hash.aset(Symbol::new("text"), "string")?;
-        hash.aset(Symbol::new("score"), "float32")?;
+        // Iterate over Arrow schema fields
+        for field in arrow_schema.fields() {
+            let field_name = Symbol::new(&field.name());
+            
+            // Handle vector columns specially
+            if let DataType::FixedSizeList(inner_field, dimension) = field.data_type() {
+                // Check if it's a vector (float list)
+                if matches!(inner_field.data_type(), DataType::Float32 | DataType::Float16) {
+                    let vector_info = ruby.hash_new();
+                    vector_info.aset(Symbol::new("type"), "vector")?;
+                    vector_info.aset(Symbol::new("dimension"), *dimension)?;
+                    hash.aset(field_name, vector_info)?;
+                    continue;
+                }
+            }
+            
+            let field_type = datatype_to_ruby_string(field.data_type());
+            hash.aset(field_name, field_type)?;
+        }
 
         Ok(hash)
     }
@@ -157,9 +201,10 @@ impl LancelotDataset {
         let result_array = ruby.ary_new();
 
         for batch in batches {
-            let documents = convert_batch_to_ruby(&batch)?;
-            for doc in documents {
-                result_array.push(doc)?;
+            let batch_docs = convert_batch_to_ruby(&batch)?;
+            // Merge arrays by pushing each element
+            for i in 0..batch_docs.len() {
+                result_array.push(batch_docs.entry::<Value>(i as isize)?)?;
             }
         }
 
@@ -191,9 +236,10 @@ impl LancelotDataset {
         let result_array = ruby.ary_new();
 
         for batch in batches {
-            let documents = convert_batch_to_ruby(&batch)?;
-            for doc in documents {
-                result_array.push(doc)?;
+            let batch_docs = convert_batch_to_ruby(&batch)?;
+            // Merge arrays by pushing each element
+            for i in 0..batch_docs.len() {
+                result_array.push(batch_docs.entry::<Value>(i as isize)?)?;
             }
         }
 
@@ -265,9 +311,10 @@ impl LancelotDataset {
         let result_array = ruby.ary_new();
 
         for batch in batches {
-            let documents = convert_batch_to_ruby(&batch)?;
-            for doc in documents {
-                result_array.push(doc)?;
+            let batch_docs = convert_batch_to_ruby(&batch)?;
+            // Merge arrays by pushing each element
+            for i in 0..batch_docs.len() {
+                result_array.push(batch_docs.entry::<Value>(i as isize)?)?;
             }
         }
 
@@ -330,9 +377,10 @@ impl LancelotDataset {
         let result_array = ruby.ary_new();
 
         for batch in batches {
-            let documents = convert_batch_to_ruby(&batch)?;
-            for doc in documents {
-                result_array.push(doc)?;
+            let batch_docs = convert_batch_to_ruby(&batch)?;
+            // Merge arrays by pushing each element
+            for i in 0..batch_docs.len() {
+                result_array.push(batch_docs.entry::<Value>(i as isize)?)?;
             }
         }
 
@@ -380,9 +428,10 @@ impl LancelotDataset {
         let result_array = ruby.ary_new();
 
         for batch in batches {
-            let documents = convert_batch_to_ruby(&batch)?;
-            for doc in documents {
-                result_array.push(doc)?;
+            let batch_docs = convert_batch_to_ruby(&batch)?;
+            // Merge arrays by pushing each element
+            for i in 0..batch_docs.len() {
+                result_array.push(batch_docs.entry::<Value>(i as isize)?)?;
             }
         }
 
@@ -422,9 +471,10 @@ impl LancelotDataset {
         let result_array = ruby.ary_new();
 
         for batch in batches {
-            let documents = convert_batch_to_ruby(&batch)?;
-            for doc in documents {
-                result_array.push(doc)?;
+            let batch_docs = convert_batch_to_ruby(&batch)?;
+            // Merge arrays by pushing each element
+            for i in 0..batch_docs.len() {
+                result_array.push(batch_docs.entry::<Value>(i as isize)?)?;
             }
         }
 

data/lib/lancelot/dataset.rb

@@ -62,8 +62,16 @@ module Lancelot
       count_rows
     end
 
-    alias_method :count, :size
     alias_method :length, :size
+    
+    # Override Enumerable's count to use our efficient count_rows when no block given
+    def count(&block)
+      if block_given?
+        super(&block)  # Use Enumerable's count with block
+      else
+        count_rows  # Use our efficient count without block
+      end
+    end
 
     def all
       scan_all

data/lib/lancelot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lancelot
-  VERSION = "0.3.1"
+  VERSION = "0.3.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lancelot
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.3
 platform: ruby
 authors:
 - Chris Petersen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-08-10 00:00:00.000000000 Z
+date: 2025-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -111,6 +111,8 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- docs/assets/lancelot-wide.png
+- docs/assets/lancelot.png
 - examples/basic_usage.rb
 - examples/full_text_search.rb
 - examples/idempotent_create.rb
@@ -129,13 +131,13 @@ files:
 - lib/lancelot/rank_fusion.rb
 - lib/lancelot/version.rb
 - sig/lancelot.rbs
-homepage: https://github.com/cpetersen/lancelot
+homepage: https://github.com/scientist-labs/lancelot
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/cpetersen/lancelot
-  source_code_uri: https://github.com/cpetersen/lancelot
-  changelog_uri: https://github.com/cpetersen/lancelot/blob/main/CHANGELOG.md
+  homepage_uri: https://github.com/scientist-labs/lancelot
+  source_code_uri: https://github.com/scientist-labs/lancelot
+  changelog_uri: https://github.com/scientist-labs/lancelot/blob/main/CHANGELOG.md
 post_install_message:
 rdoc_options: []
 require_paths: