lancelot 0.3.2 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cda5bd00de23ad9f4840b1cc7f6e96b7eb4ec124ba902d3901ef36b37594de97
-  data.tar.gz: 23e84c317e5bcd0f52870c673d0b96f9abf4f3ab97f20eb4e5b8a8047dd1cf1f
+  metadata.gz: 1861aeee032e7c7e742136fa606816034a5b22812862265e17bdf5c5ef1717cb
+  data.tar.gz: a0154bea94678710902d6696deb71f62e4b0fafafbe4adca8433ddb81c17074b
 SHA512:
-  metadata.gz: 32cb3e852ed77b8ec2831b08f93252b5ccb176942c217a366079e1cb2a2a97c4be74247b5f8410a2892852456278a5ec8e5b556bd9ca3379d740b292cc32c15b
-  data.tar.gz: 718a99dbfcd51dce872feedb87265be86417faba152d6f2512a9391f28e5fc5e115deb0154be52dc029c2771611b71c685f1f143a4d2da5157aaafc28997ffa7
+  metadata.gz: 550b2dde6c3868f62ddc1247a44045a07a46352dc4d6b3466876a35b5b4cae6d0df0b93faf41d45cdaec99092e074b72cac67a7a3dec53f268df11bacf791812
+  data.tar.gz: 32fab876ba1873e7648b4c9174e773cf69b604d158f7b71449172670363080390a7365b581f121d2d459a5516aaecc9c56f8f36925b1307b290f2f182d266f95

data/README.md

@@ -1,4 +1,4 @@
-# Lancelot
+<img src="/docs/assets/lancelot-wide.png" alt="lancelot" height="80px">
 
 Ruby bindings for [Lance](https://github.com/lancedb/lance), a modern columnar data format for ML. Lancelot provides a Ruby-native interface to Lance, enabling efficient storage and search of multimodal data including text, vectors, and more.
 
@@ -57,6 +57,25 @@ dataset.vector_search(query_embedding, column: "embedding", limit: 5).each { |r|
 
 ## Installation
 
+### System Requirements
+
+Lancelot requires the Protocol Buffers compiler (`protoc`) to build from source:
+
+```bash
+# macOS (via Homebrew)
+brew install protobuf
+
+# Ubuntu/Debian
+sudo apt-get install protobuf-compiler
+
+# Other systems
+# Download from https://github.com/protocolbuffers/protobuf/releases
+```
+
+**Note**: The `protoc` compiler is only needed when building the gem from source. Pre-built gems distributed through RubyGems.org do not require `protoc` to be installed.
+
+### Install the Gem
+
 Install the gem and add to the application's Gemfile by executing:
 
 ```bash
@@ -276,7 +295,7 @@ bundle exec rake compile
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/cpetersen/lancelot. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/cpetersen/lancelot/blob/main/CODE_OF_CONDUCT.md).
+Bug reports and pull requests are welcome on GitHub at https://github.com/scientist-labs/lancelot. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/scientist-labs/lancelot/blob/main/CODE_OF_CONDUCT.md).
 
 ## License
 
@@ -284,4 +303,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Lancelot project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/cpetersen/lancelot/blob/main/CODE_OF_CONDUCT.md).
+Everyone interacting in the Lancelot project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/scientist-labs/lancelot/blob/main/CODE_OF_CONDUCT.md).

data/ext/lancelot/src/dataset.rs

@@ -11,6 +11,26 @@ use futures::stream::TryStreamExt;
 
 use crate::schema::build_arrow_schema;
 use crate::conversion::{build_record_batch, convert_batch_to_ruby};
+use arrow_schema::DataType;
+
+/// Convert Arrow DataType to Ruby-friendly string representation
+fn datatype_to_ruby_string(dtype: &DataType) -> &'static str {
+    match dtype {
+        DataType::Utf8 | DataType::LargeUtf8 => "string",
+        DataType::Boolean => "boolean",
+        DataType::Int8 | DataType::Int16 | DataType::Int32 => "int32",
+        DataType::Int64 => "int64",
+        DataType::UInt8 | DataType::UInt16 | DataType::UInt32 => "int32",
+        DataType::UInt64 => "int64",
+        DataType::Float32 => "float32",
+        DataType::Float64 => "float64",
+        DataType::Date32 => "date",
+        DataType::Date64 => "datetime",
+        DataType::Timestamp(_, _) => "datetime",
+        DataType::FixedSizeList(_, _) => "vector", // Will be handled specially
+        _ => "unknown"
+    }
+}
 
 #[magnus::wrap(class = "Lancelot::Dataset", free_immediately, size)]
 pub struct LancelotDataset {
@@ -121,16 +141,40 @@ impl LancelotDataset {
 
     pub fn schema(&self) -> Result<RHash, Error> {
         let dataset = self.dataset.borrow();
-        let _dataset = dataset.as_ref()
+        let dataset = dataset.as_ref()
             .ok_or_else(|| Error::new(magnus::exception::runtime_error(), "Dataset not opened"))?;
 
+        // Get the actual schema from the Lance dataset
+        let schema = self.runtime.borrow_mut().block_on(async {
+            dataset.schema()
+        });
+        
+        // Convert Lance schema to Arrow schema
+        let arrow_schema: arrow_schema::Schema = schema.into();
+        let arrow_schema = Arc::new(arrow_schema);
+
         let ruby = Ruby::get().unwrap();
         let hash = ruby.hash_new();
         
-        // TODO: Read actual schema from Lance dataset once we figure out the 0.31 API
-        // For now, return a hardcoded schema that matches what we support
-        hash.aset(Symbol::new("text"), "string")?;
-        hash.aset(Symbol::new("score"), "float32")?;
+        // Iterate over Arrow schema fields
+        for field in arrow_schema.fields() {
+            let field_name = Symbol::new(&field.name());
+            
+            // Handle vector columns specially
+            if let DataType::FixedSizeList(inner_field, dimension) = field.data_type() {
+                // Check if it's a vector (float list)
+                if matches!(inner_field.data_type(), DataType::Float32 | DataType::Float16) {
+                    let vector_info = ruby.hash_new();
+                    vector_info.aset(Symbol::new("type"), "vector")?;
+                    vector_info.aset(Symbol::new("dimension"), *dimension)?;
+                    hash.aset(field_name, vector_info)?;
+                    continue;
+                }
+            }
+            
+            let field_type = datatype_to_ruby_string(field.data_type());
+            hash.aset(field_name, field_type)?;
+        }
 
         Ok(hash)
     }

data/lib/lancelot/dataset.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'fileutils'
+
 module Lancelot
   class Dataset
     class << self
@@ -15,10 +17,40 @@ module Lancelot
         dataset
       end
 
-      def open_or_create(path, schema:)
+      def open_or_create(path, schema:, mode: nil)
+        # Check if path exists
         if File.exist?(path)
-          open(path)
+          # Check if it's a file instead of directory
+          if File.file?(path)
+            if mode == "overwrite"
+              # Remove the file and create dataset
+              FileUtils.rm_f(path)
+              create(path, schema: schema)
+            else
+              raise ArgumentError, "Path #{path} exists as a file, not a directory. " \
+                                  "Use mode: 'overwrite' to replace it, or choose a different path."
+            end
+          # Path exists as directory - check if it's a valid Lance dataset
+          elsif File.exist?(File.join(path, "_versions"))
+            # Valid dataset exists - open it
+            open(path)
+          elsif !Dir.empty?(path)
+            # Non-empty directory that's not a Lance dataset
+            if mode == "overwrite"
+              # User explicitly wants to overwrite - remove and create new
+              FileUtils.rm_rf(path)
+              create(path, schema: schema)
+            else
+              # Fail safely - don't overwrite existing non-dataset directory
+              raise ArgumentError, "Directory exists at #{path} but is not a valid Lance dataset. " \
+                                  "Use mode: 'overwrite' to replace it, or choose a different path."
+            end
+          else
+            # Empty directory - safe to create dataset
+            create(path, schema: schema)
+          end
         else
+          # Path doesn't exist - create new dataset
           create(path, schema: schema)
         end
       end
@@ -62,8 +94,16 @@ module Lancelot
       count_rows
     end
 
-    alias_method :count, :size
     alias_method :length, :size
+    
+    # Override Enumerable's count to use our efficient count_rows when no block given
+    def count(&block)
+      if block_given?
+        super(&block)  # Use Enumerable's count with block
+      else
+        count_rows  # Use our efficient count without block
+      end
+    end
 
     def all
       scan_all

data/lib/lancelot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lancelot
-  VERSION = "0.3.2"
+  VERSION = "0.3.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lancelot
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.4
 platform: ruby
 authors:
 - Chris Petersen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-08-21 00:00:00.000000000 Z
+date: 2025-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -111,6 +111,8 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- docs/assets/lancelot-wide.png
+- docs/assets/lancelot.png
 - examples/basic_usage.rb
 - examples/full_text_search.rb
 - examples/idempotent_create.rb
@@ -129,13 +131,13 @@ files:
 - lib/lancelot/rank_fusion.rb
 - lib/lancelot/version.rb
 - sig/lancelot.rbs
-homepage: https://github.com/cpetersen/lancelot
+homepage: https://github.com/scientist-labs/lancelot
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/cpetersen/lancelot
-  source_code_uri: https://github.com/cpetersen/lancelot
-  changelog_uri: https://github.com/cpetersen/lancelot/blob/main/CHANGELOG.md
+  homepage_uri: https://github.com/scientist-labs/lancelot
+  source_code_uri: https://github.com/scientist-labs/lancelot
+  changelog_uri: https://github.com/scientist-labs/lancelot/blob/main/CHANGELOG.md
 post_install_message:
 rdoc_options: []
 require_paths: