chroma-db 0.2.0 → 0.4.0

data/lib/.rbnext/3.1/chroma/resources/collection.rb

@@ -0,0 +1,388 @@
+# frozen_string_literal: true
+
+module Chroma
+  using RubyNext
+
+  module Resources
+    # A Collection class represents a store for your embeddings, documents, and any additional metadata.
+    # This class can be instantiated by receiving the collection's name and metadata hash.
+    class Collection
+      include Chroma::APIOperations::Request
+
+      attr_reader :id
+      attr_reader :name
+      attr_reader :metadata
+
+      def initialize(id:, name:, metadata: nil)
+        @id = id
+        @name = name
+        @metadata = metadata
+      end
+
+      # Query the collection and return an array of embeddings.
+      #
+      # query_embeddings - An array of the embeddings to use for querying the collection.
+      # results - The maximum number of results to return. 10 by default.
+      # where - A Hash of additional conditions to filter the query results (optional).
+      # where_document - A Hash of additional conditions to filter the associated documents (optional).
+      # include  - An Array of the additional information to include in the query results (optional). Metadatas,
+      #   documents, and distances by default.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   embeddings = collection.query(query_embeddings: [[1.5, 2.9, 3.3]], results: 5)
+      #
+      # Return an Array of Embedding with query results.
+      def query(query_embeddings:, results: 10, where: {}, where_document: {}, include: %w[metadatas documents distances])
+        payload = {
+          query_embeddings: query_embeddings,
+          n_results: results,
+          where: where,
+          where_document: where_document,
+          include: include
+        }
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/query", payload)
+
+        if result.success?
+          build_embeddings_response(result.success.body)
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Get embeddings from the collection.
+      #
+      # ids - An Array of the specific embedding IDs to retrieve (optional).
+      # where - A Hash of additional conditions to filter the query results (optional).
+      # sort - The sorting criteria for the query results (optional).
+      # limit - The maximum number of embeddings to retrieve (optional).
+      # offset - The offset for pagination (optional).
+      # page - The page number for pagination (optional).
+      # page_size - The page size for pagination (optional).
+      # where_document - A Hash of additional conditions to filter the associated documents (optional).
+      # include - An Array of the additional information to include in the query results (optional). Metadatas,
+      #   and documents by default.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   embeddings = collection.get([Array#sort, "Array#each"])
+      #
+      # Returns an Array of Embeddings
+      def get(ids: nil, where: {}, sort: nil, limit: nil, offset: nil, page: nil, page_size: nil, where_document: {}, include: %w[metadatas documents])
+        if !page.nil? && !page_size.nil?
+          offset = (page - 1) * page_size
+          limit = page_size
+        end
+
+        payload = {
+          ids: ids,
+          where: where,
+          sort: sort,
+          limit: limit,
+          offset: offset,
+          where_document: where_document,
+          include: include
+        }
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/get", payload)
+
+        if result.success?
+          build_embeddings_response(result.success.body)
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Add one or many embeddings to the collection.
+      #
+      # embeddings - An Array of Embeddings or one Embedding to add.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.add(Embedding.new(id: "Array#fetch", embeddings[9.8, 2.3, 2.9], metadata: {url: "https://..."}))
+      #
+      # Returns true with success or raises a Chroma::Error on failure.
+      def add(embeddings = [])
+        embeddings_array = Array(embeddings)
+        return false if embeddings_array.size == 0
+
+        payload = build_embeddings_payload(embeddings_array)
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/add", payload)
+
+        return true if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Delete embeddings from the collection.
+      #
+      # ids [Array<Integer>, nil] The specific embedding IDs to delete (optional).
+      # where [Hash] Additional conditions to filter the embeddings to delete (optional).
+      # where_document [Hash] Additional conditions to filter the associated documents (optional).
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.delete(["Array#fetch", "Array#sort"])
+      #
+      # Returns an Array of deleted global ids.
+      def delete(ids: nil, where: {}, where_document: {})
+        payload = {
+          ids: ids,
+          where: where,
+          where_document: where_document
+        }
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/delete", payload)
+
+        return result.success.body if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Update one or many embeddings to the collection.
+      #
+      # embeddings - An Array of Embeddings or one Embedding to add.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.update(Embedding.new(id: "Array#fetch", embeddings[9.8, 2.3, 2.9], metadata: {url: "https://..."}))
+      #
+      # Returns true with success or raises a Chroma::Error on failure.
+      def update(embeddings = [])
+        embeddings_array = Array(embeddings)
+        return false if embeddings_array.size == 0
+
+        payload = build_embeddings_payload(embeddings_array)
+        payload.delete(:increment_index)
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/update", payload)
+
+        return true if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Upsert (insert or update) one or many embeddings to the collection.
+      #
+      # embeddings - An Array of Embeddings or one Embedding to add.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   embeddings = [
+      #     Embedding.new(id: "Array#fetch", embeddings[9.8, 2.3, 2.9], metadata: {url: "https://..."}),
+      #     Embedding.new(id: "Array#select", embeddings[5.6, 3.1, 4.7], metadata: {url: "https://..."})
+      #   ]
+      #   collection.upsert()
+      #
+      # Returns true with success or raises a Chroma::Error on failure.
+      def upsert(embeddings = [])
+        embeddings_array = Array(embeddings)
+        return false if embeddings_array.size == 0
+
+        payload = build_embeddings_payload(embeddings_array)
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/upsert", payload)
+
+        return true if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Count the number of embeddings in a collection.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.count
+      #
+      # Returns the count of embeddings in the collection.
+      def count
+        result = self.class.execute_request(:get, "#{Chroma.api_url}/collections/#{id}/count")
+
+        return result.success.body if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Modify the name and metadata of the current collection.
+      #
+      # new_name - The new name for the collection.
+      # new_metadata - The new metadata hash for the collection.
+      #
+      # Examples:
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.modify("ruby-3.2-documentation")
+      #
+      # Returns nothing.
+      def modify(new_name, new_metadata: {})
+        payload = {new_name: new_name}
+        payload[:new_metadata] = new_metadata if new_metadata.any?
+
+        result = self.class.execute_request(:put, "#{Chroma.api_url}/collections/#{id}", payload)
+
+        if result.success?
+          @name = new_name
+          @metadata = new_metadata
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Creates an index for the collection.
+      #
+      # Examples:
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.create_index
+      #
+      # Returns true on success or raise a Chroma::Error on failure.
+      def create_index
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/create_index")
+
+        return true if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Create a new collection on the database.
+      #
+      # name - The name of the collection. Name needs to be between 3-63 characters, starts and ends
+      #   with an alphanumeric character, contains only alphanumeric characters, underscores or hyphens (-), and
+      #   contains no two consecutive periods
+      # metadata - A hash of additional metadata associated with the collection.
+      #
+      # Examples
+      #
+      #   collection = Chorma::Resources::Collection.create("ruby-documentation", {source: "Ruby lang website"})
+      #
+      # Returns the created collection object.
+      def self.create(name, metadata = nil)
+        payload = {name: name, metadata: metadata, get_or_create: false}
+
+        result = execute_request(:post, "#{Chroma.api_url}/collections", payload)
+
+        if result.success?
+          data = result.success.body
+          new(id: data["id"], name: data["name"], metadata: data["metadata"])
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Retrieves a collection from the database.
+      #
+      # name - The name of the collection to retrieve.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resources::Colection.get("ruby-documentation")
+      #
+      # Returns The retrieved collection object. Raises Chroma::InvalidRequestError if not found.
+      def self.get(name)
+        result = execute_request(:get, "#{Chroma.api_url}/collections/#{name}")
+
+        if result.success?
+          data = result.success.body
+          new(id: data["id"], name: data["name"], metadata: data["metadata"])
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Retrieves all collections in the database.
+      #
+      # Examples
+      #
+      #   collections = Chroma::Resources::Collection.list
+      #
+      # Returns An array of all collections in the database.
+      def self.list
+        result = execute_request(:get, "#{Chroma.api_url}/collections")
+
+        if result.success?
+          data = result.success.body
+          data.map { |item| new(id: item["id"], name: item["name"], metadata: item["metadata"]) }
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Deletes a collection from the database.
+      #
+      # name - The name of the collection to delete.
+      #
+      # Examples
+      #
+      #   Chroma::Resources::Collection.delete("ruby-documentation")
+      #
+      # Returns true if the collection was successfully deleted, raise Chroma::InvalidRequestError otherwise.
+      def self.delete(name)
+        result = execute_request(:delete, "#{Chroma.api_url}/collections/#{name}")
+
+        return true if result.success?
+
+        raise_failure_error(result)
+      end
+
+      def self.raise_failure_error(result)
+        case result.failure.error
+        in Exception => exception
+          raise Chroma::APIConnectionError.new(exception.message)
+        in Net::HTTPInternalServerError => response
+          if response.body.is_a?(String) && (response.body.include?("ValueError") || response.body.include?("IndexError") || response.body.include?("TypeError"))
+            raise Chroma::InvalidRequestError.new(result.failure.body, status: result.failure.status, body: result.failure.body)
+          else
+            raise Chroma::APIConnectionError.new(result.failure.body, status: result.failure.status, body: result.failure.body)
+          end
+        else
+          raise Chroma::APIError.new(result.failure.body, status: result.failure.status, body: result.failure.body)
+        end
+      end
+      private_class_method :raise_failure_error
+
+      private
+
+      def build_embeddings_payload(embeddings, increment_index = true)
+        payload = {ids: [], embeddings: [], metadatas: [], documents: [], increment_index: increment_index}
+
+        embeddings.each do |embedding|
+          payload[:ids] << embedding.id
+          payload[:embeddings] << embedding.embedding
+          payload[:metadatas] << embedding.metadata
+          payload[:documents] << embedding.document
+        end
+
+        payload
+      end
+
+      def build_embeddings_response(result)
+        Chroma::Util.log_debug("Building embeddings from #{result.inspect}")
+
+        result_ids = result.fetch("ids", []).flatten
+        result_embeddings = (result.dig("embeddings") || []).flatten
+        result_documents = (result.dig("documents") || []).flatten
+        result_metadatas = (result.dig("metadatas") || []).flatten
+        result_distances = (result.dig("distances") || []).flatten
+
+        Chroma::Util.log_debug("Ids #{result_ids.inspect}")
+        Chroma::Util.log_debug("Embeddings #{result_embeddings.inspect}")
+        Chroma::Util.log_debug("Documents #{result_documents.inspect}")
+        Chroma::Util.log_debug("Metadatas #{result_metadatas.inspect}")
+        Chroma::Util.log_debug("distances #{result_distances.inspect}")
+
+        result_ids.map.with_index do |id, index|
+          Chroma::Resources::Embedding.new(id: id, embedding: result_embeddings[index], document: result_documents[index], metadata: result_metadatas[index], distance: result_distances[index])
+        end
+      end
+    end
+  end
+end

data/lib/chroma/api_operations/request.rb

@@ -2,13 +2,18 @@
 
 module Chroma
   module APIOperations
+    using RubyNext
+
     # Request's response Data object.
     #
     # status - HTTP status code. It is zero when a request fails due to network error.
     # body - Parsed JSON object or response body.
     # headers - HTTP response headers.
     # error - Exception or Net::HTTPResponse object if the response is not Net::HTTPSuccess
-    Response = Data.define(:status, :body, :headers, :error)
+    #
+    # NOTE: Not supported yet by Ruby Next
+    # Response = Data.define(:status, :body, :headers, :error)
+    Response = Struct.new("Response", :status, :body, :headers, :error)
 
     # Request module provides functionality to perform HTTP requests.
     module Request
@@ -77,14 +82,14 @@ module Chroma
 
         private def build_response_details(response, exception: false, parse_body: true)
           response_data = Chroma::APIOperations::Response.new(
-            status: exception ? 0 : response.code.to_i,
-            body: if exception
-                    exception.to_s
-                  else
-                    (parse_body ? body_to_json(response.body) : response.body)
-                  end,
-            headers: exception ? {} : response.each_header.to_h,
-            error: response.is_a?(Net::HTTPSuccess) ? nil : response
+            exception ? 0 : response.code.to_i,
+            if exception
+              exception.to_s
+            else
+              (parse_body ? body_to_json(response.body) : response.body)
+            end,
+            exception ? {} : response.each_header.to_h,
+            response.is_a?(Net::HTTPSuccess) ? nil : response
           )
 
           case response

data/lib/chroma/resources/collection.rb

@@ -1,16 +1,20 @@
 # frozen_string_literal: true
 
 module Chroma
+  using RubyNext
+
   module Resources
     # A Collection class represents a store for your embeddings, documents, and any additional metadata.
     # This class can be instantiated by receiving the collection's name and metadata hash.
     class Collection
       include Chroma::APIOperations::Request
 
+      attr_reader :id
       attr_reader :name
       attr_reader :metadata
 
-      def initialize(name:, metadata: nil)
+      def initialize(id:, name:, metadata: nil)
+        @id = id
         @name = name
         @metadata = metadata
       end
@@ -39,7 +43,7 @@ module Chroma
           include:
         }
 
-        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/query", payload)
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/query", payload)
 
         if result.success?
           build_embeddings_response(result.success.body)
@@ -83,7 +87,7 @@ module Chroma
           include:
         }
 
-        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/get", payload)
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/get", payload)
 
         if result.success?
           build_embeddings_response(result.success.body)
@@ -108,7 +112,7 @@ module Chroma
 
         payload = build_embeddings_payload(embeddings_array)
 
-        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/add", payload)
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/add", payload)
 
         return true if result.success?
 
@@ -134,7 +138,7 @@ module Chroma
           where_document:
         }
 
-        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/delete", payload)
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/delete", payload)
 
         return result.success.body if result.success?
 
@@ -158,7 +162,7 @@ module Chroma
         payload = build_embeddings_payload(embeddings_array)
         payload.delete(:increment_index)
 
-        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/update", payload)
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/update", payload)
 
         return true if result.success?
 
@@ -185,7 +189,7 @@ module Chroma
 
         payload = build_embeddings_payload(embeddings_array)
 
-        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/upsert", payload)
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/upsert", payload)
 
         return true if result.success?
 
@@ -201,7 +205,7 @@ module Chroma
       #
       # Returns the count of embeddings in the collection.
       def count
-        result = self.class.execute_request(:get, "#{Chroma.api_url}/collections/#{name}/count")
+        result = self.class.execute_request(:get, "#{Chroma.api_url}/collections/#{id}/count")
 
         return result.success.body if result.success?
 
@@ -223,7 +227,7 @@ module Chroma
         payload = {new_name:}
         payload[:new_metadata] = new_metadata if new_metadata.any?
 
-        result = self.class.execute_request(:put, "#{Chroma.api_url}/collections/#{name}", payload)
+        result = self.class.execute_request(:put, "#{Chroma.api_url}/collections/#{id}", payload)
 
         if result.success?
           @name = new_name
@@ -242,7 +246,7 @@ module Chroma
       #
       # Returns true on success or raise a Chroma::Error on failure.
       def create_index
-        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/create_index")
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{id}/create_index")
 
         return true if result.success?
 
@@ -268,7 +272,7 @@ module Chroma
 
         if result.success?
           data = result.success.body
-          new(name: data["name"], metadata: data["metadata"])
+          new(id: data["id"], name: data["name"], metadata: data["metadata"])
         else
           raise_failure_error(result)
         end
@@ -288,7 +292,7 @@ module Chroma
 
         if result.success?
           data = result.success.body
-          new(name: data["name"], metadata: data["metadata"])
+          new(id: data["id"], name: data["name"], metadata: data["metadata"])
         else
           raise_failure_error(result)
         end
@@ -306,7 +310,7 @@ module Chroma
 
         if result.success?
           data = result.success.body
-          data.map { |item| new(name: item["name"], metadata: item["metadata"]) }
+          data.map { |item| new(id: item["id"], name: item["name"], metadata: item["metadata"]) }
         else
           raise_failure_error(result)
         end

data/lib/chroma/resources/database.rb

@@ -4,6 +4,8 @@ module Chroma
   module Resources
     # The Database class provides methods for interacting with the Chroma database server.
     class Database
+      using RubyNext
+
       include Chroma::APIOperations::Request
       # Get the version of the Chroma database server.
       #

data/lib/chroma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Chroma
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

data/lib/chroma-db.rb

@@ -6,13 +6,17 @@ require "uri"
 require "json"
 require "logger"
 require "forwardable"
+require "ruby-next"
+require "ruby-next/language/setup"
 
-require_relative "chroma/version"
-require_relative "chroma/util"
-require_relative "chroma/chroma_configuration"
-require_relative "chroma/chroma"
-require_relative "chroma/api_operations/request"
-require_relative "chroma/errors"
-require_relative "chroma/resources/embedding"
-require_relative "chroma/resources/collection"
-require_relative "chroma/resources/database"
+RubyNext::Language.setup_gem_load_path(transpile: true)
+
+require "chroma/version"
+require "chroma/util"
+require "chroma/chroma_configuration"
+require "chroma/chroma"
+require "chroma/api_operations/request"
+require "chroma/errors"
+require "chroma/resources/embedding"
+require "chroma/resources/collection"
+require "chroma/resources/database"