chroma-db 0.1.0 → 0.3.0

data/lib/chroma/resources/collection.rb

@@ -0,0 +1,385 @@
+# frozen_string_literal: true
+
+module Chroma
+    using RubyNext
+  module Resources
+    # A Collection class represents a store for your embeddings, documents, and any additional metadata.
+    # This class can be instantiated by receiving the collection's name and metadata hash.
+    class Collection
+      include Chroma::APIOperations::Request
+
+      attr_reader :name
+      attr_reader :metadata
+
+      def initialize(name:, metadata: nil)
+        @name = name
+        @metadata = metadata
+      end
+
+      # Query the collection and return an array of embeddings.
+      #
+      # query_embeddings - An array of the embeddings to use for querying the collection.
+      # results - The maximum number of results to return. 10 by default.
+      # where - A Hash of additional conditions to filter the query results (optional).
+      # where_document - A Hash of additional conditions to filter the associated documents (optional).
+      # include  - An Array of the additional information to include in the query results (optional). Metadatas,
+      #   documents, and distances by default.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   embeddings = collection.query(query_embeddings: [[1.5, 2.9, 3.3]], results: 5)
+      #
+      # Return an Array of Embedding with query results.
+      def query(query_embeddings:, results: 10, where: {}, where_document: {}, include: %w[metadatas documents distances])
+        payload = {
+          query_embeddings:,
+          n_results: results,
+          where:,
+          where_document:,
+          include:
+        }
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/query", payload)
+
+        if result.success?
+          build_embeddings_response(result.success.body)
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Get embeddings from the collection.
+      #
+      # ids - An Array of the specific embedding IDs to retrieve (optional).
+      # where - A Hash of additional conditions to filter the query results (optional).
+      # sort - The sorting criteria for the query results (optional).
+      # limit - The maximum number of embeddings to retrieve (optional).
+      # offset - The offset for pagination (optional).
+      # page - The page number for pagination (optional).
+      # page_size - The page size for pagination (optional).
+      # where_document - A Hash of additional conditions to filter the associated documents (optional).
+      # include - An Array of the additional information to include in the query results (optional). Metadatas,
+      #   and documents by default.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   embeddings = collection.get([Array#sort, "Array#each"])
+      #
+      # Returns an Array of Embeddings
+      def get(ids: nil, where: {}, sort: nil, limit: nil, offset: nil, page: nil, page_size: nil, where_document: {}, include: %w[metadatas documents])
+        if !page.nil? && !page_size.nil?
+          offset = (page - 1) * page_size
+          limit = page_size
+        end
+
+        payload = {
+          ids:,
+          where:,
+          sort:,
+          limit:,
+          offset:,
+          where_document:,
+          include:
+        }
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/get", payload)
+
+        if result.success?
+          build_embeddings_response(result.success.body)
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Add one or many embeddings to the collection.
+      #
+      # embeddings - An Array of Embeddings or one Embedding to add.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.add(Embedding.new(id: "Array#fetch", embeddings[9.8, 2.3, 2.9], metadata: {url: "https://..."}))
+      #
+      # Returns true with success or raises a Chroma::Error on failure.
+      def add(embeddings = [])
+        embeddings_array = Array(embeddings)
+        return false if embeddings_array.size == 0
+
+        payload = build_embeddings_payload(embeddings_array)
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/add", payload)
+
+        return true if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Delete embeddings from the collection.
+      #
+      # ids [Array<Integer>, nil] The specific embedding IDs to delete (optional).
+      # where [Hash] Additional conditions to filter the embeddings to delete (optional).
+      # where_document [Hash] Additional conditions to filter the associated documents (optional).
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.delete(["Array#fetch", "Array#sort"])
+      #
+      # Returns an Array of deleted global ids.
+      def delete(ids: nil, where: {}, where_document: {})
+        payload = {
+          ids:,
+          where:,
+          where_document:
+        }
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/delete", payload)
+
+        return result.success.body if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Update one or many embeddings to the collection.
+      #
+      # embeddings - An Array of Embeddings or one Embedding to add.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.update(Embedding.new(id: "Array#fetch", embeddings[9.8, 2.3, 2.9], metadata: {url: "https://..."}))
+      #
+      # Returns true with success or raises a Chroma::Error on failure.
+      def update(embeddings = [])
+        embeddings_array = Array(embeddings)
+        return false if embeddings_array.size == 0
+
+        payload = build_embeddings_payload(embeddings_array)
+        payload.delete(:increment_index)
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/update", payload)
+
+        return true if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Upsert (insert or update) one or many embeddings to the collection.
+      #
+      # embeddings - An Array of Embeddings or one Embedding to add.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   embeddings = [
+      #     Embedding.new(id: "Array#fetch", embeddings[9.8, 2.3, 2.9], metadata: {url: "https://..."}),
+      #     Embedding.new(id: "Array#select", embeddings[5.6, 3.1, 4.7], metadata: {url: "https://..."})
+      #   ]
+      #   collection.upsert()
+      #
+      # Returns true with success or raises a Chroma::Error on failure.
+      def upsert(embeddings = [])
+        embeddings_array = Array(embeddings)
+        return false if embeddings_array.size == 0
+
+        payload = build_embeddings_payload(embeddings_array)
+
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/upsert", payload)
+
+        return true if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Count the number of embeddings in a collection.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.count
+      #
+      # Returns the count of embeddings in the collection.
+      def count
+        result = self.class.execute_request(:get, "#{Chroma.api_url}/collections/#{name}/count")
+
+        return result.success.body if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Modify the name and metadata of the current collection.
+      #
+      # new_name - The new name for the collection.
+      # new_metadata - The new metadata hash for the collection.
+      #
+      # Examples:
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.modify("ruby-3.2-documentation")
+      #
+      # Returns nothing.
+      def modify(new_name, new_metadata: {})
+        payload = {new_name:}
+        payload[:new_metadata] = new_metadata if new_metadata.any?
+
+        result = self.class.execute_request(:put, "#{Chroma.api_url}/collections/#{name}", payload)
+
+        if result.success?
+          @name = new_name
+          @metadata = new_metadata
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Creates an index for the collection.
+      #
+      # Examples:
+      #
+      #   collection = Chroma::Resource::Collection.get("ruby-documentation")
+      #   collection.create_index
+      #
+      # Returns true on success or raise a Chroma::Error on failure.
+      def create_index
+        result = self.class.execute_request(:post, "#{Chroma.api_url}/collections/#{name}/create_index")
+
+        return true if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Create a new collection on the database.
+      #
+      # name - The name of the collection. Name needs to be between 3-63 characters, starts and ends
+      #   with an alphanumeric character, contains only alphanumeric characters, underscores or hyphens (-), and
+      #   contains no two consecutive periods
+      # metadata - A hash of additional metadata associated with the collection.
+      #
+      # Examples
+      #
+      #   collection = Chorma::Resources::Collection.create("ruby-documentation", {source: "Ruby lang website"})
+      #
+      # Returns the created collection object.
+      def self.create(name, metadata = nil)
+        payload = {name:, metadata:, get_or_create: false}
+
+        result = execute_request(:post, "#{Chroma.api_url}/collections", payload)
+
+        if result.success?
+          data = result.success.body
+          new(name: data["name"], metadata: data["metadata"])
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Retrieves a collection from the database.
+      #
+      # name - The name of the collection to retrieve.
+      #
+      # Examples
+      #
+      #   collection = Chroma::Resources::Colection.get("ruby-documentation")
+      #
+      # Returns The retrieved collection object. Raises Chroma::InvalidRequestError if not found.
+      def self.get(name)
+        result = execute_request(:get, "#{Chroma.api_url}/collections/#{name}")
+
+        if result.success?
+          data = result.success.body
+          new(name: data["name"], metadata: data["metadata"])
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Retrieves all collections in the database.
+      #
+      # Examples
+      #
+      #   collections = Chroma::Resources::Collection.list
+      #
+      # Returns An array of all collections in the database.
+      def self.list
+        result = execute_request(:get, "#{Chroma.api_url}/collections")
+
+        if result.success?
+          data = result.success.body
+          data.map { |item| new(name: item["name"], metadata: item["metadata"]) }
+        else
+          raise_failure_error(result)
+        end
+      end
+
+      # Deletes a collection from the database.
+      #
+      # name - The name of the collection to delete.
+      #
+      # Examples
+      #
+      #   Chroma::Resources::Collection.delete("ruby-documentation")
+      #
+      # Returns true if the collection was successfully deleted, raise Chroma::InvalidRequestError otherwise.
+      def self.delete(name)
+        result = execute_request(:delete, "#{Chroma.api_url}/collections/#{name}")
+
+        return true if result.success?
+
+        raise_failure_error(result)
+      end
+
+      def self.raise_failure_error(result)
+        case result.failure.error
+        in Exception => exception
+          raise Chroma::APIConnectionError.new(exception.message)
+        in Net::HTTPInternalServerError => response
+          if response.body.is_a?(String) && (response.body.include?("ValueError") || response.body.include?("IndexError") || response.body.include?("TypeError"))
+            raise Chroma::InvalidRequestError.new(result.failure.body, status: result.failure.status, body: result.failure.body)
+          else
+            raise Chroma::APIConnectionError.new(result.failure.body, status: result.failure.status, body: result.failure.body)
+          end
+        else
+          raise Chroma::APIError.new(result.failure.body, status: result.failure.status, body: result.failure.body)
+        end
+      end
+      private_class_method :raise_failure_error
+
+      private
+
+      def build_embeddings_payload(embeddings, increment_index = true)
+        payload = {ids: [], embeddings: [], metadatas: [], documents: [], increment_index: increment_index}
+
+        embeddings.each do |embedding|
+          payload[:ids] << embedding.id
+          payload[:embeddings] << embedding.embedding
+          payload[:metadatas] << embedding.metadata
+          payload[:documents] << embedding.document
+        end
+
+        payload
+      end
+
+      def build_embeddings_response(result)
+        Chroma::Util.log_debug("Building embeddings from #{result.inspect}")
+
+        result_ids = result.fetch("ids", []).flatten
+        result_embeddings = (result.dig("embeddings") || []).flatten
+        result_documents = (result.dig("documents") || []).flatten
+        result_metadatas = (result.dig("metadatas") || []).flatten
+        result_distances = (result.dig("distances") || []).flatten
+
+        Chroma::Util.log_debug("Ids #{result_ids.inspect}")
+        Chroma::Util.log_debug("Embeddings #{result_embeddings.inspect}")
+        Chroma::Util.log_debug("Documents #{result_documents.inspect}")
+        Chroma::Util.log_debug("Metadatas #{result_metadatas.inspect}")
+        Chroma::Util.log_debug("distances #{result_distances.inspect}")
+
+        result_ids.map.with_index do |id, index|
+          Chroma::Resources::Embedding.new(id: id, embedding: result_embeddings[index], document: result_documents[index], metadata: result_metadatas[index], distance: result_distances[index])
+        end
+      end
+    end
+  end
+end

data/lib/chroma/resources/database.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Chroma
+  module Resources
+    # The Database class provides methods for interacting with the Chroma database server.
+    class Database
+      using RubyNext
+
+      include Chroma::APIOperations::Request
+      # Get the version of the Chroma database server.
+      #
+      # Returns the version of the Chroma database server.
+      def self.version
+        result = execute_request(:get, "#{Chroma.api_url}/version")
+
+        return result.success.body if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Reset the Chroma database server. This can't be undone.
+      #
+      # Returns true on success or raise a Chroma::Error on failure.
+      def self.reset
+        result = execute_request(:post, "#{Chroma.api_url}/reset")
+
+        return result.success.body if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Persist Chroma database data.
+      #
+      # Resturn true on success or raise a Chroma::Error on failure.
+      def self.persist
+        result = execute_request(:post, "#{Chroma.api_url}/persist")
+
+        return result.success.body if result.success?
+
+        raise_failure_error(result)
+      end
+
+      # Check the heartbeat of the Chroma database server.
+      #
+      # Return a Hash with a timestamp.
+      def self.heartbeat
+        result = execute_request(:get, "#{Chroma.api_url}/heartbeat")
+
+        return result.success.body if result.success?
+
+        raise_failure_error(result)
+      end
+
+      def self.raise_failure_error(result)
+        case result.failure.error
+        in Exception => exception
+          raise Chroma::APIConnectionError.new(exception.message)
+        in Net::HTTPInternalServerError => response
+          if response.body.is_a?(String) && (response.body.include?("ValueError") || response.body.include?("IndexError") || response.body.include?("TypeError"))
+            raise Chroma::InvalidRequestError.new(result.failure.body, status: result.failure.status, body: result.failure.body)
+          else
+            raise Chroma::APIConnectionError.new(result.failure.body, status: result.failure.status, body: result.failure.body)
+          end
+        else
+          raise Chroma::APIError.new(result.failure.body, status: result.failure.status, body: result.failure.body)
+        end
+      end
+      private_class_method :raise_failure_error
+    end
+  end
+end

data/lib/chroma/resources/embedding.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Chroma
+  module Resources
+    # A Embedding class represents an embedding by its Id, metadata, and document.
+    # This class is used by Collection class.
+    class Embedding
+      attr_reader :id
+      attr_reader :embedding
+      attr_reader :metadata
+      attr_reader :document
+      attr_reader :distance
+
+      def initialize(id:, embedding: nil, metadata: nil, document: nil, distance: nil)
+        @id = id
+        @embedding = embedding
+        @metadata = metadata
+        @document = document
+        @distance = distance
+      end
+    end
+  end
+end

data/lib/chroma/util.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Chroma
+  # The Util module provides utility methods for logging messages and data at different levels of severity.
+  module Util
+    # Logs an error message with the given data using the provided Logger instance.
+    #
+    # message - A String message to be logged.
+    # data - A Hash of additional data to be included in the log entry.
+    #
+    # Examples
+    #
+    #   Util.log_error("An error occurred", { user_id: 123, error_code: "404" })
+    #
+    # Returns nothing.
+    def self.log_error(message, data = {})
+      config = data.delete(:config) || Chroma.config
+      logger = config.logger || Chroma.logger
+
+      if (!logger.nil? || !config.log_level.nil?) && config.log_level <= Chroma::LEVEL_ERROR
+        log_internal(message, data, level: Chroma::LEVEL_ERROR, logger: Chroma.logger)
+      end
+    end
+
+    # Logs a debug message with the given data using the provided Logger instance.
+    #
+    # message - A String message to be logged.
+    # data - A Hash of additional data to be included in the log entry.
+    #
+    # Examples
+    #
+    #   Util.log_debug("Debugging information", { user_id: 123, action: "update" })
+    #
+    # Returns nothing.
+    def self.log_debug(message, data = {})
+      config = data.delete(:config) || Chroma.config
+      logger = config.logger || Chroma.logger
+
+      if (!logger.nil? || !config.log_level.nil?) && config.log_level <= Chroma::LEVEL_DEBUG
+        log_internal(message, data, level: Chroma::LEVEL_DEBUG, logger: Chroma.logger)
+      end
+    end
+
+    # Logs an informational message with the given data using the provided Logger instance.
+    #
+    # message - A String message to be logged.
+    # data - A Hash of additional data to be included in the log entry.
+    #
+    # Examples
+    #
+    #   Util.log_info("Processing request", { request_id: "abc123", route: "/users" })
+    #
+    # Returns nothing.
+    def self.log_info(message, data = {})
+      config = data.delete(:config) || Chroma.config
+      logger = config.logger || Chroma.logger
+
+      if (!logger.nil? || !config.log_level.nil?) && config.log_level <= Chroma::LEVEL_INFO
+        log_internal(message, data, level: Chroma::LEVEL_INFO, logger: Chroma.logger)
+      end
+    end
+
+    def self.log_internal(message, data = {}, level:, logger:)
+      data_str = data.reject { |_k, v| v.nil? }.map { |(k, v)| "#{k}=#{v}" }.join(" ")
+
+      logger&.log(level, "message=#{message} #{data_str}".strip)
+    end
+    private_class_method :log_internal
+  end
+end

data/lib/chroma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Chroma
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

data/lib/chroma-db.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "dry-monads"
+require "net/http"
+require "uri"
+require "json"
+require "logger"
+require "forwardable"
+require "ruby-next"
+require "ruby-next/language/setup"
+
+RubyNext::Language.setup_gem_load_path(transpile: true)
+
+require "chroma/version"
+require "chroma/util"
+require "chroma/chroma_configuration"
+require "chroma/chroma"
+require "chroma/api_operations/request"
+require "chroma/errors"
+require "chroma/resources/embedding"
+require "chroma/resources/collection"
+require "chroma/resources/database"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chroma-db
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Mario Alberto Chávez
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-04 00:00:00.000000000 Z
+date: 2023-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-monads
@@ -24,6 +24,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: ruby-next-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.15.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.15.0
+- !ruby/object:Gem::Dependency
+  name: ruby-next
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.15.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.15.0
 description: Chroma is the open-source embedding database. Chroma makes it easy to
   build LLM apps by making knowledge, facts, and skills pluggable for LLMs.
 email:
@@ -32,6 +60,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".rbnextrc"
+- ".ruby-version"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -39,7 +69,20 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- lib/chroma.rb
+- lib/.rbnext/2.7/chroma/api_operations/request.rb
+- lib/.rbnext/2.7/chroma/resources/collection.rb
+- lib/.rbnext/2.7/chroma/resources/database.rb
+- lib/.rbnext/3.1/chroma/api_operations/request.rb
+- lib/.rbnext/3.1/chroma/resources/collection.rb
+- lib/chroma-db.rb
+- lib/chroma/api_operations/request.rb
+- lib/chroma/chroma.rb
+- lib/chroma/chroma_configuration.rb
+- lib/chroma/errors.rb
+- lib/chroma/resources/collection.rb
+- lib/chroma/resources/database.rb
+- lib/chroma/resources/embedding.rb
+- lib/chroma/util.rb
 - lib/chroma/version.rb
 - sig/chroma.rbs
 homepage: https://mariochavez.io
@@ -57,7 +100,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 2.7.8
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -67,5 +110,5 @@ requirements: []
 rubygems_version: 3.4.12
 signing_key:
 specification_version: 4
-summary: Ruby bindings for Chroma DB.
+summary: Ruby client for Chroma DB.
 test_files: []

data/lib/chroma.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "chroma/version"
-
-module Chroma
-end