langchainrb 0.6.18 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 437c6387ded139ed1a513414bfb7242cdbadf1ba6526c7a89346aa2fa9490fc2
-  data.tar.gz: dd6f437a4bbc4807a16631dd790f66c9de4e9456011b2c4f84302fe3fab1377b
+  metadata.gz: 4efc896c1c0fa895ebd11bdb3c4d5604ccd34878aa29472419efca85800072da
+  data.tar.gz: c703f8150c7a6a6cb802260da2eeb95d4f7542cc0c5490794cea844351b4fe7c
 SHA512:
-  metadata.gz: 24748539de50dfa816fdb71173ef00a6b04f9737f32926fca919865a49b9812dd9f1fdb286c361c98e33cc994f67e8988ab688bfdf6bf3020d954eb0c791177c
-  data.tar.gz: 283b10460187cada7485e08a19c89e7485925ab2f73a5ad51b06a72e8fd9ee1600ddac9d000f13c0c1af13f6defece9fdcc272489d0df803f94da96fe1c76cfd
+  metadata.gz: 18b0c48b747978b2a92ecd8d118f31b199d5f58469a5dd55ccbfc3a56a9044faf42784e4760ff9a9fca94da019107629c107a3fe586b7d55243aa92bd1c5b949
+  data.tar.gz: d8513d2018ce48a60fbecc9ca3efb91411385fd71301e0c79ac3612d91b2b504427f2ea19b41991f2f4b43a8bd3b650035684b02a46ae0b114d83343ef5bce18

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [0.7.0] - 2023-10-22
+- BREAKING: Moving Rails-specific code to `langchainrb_rails` gem
+
+## [0.6.19] - 2023-10-18
+- Elasticsearch vector search support
+- Fix `lib/langchain/railtie.rb` not being loaded with the gem
+
 ## [0.6.18] - 2023-10-16
 - Introduce `Langchain::LLM::Response`` object
 - Introduce `Langchain::Chunk` object

data/README.md

@@ -19,11 +19,11 @@ Langchain.rb is a library that's an abstraction layer on top many emergent AI, M
 
 Install the gem and add to the application's Gemfile by executing:
 
-    $ bundle add langchainrb
+    bundle add langchainrb
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
-    $ gem install langchainrb
+    gem install langchainrb
 
 ## Usage
 
@@ -37,7 +37,7 @@ require "langchain"
 | -------- |:------------------:| -------:| -----------------:| -------:| -----------------:|
 | [Chroma](https://trychroma.com/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | :white_check_mark: |
 | [Hnswlib](https://github.com/nmslib/hnswlib/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
-| [Milvus](https://milvus.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Milvus](https://milvus.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | :white_check_mark: |
 | [Pinecone](https://www.pinecone.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | :white_check_mark: |
 | [Pgvector](https://github.com/pgvector/pgvector) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | :white_check_mark: |
 | [Qdrant](https://qdrant.tech/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | :white_check_mark: |

data/lib/langchain/vectorsearch/base.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "forwardable"
-
 module Langchain::Vectorsearch
   # = Vector Databases
   # A vector database a type of database that stores data as high-dimensional vectors, which are mathematical representations of features or attributes. Each vector has a certain number of dimensions, which can range from tens to thousands, depending on the complexity and granularity of the data.
@@ -9,11 +7,13 @@ module Langchain::Vectorsearch
   # == Available vector databases
   #
   # - {Langchain::Vectorsearch::Chroma}
+  # - {Langchain::Vectorsearch::Elasticsearch}
+  # - {Langchain::Vectorsearch::Hnswlib}
   # - {Langchain::Vectorsearch::Milvus}
+  # - {Langchain::Vectorsearch::Pgvector}
   # - {Langchain::Vectorsearch::Pinecone}
   # - {Langchain::Vectorsearch::Qdrant}
   # - {Langchain::Vectorsearch::Weaviate}
-  # - {Langchain::Vectorsearch::Pgvector}
   #
   # == Usage
   #
@@ -150,9 +150,6 @@ module Langchain::Vectorsearch
       raise NotImplementedError, "#{self.class.name} does not support asking questions"
     end
 
-    def_delegators :llm,
-      :default_dimension
-
     # HyDE-style prompt
     #
     # @param [String] User's question

data/lib/langchain/vectorsearch/chroma.rb

@@ -5,7 +5,8 @@ module Langchain::Vectorsearch
     #
     # Wrapper around Chroma DB
     #
-    # Gem requirements: gem "chroma-db", "~> 0.6.0"
+    # Gem requirements:
+    #     gem "chroma-db", "~> 0.6.0"
     #
     # Usage:
     # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, llm_api_key:, api_key: nil)
@@ -29,14 +30,15 @@ module Langchain::Vectorsearch
 
     # Add a list of texts to the index
     # @param texts [Array<String>] The list of texts to add
+    # @param ids [Array<String>] The list of ids to use for the texts (optional)
+    # @param metadatas [Array<Hash>] The list of metadata to use for the texts (optional)
     # @return [Hash] The response from the server
-    def add_texts(texts:, ids: [])
+    def add_texts(texts:, ids: [], metadatas: [])
       embeddings = Array(texts).map.with_index do |text, i|
         ::Chroma::Resources::Embedding.new(
           id: ids[i] ? ids[i].to_s : SecureRandom.uuid,
           embedding: llm.embed(text: text).embedding,
-          # TODO: Add support for passing metadata
-          metadata: {}, # metadatas[index],
+          metadata: metadatas[i] || {},
           document: text # Do we actually need to store the whole original document?
         )
       end
@@ -45,13 +47,12 @@ module Langchain::Vectorsearch
       collection.add(embeddings)
     end
 
-    def update_texts(texts:, ids:)
+    def update_texts(texts:, ids:, metadatas: [])
       embeddings = Array(texts).map.with_index do |text, i|
         ::Chroma::Resources::Embedding.new(
           id: ids[i].to_s,
           embedding: llm.embed(text: text).embedding,
-          # TODO: Add support for passing metadata
-          metadata: [], # metadatas[index],
+          metadata: metadatas[i] || {},
           document: text # Do we actually need to store the whole original document?
         )
       end

data/lib/langchain/vectorsearch/elasticsearch.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Langchain::Vectorsearch
+  class Elasticsearch < Base
+    #
+    # Wrapper around Elasticsearch vector search capabilities.
+    #
+    # Setting up Elasticsearch:
+    # 1. Get Elasticsearch up and running with Docker: https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html
+    # 2. Copy the HTTP CA certificate SHA-256 fingerprint and set the ELASTICSEARCH_CA_FINGERPRINT environment variable
+    # 3. Set the ELASTICSEARCH_URL environment variable
+    #
+    # Gem requirements:
+    #     gem "elasticsearch", "~> 8.0.0"
+    #
+    # Usage:
+    #     llm = Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
+    #     es = Langchain::Vectorsearch::Elasticsearch.new(
+    #       url: ENV["ELASTICSEARCH_URL"],
+    #       index_name: "docs",
+    #       llm: llm,
+    #       es_options: {
+    #         transport_options: {ssl: {verify: false}},
+    #         ca_fingerprint: ENV["ELASTICSEARCH_CA_FINGERPRINT"]
+    #       }
+    #     )
+    #
+    #     es.create_default_schema
+    #     es.add_texts(texts: ["..."])
+    #     es.similarity_search(text: "...")
+    #
+    attr_accessor :es_client, :index_name, :options
+
+    def initialize(url:, index_name:, llm:, api_key: nil, es_options: {})
+      require "elasticsearch"
+
+      @options = {
+        url: url,
+        request_timeout: 20,
+        log: false
+      }.merge(es_options)
+
+      @es_client = ::Elasticsearch::Client.new(**options)
+      @index_name = index_name
+
+      super(llm: llm)
+    end
+
+    def add_texts(texts: [])
+      body = texts.map do |text|
+        [
+          {index: {_index: index_name}},
+          {input: text, input_vector: llm.embed(text: text).embedding}
+        ]
+      end.flatten
+
+      es_client.bulk(body: body)
+    end
+
+    def update_texts(texts: [], ids: [])
+      body = texts.map.with_index do |text, i|
+        [
+          {index: {_index: index_name, _id: ids[i]}},
+          {input: text, input_vector: llm.embed(text: text).embedding}
+        ]
+      end.flatten
+
+      es_client.bulk(body: body)
+    end
+
+    def create_default_schema
+      es_client.indices.create(
+        index: index_name,
+        body: default_schema
+      )
+    end
+
+    def delete_default_schema
+      es_client.indices.delete(
+        index: index_name
+      )
+    end
+
+    def default_vector_settings
+      {type: "dense_vector", dims: llm.default_dimension}
+    end
+
+    def vector_settings
+      options[:vector_settings] || default_vector_settings
+    end
+
+    def default_schema
+      {
+        mappings: {
+          properties: {
+            input: {
+              type: "text"
+            },
+            input_vector: vector_settings
+          }
+        }
+      }
+    end
+
+    def default_query(query_vector)
+      {
+        script_score: {
+          query: {match_all: {}},
+          script: {
+            source: "cosineSimilarity(params.query_vector, 'input_vector') + 1.0",
+            params: {
+              query_vector: query_vector
+            }
+          }
+        }
+      }
+    end
+
+    # TODO: Implement this
+    # def ask()
+    # end
+
+    def similarity_search(text: "", k: 10, query: {})
+      if text.empty? && query.empty?
+        raise "Either text or query should pass as an argument"
+      end
+
+      if query.empty?
+        query_vector = llm.embed(text: text).embedding
+
+        query = default_query(query_vector)
+      end
+
+      es_client.search(body: {query: query, size: k}).body
+    end
+
+    def similarity_search_by_vector(embedding: [], k: 10, query: {})
+      if embedding.empty? && query.empty?
+        raise "Either embedding or query should pass as an argument"
+      end
+
+      query = default_query(embedding) if query.empty?
+
+      es_client.search(body: {query: query, size: k}).body
+    end
+  end
+end

data/lib/langchain/vectorsearch/milvus.rb

@@ -5,7 +5,8 @@ module Langchain::Vectorsearch
     #
     # Wrapper around Milvus REST APIs.
     #
-    # Gem requirements: gem "milvus", "~> 0.9.2"
+    # Gem requirements:
+    #     gem "milvus", "~> 0.9.2"
     #
     # Usage:
     # milvus = Langchain::Vectorsearch::Milvus.new(url:, index_name:, llm:, api_key:)
@@ -70,7 +71,7 @@ module Langchain::Vectorsearch
             type_params: [
               {
                 key: "dim",
-                value: default_dimension.to_s
+                value: llm.default_dimension.to_s
               }
             ]
           }

data/lib/langchain/vectorsearch/pgvector.rb

@@ -5,10 +5,12 @@ module Langchain::Vectorsearch
     #
     # The PostgreSQL vector search adapter
     #
-    # Gem requirements: gem "pgvector", "~> 0.2"
+    # Gem requirements:
+    #     gem "sequel", "~> 5.68.0"
+    #     gem "pgvector", "~> 0.2"
     #
     # Usage:
-    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:, namespace: nil)
+    #     pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:, namespace: nil)
     #
 
     # The operators supported by the PostgreSQL vector search adapter
@@ -91,7 +93,7 @@ module Langchain::Vectorsearch
     def create_default_schema
       db.run "CREATE EXTENSION IF NOT EXISTS vector"
       namespace_column = @namespace_column
-      vector_dimension = default_dimension
+      vector_dimension = llm.default_dimension
       db.create_table? table_name.to_sym do
         primary_key :id
         text :content

data/lib/langchain/vectorsearch/pinecone.rb

@@ -5,10 +5,11 @@ module Langchain::Vectorsearch
     #
     # Wrapper around Pinecone API.
     #
-    # Gem requirements: gem "pinecone", "~> 0.1.6"
+    # Gem requirements:
+    #     gem "pinecone", "~> 0.1.6"
     #
     # Usage:
-    # pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:, llm_api_key:)
+    #     pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
     # Initialize the Pinecone client
@@ -102,7 +103,7 @@ module Langchain::Vectorsearch
       client.create_index(
         metric: DEFAULT_METRIC,
         name: index_name,
-        dimension: default_dimension
+        dimension: llm.default_dimension
       )
     end
 

data/lib/langchain/vectorsearch/qdrant.rb

@@ -5,10 +5,11 @@ module Langchain::Vectorsearch
     #
     # Wrapper around Qdrant
     #
-    # Gem requirements: gem "qdrant-ruby", "~> 0.9.3"
+    # Gem requirements:
+    #     gem "qdrant-ruby", "~> 0.9.3"
     #
     # Usage:
-    # qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:, llm_api_key:)
+    #     qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
     # Initialize the Qdrant client
@@ -82,7 +83,7 @@ module Langchain::Vectorsearch
         collection_name: index_name,
         vectors: {
           distance: DEFAULT_METRIC.capitalize,
-          size: default_dimension
+          size: llm.default_dimension
         }
       )
     end

data/lib/langchain/vectorsearch/weaviate.rb

@@ -5,10 +5,11 @@ module Langchain::Vectorsearch
     #
     # Wrapper around Weaviate
     #
-    # Gem requirements: gem "weaviate-ruby", "~> 0.8.3"
+    # Gem requirements:
+    #     gem "weaviate-ruby", "~> 0.8.3"
     #
     # Usage:
-    # weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:, llm_api_key:)
+    #     weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
     # Initialize the Weaviate adapter

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.6.18"
+  VERSION = "0.7.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.6.18
+  version: 0.7.0
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-10-17 00:00:00.000000000 Z
+date: 2023-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: baran
@@ -70,16 +70,16 @@ dependencies:
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.11
+        version: '2.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.11
+        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: pragmatic_segmenter
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +220,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 8.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 8.2.0
 - !ruby/object:Gem::Dependency
   name: eqn
   requirement: !ruby/object:Gem::Requirement
@@ -525,7 +539,6 @@ files:
 - LICENSE.txt
 - README.md
 - lib/langchain.rb
-- lib/langchain/active_record/hooks.rb
 - lib/langchain/agent/base.rb
 - lib/langchain/agent/react_agent.rb
 - lib/langchain/agent/react_agent/react_agent_prompt.yaml
@@ -587,7 +600,6 @@ files:
 - lib/langchain/prompt/few_shot_prompt_template.rb
 - lib/langchain/prompt/loading.rb
 - lib/langchain/prompt/prompt_template.rb
-- lib/langchain/railtie.rb
 - lib/langchain/tool/base.rb
 - lib/langchain/tool/calculator.rb
 - lib/langchain/tool/database.rb
@@ -603,6 +615,7 @@ files:
 - lib/langchain/utils/token_length/token_limit_exceeded.rb
 - lib/langchain/vectorsearch/base.rb
 - lib/langchain/vectorsearch/chroma.rb
+- lib/langchain/vectorsearch/elasticsearch.rb
 - lib/langchain/vectorsearch/hnswlib.rb
 - lib/langchain/vectorsearch/milvus.rb
 - lib/langchain/vectorsearch/pgvector.rb

data/lib/langchain/active_record/hooks.rb

@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-
-module Langchain
-  module ActiveRecord
-    # This module adds the following functionality to your ActiveRecord models:
-    # * `vectorsearch` class method to set the vector search provider
-    # * `similarity_search` class method to search for similar texts
-    # * `upsert_to_vectorsearch` instance method to upsert the record to the vector search provider
-    #
-    # Usage:
-    #     class Recipe < ActiveRecord::Base
-    #       vectorsearch provider: Langchain::Vectorsearch::Weaviate.new(
-    #                    api_key: ENV["WEAVIATE_API_KEY"],
-    #                    url: ENV["WEAVIATE_URL"],
-    #                    index_name: "Recipes",
-    #                    llm: Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
-    #                 )
-    #
-    #       after_save :upsert_to_vectorsearch
-    #
-    #       # Overwriting how the model is serialized before it's indexed
-    #       def as_vector
-    #         [
-    #           "Title: #{title}",
-    #           "Description: #{description}",
-    #           ...
-    #         ]
-    #         .compact
-    #         .join("\n")
-    #       end
-    #     end
-    #
-    # Create the default schema
-    #     Recipe.class_variable_get(:@@provider).create_default_schema
-    # Query the vector search provider
-    #     Recipe.similarity_search("carnivore dish")
-    # Delete the default schema to start over
-    #     Recipe.class_variable_get(:@@provider).destroy_default_schema
-    #
-    module Hooks
-      def self.included(base)
-        base.extend ClassMethods
-      end
-
-      # Index the text to the vector search provider
-      # You'd typically call this method in an ActiveRecord callback
-      #
-      # @return [Boolean] true
-      # @raise [Error] Indexing to vector search DB failed
-      def upsert_to_vectorsearch
-        if previously_new_record?
-          self.class.class_variable_get(:@@provider).add_texts(
-            texts: [as_vector],
-            ids: [id]
-          )
-        else
-          self.class.class_variable_get(:@@provider).update_texts(
-            texts: [as_vector],
-            ids: [id]
-          )
-        end
-      end
-
-      # Used to serialize the DB record to an indexable vector text
-      # Overwrite this method in your model to customize
-      #
-      # @return [String] the text representation of the model
-      def as_vector
-        to_json
-      end
-
-      module ClassMethods
-        # Set the vector search provider
-        #
-        # @param provider [Object] The `Langchain::Vectorsearch::*` instance
-        def vectorsearch(provider:)
-          class_variable_set(:@@provider, provider)
-        end
-
-        # Search for similar texts
-        #
-        # @param query [String] The query to search for
-        # @param k [Integer] The number of results to return
-        # @return [ActiveRecord::Relation] The ActiveRecord relation
-        def similarity_search(query, k: 1)
-          records = class_variable_get(:@@provider).similarity_search(
-            query: query,
-            k: k
-          )
-
-          # We use "__id" when Weaviate is the provider
-          ids = records.map { |record| record.dig("id") || record.dig("__id") }
-          where(id: ids)
-        end
-
-        # Ask a question and return the answer
-        #
-        # @param question [String] The question to ask
-        # @param k [Integer] The number of results to have in context
-        # @yield [String] Stream responses back one String at a time
-        # @return [String] The answer to the question
-        def ask(question:, k: 4, &block)
-          class_variable_get(:@@provider).ask(
-            question: question,
-            k: k,
-            &block
-          )
-        end
-      end
-    end
-  end
-end

data/lib/langchain/railtie.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Langchain
-  class Railtie < Rails::Railtie
-    initializer "langchain" do
-      ActiveSupport.on_load(:active_record) do
-        ::ActiveRecord::Base.include Langchain::ActiveRecord::Hooks
-      end
-    end
-  end
-end