langchainrb 0.6.18 → 0.6.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 437c6387ded139ed1a513414bfb7242cdbadf1ba6526c7a89346aa2fa9490fc2
-  data.tar.gz: dd6f437a4bbc4807a16631dd790f66c9de4e9456011b2c4f84302fe3fab1377b
+  metadata.gz: d7be5e031274fba7a4c0d7fc2cd3f472ed83fb66d8c6b355fb71fbf69a825b73
+  data.tar.gz: 745cbc4f3d7b569d2e1407acc8be123f77a0aac2964840d7c3dca215592811ee
 SHA512:
-  metadata.gz: 24748539de50dfa816fdb71173ef00a6b04f9737f32926fca919865a49b9812dd9f1fdb286c361c98e33cc994f67e8988ab688bfdf6bf3020d954eb0c791177c
-  data.tar.gz: 283b10460187cada7485e08a19c89e7485925ab2f73a5ad51b06a72e8fd9ee1600ddac9d000f13c0c1af13f6defece9fdcc272489d0df803f94da96fe1c76cfd
+  metadata.gz: e1392abe2fc0c4928593bd77d0e62688e3959ec39fd3f7bb5effc784b47599402c611ecc545868178b5d04ec688d68d6406f220697e8bfe40771cc593292a192
+  data.tar.gz: 926bccf20c71af3d31d942cf439336df9edc489a8e5e0359a6c24bb26e5b818be048a7ef63ebcce721bb99392b49e407288ffdd7387dd33d3f0161e92ff6e045

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [0.6.19] - 2023-10-18
+- Elasticsearch vector search support
+- Fix `lib/langchain/railtie.rb` not being loaded with the gem
+
 ## [0.6.18] - 2023-10-16
 - Introduce `Langchain::LLM::Response`` object
 - Introduce `Langchain::Chunk` object

data/README.md

@@ -19,11 +19,11 @@ Langchain.rb is a library that's an abstraction layer on top many emergent AI, M
 
 Install the gem and add to the application's Gemfile by executing:
 
-    $ bundle add langchainrb
+    bundle add langchainrb
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
-    $ gem install langchainrb
+    gem install langchainrb
 
 ## Usage
 
@@ -37,7 +37,7 @@ require "langchain"
 | -------- |:------------------:| -------:| -----------------:| -------:| -----------------:|
 | [Chroma](https://trychroma.com/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | :white_check_mark: |
 | [Hnswlib](https://github.com/nmslib/hnswlib/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
-| [Milvus](https://milvus.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Milvus](https://milvus.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | :white_check_mark: |
 | [Pinecone](https://www.pinecone.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | :white_check_mark: |
 | [Pgvector](https://github.com/pgvector/pgvector) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | :white_check_mark: |
 | [Qdrant](https://qdrant.tech/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | :white_check_mark: |

data/lib/langchain/vectorsearch/base.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "forwardable"
-
 module Langchain::Vectorsearch
   # = Vector Databases
   # A vector database a type of database that stores data as high-dimensional vectors, which are mathematical representations of features or attributes. Each vector has a certain number of dimensions, which can range from tens to thousands, depending on the complexity and granularity of the data.
@@ -9,11 +7,13 @@ module Langchain::Vectorsearch
   # == Available vector databases
   #
   # - {Langchain::Vectorsearch::Chroma}
+  # - {Langchain::Vectorsearch::Elasticsearch}
+  # - {Langchain::Vectorsearch::Hnswlib}
   # - {Langchain::Vectorsearch::Milvus}
+  # - {Langchain::Vectorsearch::Pgvector}
   # - {Langchain::Vectorsearch::Pinecone}
   # - {Langchain::Vectorsearch::Qdrant}
   # - {Langchain::Vectorsearch::Weaviate}
-  # - {Langchain::Vectorsearch::Pgvector}
   #
   # == Usage
   #
@@ -150,9 +150,6 @@ module Langchain::Vectorsearch
       raise NotImplementedError, "#{self.class.name} does not support asking questions"
     end
 
-    def_delegators :llm,
-      :default_dimension
-
     # HyDE-style prompt
     #
     # @param [String] User's question

data/lib/langchain/vectorsearch/chroma.rb

@@ -5,7 +5,8 @@ module Langchain::Vectorsearch
     #
     # Wrapper around Chroma DB
     #
-    # Gem requirements: gem "chroma-db", "~> 0.6.0"
+    # Gem requirements:
+    #     gem "chroma-db", "~> 0.6.0"
     #
     # Usage:
     # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, llm_api_key:, api_key: nil)
@@ -29,14 +30,15 @@ module Langchain::Vectorsearch
 
     # Add a list of texts to the index
     # @param texts [Array<String>] The list of texts to add
+    # @param ids [Array<String>] The list of ids to use for the texts (optional)
+    # @param metadatas [Array<Hash>] The list of metadata to use for the texts (optional)
     # @return [Hash] The response from the server
-    def add_texts(texts:, ids: [])
+    def add_texts(texts:, ids: [], metadatas: [])
       embeddings = Array(texts).map.with_index do |text, i|
         ::Chroma::Resources::Embedding.new(
           id: ids[i] ? ids[i].to_s : SecureRandom.uuid,
           embedding: llm.embed(text: text).embedding,
-          # TODO: Add support for passing metadata
-          metadata: {}, # metadatas[index],
+          metadata: metadatas[i] || {},
           document: text # Do we actually need to store the whole original document?
         )
       end
@@ -45,13 +47,12 @@ module Langchain::Vectorsearch
       collection.add(embeddings)
     end
 
-    def update_texts(texts:, ids:)
+    def update_texts(texts:, ids:, metadatas: [])
       embeddings = Array(texts).map.with_index do |text, i|
         ::Chroma::Resources::Embedding.new(
           id: ids[i].to_s,
           embedding: llm.embed(text: text).embedding,
-          # TODO: Add support for passing metadata
-          metadata: [], # metadatas[index],
+          metadata: metadatas[i] || {},
           document: text # Do we actually need to store the whole original document?
         )
       end

data/lib/langchain/vectorsearch/elasticsearch.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Langchain::Vectorsearch
+  class Elasticsearch < Base
+    #
+    # Wrapper around Elasticsearch vector search capabilities.
+    #
+    # Setting up Elasticsearch:
+    # 1. Get Elasticsearch up and running with Docker: https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html
+    # 2. Copy the HTTP CA certificate SHA-256 fingerprint and set the ELASTICSEARCH_CA_FINGERPRINT environment variable
+    # 3. Set the ELASTICSEARCH_URL environment variable
+    #
+    # Gem requirements:
+    #     gem "elasticsearch", "~> 8.0.0"
+    #
+    # Usage:
+    #     llm = Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
+    #     es = Langchain::Vectorsearch::Elasticsearch.new(
+    #       url: ENV["ELASTICSEARCH_URL"],
+    #       index_name: "docs",
+    #       llm: llm,
+    #       es_options: {
+    #         transport_options: {ssl: {verify: false}},
+    #         ca_fingerprint: ENV["ELASTICSEARCH_CA_FINGERPRINT"]
+    #       }
+    #     )
+    #
+    #     es.create_default_schema
+    #     es.add_texts(texts: ["..."])
+    #     es.similarity_search(text: "...")
+    #
+    attr_accessor :es_client, :index_name, :options
+
+    def initialize(url:, index_name:, llm:, api_key: nil, es_options: {})
+      require "elasticsearch"
+
+      @options = {
+        url: url,
+        request_timeout: 20,
+        log: false
+      }.merge(es_options)
+
+      @es_client = ::Elasticsearch::Client.new(**options)
+      @index_name = index_name
+
+      super(llm: llm)
+    end
+
+    def add_texts(texts: [])
+      body = texts.map do |text|
+        [
+          {index: {_index: index_name}},
+          {input: text, input_vector: llm.embed(text: text).embedding}
+        ]
+      end.flatten
+
+      es_client.bulk(body: body)
+    end
+
+    def update_texts(texts: [], ids: [])
+      body = texts.map.with_index do |text, i|
+        [
+          {index: {_index: index_name, _id: ids[i]}},
+          {input: text, input_vector: llm.embed(text: text).embedding}
+        ]
+      end.flatten
+
+      es_client.bulk(body: body)
+    end
+
+    def create_default_schema
+      es_client.indices.create(
+        index: index_name,
+        body: default_schema
+      )
+    end
+
+    def delete_default_schema
+      es_client.indices.delete(
+        index: index_name
+      )
+    end
+
+    def default_vector_settings
+      {type: "dense_vector", dims: llm.default_dimension}
+    end
+
+    def vector_settings
+      options[:vector_settings] || default_vector_settings
+    end
+
+    def default_schema
+      {
+        mappings: {
+          properties: {
+            input: {
+              type: "text"
+            },
+            input_vector: vector_settings
+          }
+        }
+      }
+    end
+
+    def default_query(query_vector)
+      {
+        script_score: {
+          query: {match_all: {}},
+          script: {
+            source: "cosineSimilarity(params.query_vector, 'input_vector') + 1.0",
+            params: {
+              query_vector: query_vector
+            }
+          }
+        }
+      }
+    end
+
+    # TODO: Implement this
+    # def ask()
+    # end
+
+    def similarity_search(text: "", k: 10, query: {})
+      if text.empty? && query.empty?
+        raise "Either text or query should pass as an argument"
+      end
+
+      if query.empty?
+        query_vector = llm.embed(text: text).embedding
+
+        query = default_query(query_vector)
+      end
+
+      es_client.search(body: {query: query, size: k}).body
+    end
+
+    def similarity_search_by_vector(embedding: [], k: 10, query: {})
+      if embedding.empty? && query.empty?
+        raise "Either embedding or query should pass as an argument"
+      end
+
+      query = default_query(embedding) if query.empty?
+
+      es_client.search(body: {query: query, size: k}).body
+    end
+  end
+end

data/lib/langchain/vectorsearch/milvus.rb

@@ -5,7 +5,8 @@ module Langchain::Vectorsearch
     #
     # Wrapper around Milvus REST APIs.
     #
-    # Gem requirements: gem "milvus", "~> 0.9.2"
+    # Gem requirements:
+    #     gem "milvus", "~> 0.9.2"
     #
     # Usage:
     # milvus = Langchain::Vectorsearch::Milvus.new(url:, index_name:, llm:, api_key:)
@@ -70,7 +71,7 @@ module Langchain::Vectorsearch
             type_params: [
               {
                 key: "dim",
-                value: default_dimension.to_s
+                value: llm.default_dimension.to_s
               }
             ]
           }

data/lib/langchain/vectorsearch/pgvector.rb

@@ -5,10 +5,12 @@ module Langchain::Vectorsearch
     #
     # The PostgreSQL vector search adapter
     #
-    # Gem requirements: gem "pgvector", "~> 0.2"
+    # Gem requirements:
+    #     gem "sequel", "~> 5.68.0"
+    #     gem "pgvector", "~> 0.2"
     #
     # Usage:
-    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:, namespace: nil)
+    #     pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:, namespace: nil)
     #
 
     # The operators supported by the PostgreSQL vector search adapter
@@ -91,7 +93,7 @@ module Langchain::Vectorsearch
     def create_default_schema
       db.run "CREATE EXTENSION IF NOT EXISTS vector"
       namespace_column = @namespace_column
-      vector_dimension = default_dimension
+      vector_dimension = llm.default_dimension
       db.create_table? table_name.to_sym do
         primary_key :id
         text :content
@@ -150,3 +152,7 @@ module Langchain::Vectorsearch
     end
   end
 end
+
+# Rails connection when configuring vectorsearch
+# Update READMEs
+# Rails migration to create a migration

data/lib/langchain/vectorsearch/pinecone.rb

@@ -5,10 +5,11 @@ module Langchain::Vectorsearch
     #
     # Wrapper around Pinecone API.
     #
-    # Gem requirements: gem "pinecone", "~> 0.1.6"
+    # Gem requirements:
+    #     gem "pinecone", "~> 0.1.6"
     #
     # Usage:
-    # pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:, llm_api_key:)
+    #     pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
     # Initialize the Pinecone client
@@ -102,7 +103,7 @@ module Langchain::Vectorsearch
       client.create_index(
         metric: DEFAULT_METRIC,
         name: index_name,
-        dimension: default_dimension
+        dimension: llm.default_dimension
       )
     end
 

data/lib/langchain/vectorsearch/qdrant.rb

@@ -5,10 +5,11 @@ module Langchain::Vectorsearch
     #
     # Wrapper around Qdrant
     #
-    # Gem requirements: gem "qdrant-ruby", "~> 0.9.3"
+    # Gem requirements:
+    #     gem "qdrant-ruby", "~> 0.9.3"
     #
     # Usage:
-    # qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:, llm_api_key:)
+    #     qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
     # Initialize the Qdrant client
@@ -82,7 +83,7 @@ module Langchain::Vectorsearch
         collection_name: index_name,
         vectors: {
           distance: DEFAULT_METRIC.capitalize,
-          size: default_dimension
+          size: llm.default_dimension
         }
       )
     end

data/lib/langchain/vectorsearch/weaviate.rb

@@ -5,10 +5,11 @@ module Langchain::Vectorsearch
     #
     # Wrapper around Weaviate
     #
-    # Gem requirements: gem "weaviate-ruby", "~> 0.8.3"
+    # Gem requirements:
+    #     gem "weaviate-ruby", "~> 0.8.3"
     #
     # Usage:
-    # weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:, llm_api_key:)
+    #     weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
     # Initialize the Weaviate adapter

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.6.18"
+  VERSION = "0.6.19"
 end

data/lib/langchain.rb

@@ -89,3 +89,5 @@ module Langchain
     class BaseError < StandardError; end
   end
 end
+
+require "langchain/railtie" if defined?(Rails)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.6.18
+  version: 0.6.19
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-10-17 00:00:00.000000000 Z
+date: 2023-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: baran
@@ -220,6 +220,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 8.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 8.2.0
 - !ruby/object:Gem::Dependency
   name: eqn
   requirement: !ruby/object:Gem::Requirement
@@ -603,6 +617,7 @@ files:
 - lib/langchain/utils/token_length/token_limit_exceeded.rb
 - lib/langchain/vectorsearch/base.rb
 - lib/langchain/vectorsearch/chroma.rb
+- lib/langchain/vectorsearch/elasticsearch.rb
 - lib/langchain/vectorsearch/hnswlib.rb
 - lib/langchain/vectorsearch/milvus.rb
 - lib/langchain/vectorsearch/pgvector.rb