langchainrb 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ff52d2013c31933c6c12e3b824db979122decf0f7a4be2a99ec22da22ecb4f3
-  data.tar.gz: f632e80bd634e1ff899216498a13e8dc24073d435754077612f8753f51c9fbcf
+  metadata.gz: 87647b8a7e2dc49359f3f6d655eda501dcac26ebdd14247ad6c583be8dc1a71c
+  data.tar.gz: fb7b4321caa4ff026439158f5ccfc2ae9e7b515a69c35cba87385f2cb367fa85
 SHA512:
-  metadata.gz: e79f3c438f18b90d6f926088a613dfafdf7009e3c9d768adcb4ba097f688731073af1a983ddbeb754e479d82ca34492c085a05d9bb6d2ddcd4996a5be418d03c
-  data.tar.gz: 32682afc3ce3d3ce63d351ff10695d97f46f19b725f975ed6901b4970f6254a95c50fc3575bfc3a3003c95c0cd5a955e1ccdbb17e062cbcf65c1fa9f541a286a
+  metadata.gz: 4f80677c43c00e6d50e0494aa79cb7648b9f3878ed8d2a5f4f2dc90e308a3639589f8457a4615821b70b44c5a43ae4f26fcf00d7548684740e4c05dbcc165bf8
+  data.tar.gz: 4722233dbed83d21f2dadff19a9b79a30d8fd208d6e30bd057f018060c602b7f00f0526ee0364823597806388e2a8da48e883a5e6fa77b31490199685644b4d2

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ## [Unreleased]
 
+## [0.5.4] - 2023-06-10
+- 🔍 Vectorsearch
+  - Introducing support for HNSWlib
+- Improved and new `Langchain::Chat` interface that persists chat history in memory
+
 ## [0.5.3] - 2023-06-09
 - 🗣️ LLMs
   - Chat message history support for Langchain::LLM::GooglePalm and Langchain::LLM::OpenAI

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.5.3)
+    langchainrb (0.5.4)
       colorize (~> 0.8.1)
       tiktoken_ruby (~> 0.0.5)
 
@@ -135,6 +135,7 @@ GEM
       activesupport (>= 3.0)
       graphql
     hashery (2.1.2)
+    hnswlib (0.8.1)
     httparty (0.21.0)
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
@@ -312,6 +313,7 @@ DEPENDENCIES
   eqn (~> 1.6.5)
   google_palm_api (~> 0.1.1)
   google_search_results (~> 2.0.0)
+  hnswlib (~> 0.8.1)
   hugging-face (~> 0.3.4)
   langchainrb!
   milvus (~> 0.9.0)

data/README.md

@@ -34,6 +34,7 @@ require "langchain"
 | Database | Querying           | Storage | Schema Management | Backups | Rails Integration |
 | -------- |:------------------:| -------:| -----------------:| -------:| -----------------:|
 | [Chroma](https://trychroma.com/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Hnswlib](https://github.com/nmslib/hnswlib/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Milvus](https://milvus.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Pinecone](https://www.pinecone.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Pgvector](https://github.com/pgvector/pgvector) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
@@ -56,11 +57,12 @@ client = Langchain::Vectorsearch::Weaviate.new(
 )
 
 # You can instantiate any other supported vector search database:
+client = Langchain::Vectorsearch::Chroma.new(...) # `gem "chroma-db", "~> 0.3.0"`
+client = Langchain::Vectorsearch::Hnswlib.new(...) # `gem "hnswlib", "~> 0.8.1"`
 client = Langchain::Vectorsearch::Milvus.new(...) # `gem "milvus", "~> 0.9.0"`
-client = Langchain::Vectorsearch::Qdrant.new(...) # `gem"qdrant-ruby", "~> 0.9.0"`
 client = Langchain::Vectorsearch::Pinecone.new(...) # `gem "pinecone", "~> 0.1.6"`
-client = Langchain::Vectorsearch::Chroma.new(...) # `gem "chroma-db", "~> 0.3.0"`
 client = Langchain::Vectorsearch::Pgvector.new(...) # `gem "pgvector", "~> 0.2"`
+client = Langchain::Vectorsearch::Qdrant.new(...) # `gem"qdrant-ruby", "~> 0.9.0"`
 ```
 
 ```ruby

data/lib/langchain/agent/chain_of_thought_agent/chain_of_thought_agent.rb

@@ -101,7 +101,7 @@ module Langchain::Agent
         tool_names: "[#{tool_list.join(", ")}]",
         tools: tools.map do |tool|
           tool_name = tool.tool_name
-          tool_description = tool.class.const_get(:DESCRIPTION)
+          tool_description = tool.tool_description
           "#{tool_name}: #{tool_description}"
         end.join("\n")
       )

data/lib/langchain/chat.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Langchain
+  class Chat
+    attr_reader :context
+
+    def initialize(llm:, **options)
+      @llm = llm
+      @context = nil
+      @examples = []
+      @messages = []
+    end
+
+    # Set the context of the conversation. Usually used to set the model's persona.
+    # @param message [String] The context of the conversation
+    def set_context(message)
+      @context = message
+    end
+
+    # Add examples to the conversation. Used to give the model a sense of the conversation.
+    # @param examples [Array<Hash>] The examples to add to the conversation
+    def add_examples(examples)
+      @examples.concat examples
+    end
+
+    # Message the model with a prompt and return the response.
+    # @param message [String] The prompt to message the model with
+    # @return [String] The response from the model
+    def message(message)
+      append_user_message(message)
+      response = llm_response(message)
+      append_ai_message(response)
+      response
+    end
+
+    private
+
+    def llm_response(prompt)
+      @llm.chat(messages: @messages, context: @context, examples: @examples)
+    end
+
+    def append_ai_message(message)
+      @messages << {role: "ai", content: message}
+    end
+
+    def append_user_message(message)
+      @messages << {role: "user", content: message}
+    end
+  end
+end

data/lib/langchain/llm/google_palm.rb

@@ -80,28 +80,31 @@ module Langchain::LLM
     # @param params extra parameters passed to GooglePalmAPI::Client#generate_chat_message
     # @return [String] The chat completion
     #
-    def chat(prompt: "", messages: [], **params)
+    def chat(prompt: "", messages: [], context: "", examples: [], **options)
       raise ArgumentError.new(":prompt or :messages argument is expected") if prompt.empty? && messages.empty?
 
-      messages << {author: "0", content: prompt} if !prompt.empty?
-
-      # TODO: Figure out how to introduce persisted conversations
       default_params = {
         temperature: DEFAULTS[:temperature],
-        messages: messages
+        context: context,
+        messages: compose_chat_messages(prompt: prompt, messages: messages),
+        examples: compose_examples(examples)
       }
 
-      if params[:stop_sequences]
-        default_params[:stop] = params.delete(:stop_sequences)
+      Langchain::Utils::TokenLength::GooglePalmValidator.validate_max_tokens!(self, default_params[:messages], "chat-bison-001")
+
+      if options[:stop_sequences]
+        default_params[:stop] = options.delete(:stop_sequences)
       end
 
-      if params[:max_tokens]
-        default_params[:max_output_tokens] = params.delete(:max_tokens)
+      if options[:max_tokens]
+        default_params[:max_output_tokens] = options.delete(:max_tokens)
       end
 
-      default_params.merge!(params)
+      default_params.merge!(options)
 
       response = client.generate_chat_message(**default_params)
+      raise "GooglePalm API returned an error: #{response}" if response.dig("error")
+
       response.dig("candidates", 0, "content")
     end
 
@@ -124,5 +127,39 @@ module Langchain::LLM
         max_tokens: 2048
       )
     end
+
+    private
+
+    def compose_chat_messages(prompt:, messages:)
+      history = []
+      history.concat transform_messages(messages) unless messages.empty?
+
+      unless prompt.empty?
+        if history.last && history.last[:role] == "user"
+          history.last[:content] += "\n#{prompt}"
+        else
+          history.append({author: "user", content: prompt})
+        end
+      end
+      history
+    end
+
+    def compose_examples(examples)
+      examples.each_slice(2).map do |example|
+        {
+          input: {content: example.first[:content]},
+          output: {content: example.last[:content]}
+        }
+      end
+    end
+
+    def transform_messages(messages)
+      messages.map do |message|
+        {
+          author: message[:role],
+          content: message[:content]
+        }
+      end
+    end
   end
 end

data/lib/langchain/llm/openai.rb

@@ -35,7 +35,7 @@ module Langchain::LLM
     def embed(text:, **params)
       parameters = {model: DEFAULTS[:embeddings_model_name], input: text}
 
-      Langchain::Utils::TokenLengthValidator.validate_max_tokens!(text, parameters[:model])
+      Langchain::Utils::TokenLength::OpenAIValidator.validate_max_tokens!(text, parameters[:model])
 
       response = client.embeddings(parameters: parameters.merge(params))
       response.dig("data").first.dig("embedding")
@@ -52,7 +52,7 @@ module Langchain::LLM
       parameters = compose_parameters DEFAULTS[:completion_model_name], params
 
       parameters[:prompt] = prompt
-      parameters[:max_tokens] = Langchain::Utils::TokenLengthValidator.validate_max_tokens!(prompt, parameters[:model])
+      parameters[:max_tokens] = Langchain::Utils::TokenLength::OpenAIValidator.validate_max_tokens!(prompt, parameters[:model])
 
       response = client.completions(parameters: parameters)
       response.dig("choices", 0, "text")
@@ -63,19 +63,22 @@ module Langchain::LLM
     #
     # @param prompt [String] The prompt to generate a chat completion for
     # @param messages [Array] The messages that have been sent in the conversation
-    # @param params extra parameters passed to OpenAI::Client#chat
+    # @param context [String] The context of the conversation
+    # @param examples [Array] Examples of messages provide model with
+    # @param options extra parameters passed to OpenAI::Client#chat
     # @return [String] The chat completion
     #
-    def chat(prompt: "", messages: [], **params)
+    def chat(prompt: "", messages: [], context: "", examples: [], **options)
       raise ArgumentError.new(":prompt or :messages argument is expected") if prompt.empty? && messages.empty?
 
-      messages << {role: "user", content: prompt} if !prompt.empty?
-
-      parameters = compose_parameters DEFAULTS[:chat_completion_model_name], params
-      parameters[:messages] = messages
-      parameters[:max_tokens] = validate_max_tokens(messages, parameters[:model])
+      parameters = compose_parameters DEFAULTS[:chat_completion_model_name], options
+      parameters[:messages] = compose_chat_messages(prompt: prompt, messages: messages, context: context, examples: examples)
+      parameters[:max_tokens] = validate_max_tokens(parameters[:messages], parameters[:model])
 
       response = client.chat(parameters: parameters)
+
+      raise "Chat completion failed: #{response}" if response.dig("error")
+
       response.dig("choices", 0, "message", "content")
     end
 
@@ -104,8 +107,40 @@ module Langchain::LLM
       default_params.merge(params)
     end
 
+    def compose_chat_messages(prompt:, messages:, context:, examples:)
+      history = []
+
+      history.concat transform_messages(examples) unless examples.empty?
+
+      history.concat transform_messages(messages) unless messages.empty?
+
+      unless context.nil? || context.empty?
+        history.reject! { |message| message[:role] == "system" }
+        history.prepend({role: "system", content: context})
+      end
+
+      unless prompt.empty?
+        if history.last && history.last[:role] == "user"
+          history.last[:content] += "\n#{prompt}"
+        else
+          history.append({role: "user", content: prompt})
+        end
+      end
+
+      history
+    end
+
+    def transform_messages(messages)
+      messages.map do |message|
+        {
+          content: message[:content],
+          role: (message[:role] == "ai") ? "assistant" : message[:role]
+        }
+      end
+    end
+
     def validate_max_tokens(messages, model)
-      Langchain::Utils::TokenLengthValidator.validate_max_tokens!(messages, model)
+      Langchain::Utils::TokenLength::OpenAIValidator.validate_max_tokens!(messages, model)
     end
   end
 end

data/lib/langchain/tool/base.rb

@@ -57,6 +57,15 @@ module Langchain::Tool
       self.class.const_get(:NAME)
     end
 
+    #
+    # Returns the DESCRIPTION constant of the tool
+    #
+    # @return [String] tool description
+    #
+    def tool_description
+      self.class.const_get(:DESCRIPTION)
+    end
+
     #
     # Sets the DESCRIPTION constant of the tool
     #

data/lib/langchain/utils/token_length/google_palm_validator.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Utils
+    module TokenLength
+      #
+      # This class is meant to validate the length of the text passed in to Google Palm's API.
+      # It is used to validate the token length before the API call is made
+      #
+      class GooglePalmValidator
+        TOKEN_LIMITS = {
+          # Source:
+          # This data can be pulled when `list_models()` method is called: https://github.com/andreibondarev/google_palm_api#usage
+
+          # chat-bison-001 is the only model that currently supports countMessageTokens functions
+          "chat-bison-001" => {
+            "input_token_limit" => 4000, # 4096 is the limit but the countMessageTokens does not return anything higher than 4000
+            "output_token_limit" => 1024
+          }
+          # "text-bison-001" => {
+          #   "input_token_limit" => 8196,
+          #   "output_token_limit" => 1024
+          # },
+          # "embedding-gecko-001" => {
+          #   "input_token_limit" => 1024
+          # }
+        }.freeze
+
+        #
+        # Validate the context length of the text
+        #
+        # @param content [String | Array<String>] The text or array of texts to validate
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] Whether the text is valid or not
+        # @raise [TokenLimitExceeded] If the text is too long
+        #
+        def self.validate_max_tokens!(google_palm_llm, content, model_name)
+          text_token_length = if content.is_a?(Array)
+            content.sum { |item| token_length(google_palm_llm, item.to_json, model_name) }
+          else
+            token_length(google_palm_llm, content, model_name)
+          end
+
+          leftover_tokens = TOKEN_LIMITS.dig(model_name, "input_token_limit") - text_token_length
+
+          # Raise an error even if whole prompt is equal to the model's token limit (leftover_tokens == 0)
+          if leftover_tokens <= 0
+            raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS.dig(model_name, "input_token_limit")} tokens, but the given text is #{text_token_length} tokens long."
+          end
+
+          leftover_tokens
+        end
+
+        #
+        # Calculate token length for a given text and model name
+        #
+        # @param llm [Langchain::LLM:GooglePalm] The Langchain::LLM:GooglePalm instance
+        # @param text [String] The text to calculate the token length for
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] The token length of the text
+        #
+        def self.token_length(llm, text, model_name = "chat-bison-001")
+          response = llm.client.count_message_tokens(model: model_name, prompt: text)
+          response.dig("tokenCount")
+        end
+      end
+    end
+  end
+end

data/lib/langchain/utils/token_length/openai_validator.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "tiktoken_ruby"
+
+module Langchain
+  module Utils
+    module TokenLength
+      #
+      # This class is meant to validate the length of the text passed in to OpenAI's API.
+      # It is used to validate the token length before the API call is made
+      #
+      class OpenAIValidator
+        TOKEN_LIMITS = {
+          # Source:
+          # https://platform.openai.com/docs/api-reference/embeddings
+          # https://platform.openai.com/docs/models/gpt-4
+          "text-embedding-ada-002" => 8191,
+          "gpt-3.5-turbo" => 4096,
+          "gpt-3.5-turbo-0301" => 4096,
+          "text-davinci-003" => 4097,
+          "text-davinci-002" => 4097,
+          "code-davinci-002" => 8001,
+          "gpt-4" => 8192,
+          "gpt-4-0314" => 8192,
+          "gpt-4-32k" => 32768,
+          "gpt-4-32k-0314" => 32768,
+          "text-curie-001" => 2049,
+          "text-babbage-001" => 2049,
+          "text-ada-001" => 2049,
+          "davinci" => 2049,
+          "curie" => 2049,
+          "babbage" => 2049,
+          "ada" => 2049
+        }.freeze
+
+        #
+        # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
+        #
+        # @param content [String | Array<String>] The text or array of texts to validate
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] Whether the text is valid or not
+        # @raise [TokenLimitExceeded] If the text is too long
+        #
+        def self.validate_max_tokens!(content, model_name)
+          text_token_length = if content.is_a?(Array)
+            content.sum { |item| token_length(item.to_json, model_name) }
+          else
+            token_length(content, model_name)
+          end
+
+          max_tokens = TOKEN_LIMITS[model_name] - text_token_length
+
+          # Raise an error even if whole prompt is equal to the model's token limit (max_tokens == 0) since not response will be returned
+          if max_tokens <= 0
+            raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS[model_name]} tokens, but the given text is #{text_token_length} tokens long."
+          end
+
+          max_tokens
+        end
+
+        #
+        # Calculate token length for a given text and model name
+        #
+        # @param text [String] The text to calculate the token length for
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] The token length of the text
+        #
+        def self.token_length(text, model_name)
+          encoder = Tiktoken.encoding_for_model(model_name)
+          encoder.encode(text).length
+        end
+      end
+    end
+  end
+end

data/lib/langchain/vectorsearch/chroma.rb

@@ -8,7 +8,7 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "chroma-db", "~> 0.3.0"
     #
     # Usage:
-    # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, api_key: nil)
+    # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, llm_api_key:, api_key: nil)
     #
 
     # Initialize the Chroma client

data/lib/langchain/vectorsearch/hnswlib.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Langchain::Vectorsearch
+  class Hnswlib < Base
+    #
+    # Wrapper around HNSW (Hierarchical Navigable Small World) library.
+    # HNSWLib is an in-memory vectorstore that can be saved to a file on disk.
+    #
+    # Gem requirements:
+    #     gem "hnswlib", "~> 0.8.1"
+    #
+    # Usage:
+    #     hnsw = Langchain::Vectorsearch::Hnswlib.new(llm:, url:, index_name:)
+    #
+
+    attr_reader :client, :path_to_index
+
+    #
+    # Initialize the HNSW vector search
+    #
+    # @param llm [Object] The LLM client to use
+    # @param path_to_index [String] The local path to the index file, e.g.: "/storage/index.ann"
+    # @return [Langchain::Vectorsearch::Hnswlib] Class instance
+    #
+    def initialize(llm:, path_to_index:)
+      depends_on "hnswlib"
+      require "hnswlib"
+
+      super(llm: llm)
+
+      @client = ::Hnswlib::HierarchicalNSW.new(space: DEFAULT_METRIC, dim: llm.default_dimension)
+      @path_to_index = path_to_index
+
+      initialize_index
+    end
+
+    #
+    # Add a list of texts and corresponding IDs to the index
+    #
+    # @param texts [Array] The list of texts to add
+    # @param ids [Array] The list of corresponding IDs (integers) to the texts
+    # @return [Boolean] The response from the HNSW library
+    #
+    def add_texts(texts:, ids:)
+      resize_index(texts.size)
+
+      Array(texts).each_with_index do |text, i|
+        embedding = llm.embed(text: text)
+
+        client.add_point(embedding, ids[i])
+      end
+
+      client.save_index(path_to_index)
+    end
+
+    #
+    # Search for similar texts
+    #
+    # @param query [String] The text to search for
+    # @param k [Integer] The number of results to return
+    # @return [Array] Results in the format `[[id1, distance3], [id2, distance2]]`
+    #
+    def similarity_search(
+      query:,
+      k: 4
+    )
+      embedding = llm.embed(text: query)
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    #
+    # Search for the K nearest neighbors of a given vector
+    #
+    # @param embedding [Array] The embedding to search for
+    # @param k [Integer] The number of results to return
+    # @return [Array] Results in the format `[[id1, distance3], [id2, distance2]]`
+    #
+    def similarity_search_by_vector(
+      embedding:,
+      k: 4
+    )
+      client.search_knn(embedding, k)
+    end
+
+    private
+
+    #
+    # Optionally resizes the index if there's no space for new data
+    #
+    # @param num_of_elements_to_add [Integer] The number of elements to add to the index
+    #
+    def resize_index(num_of_elements_to_add)
+      current_count = client.current_count
+
+      if (current_count + num_of_elements_to_add) > client.max_elements
+        new_size = current_count + num_of_elements_to_add
+
+        client.resize_index(new_size)
+      end
+    end
+
+    #
+    # Loads or initializes the new index
+    #
+    def initialize_index
+      if File.exist?(path_to_index)
+        client.load_index(path_to_index)
+
+        Langchain.logger.info("[#{self.class.name}]".blue + ": Successfully loaded the index at \"#{path_to_index}\"")
+      else
+        # Default max_elements: 100, but we constantly resize the index as new data is written to it
+        client.init_index(max_elements: 100)
+
+        Langchain.logger.info("[#{self.class.name}]".blue + ": Creating a new index at \"#{path_to_index}\"")
+      end
+    end
+  end
+end

data/lib/langchain/vectorsearch/milvus.rb

@@ -8,17 +8,9 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "milvus", "~> 0.9.0"
     #
     # Usage:
-    # milvus = Langchain::Vectorsearch::Milvus.new(url:, index_name:, llm:)
+    # milvus = Langchain::Vectorsearch::Milvus.new(url:, index_name:, llm:, llm_api_key:)
     #
 
-    #
-    # Initialize the Milvus client
-    #
-    # @param url [String] The URL of the Milvus server
-    # @param api_key [String] The API key to use
-    # @param index_name [String] The name of the index to use
-    # @param llm [Object] The LLM client to use
-    #
     def initialize(url:, index_name:, llm:, api_key: nil)
       depends_on "milvus"
       require "milvus"
@@ -29,11 +21,6 @@ module Langchain::Vectorsearch
       super(llm: llm)
     end
 
-    #
-    # Add a list of texts to the index
-    #
-    # @param texts [Array] The list of texts to add
-    #
     def add_texts(texts:)
       client.entities.insert(
         collection_name: index_name,

data/lib/langchain/vectorsearch/pgvector.rb

@@ -8,7 +8,7 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "pgvector", "~> 0.2"
     #
     # Usage:
-    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:)
+    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:, llm_api_key:)
     #
 
     # The operators supported by the PostgreSQL vector search adapter
@@ -20,14 +20,10 @@ module Langchain::Vectorsearch
 
     attr_reader :operator, :quoted_table_name
 
-    #
-    # Initialize the PostgreSQL client
-    #
     # @param url [String] The URL of the PostgreSQL database
     # @param index_name [String] The name of the table to use for the index
     # @param llm [Object] The LLM client to use
     # @param api_key [String] The API key for the Vectorsearch DB (not used for PostgreSQL)
-    #
     def initialize(url:, index_name:, llm:, api_key: nil)
       require "pg"
       require "pgvector"

data/lib/langchain/vectorsearch/pinecone.rb

@@ -8,17 +8,14 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "pinecone", "~> 0.1.6"
     #
     # Usage:
-    # pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:)
+    # pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
-    #
     # Initialize the Pinecone client
-    #
     # @param environment [String] The environment to use
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
-    #
     def initialize(environment:, api_key:, index_name:, llm:)
       depends_on "pinecone"
       require "pinecone"

data/lib/langchain/vectorsearch/qdrant.rb

@@ -8,17 +8,14 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "qdrant-ruby", "~> 0.9.0"
     #
     # Usage:
-    # qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:)
+    # qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
-    #
     # Initialize the Qdrant client
-    #
     # @param url [String] The URL of the Qdrant server
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
-    #
     def initialize(url:, api_key:, index_name:, llm:)
       depends_on "qdrant-ruby"
       require "qdrant"

data/lib/langchain/vectorsearch/weaviate.rb

@@ -8,17 +8,14 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "weaviate-ruby", "~> 0.8.0"
     #
     # Usage:
-    # weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:)
+    # weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
-    #
     # Initialize the Weaviate adapter
-    #
     # @param url [String] The URL of the Weaviate instance
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
-    #
     def initialize(url:, api_key:, index_name:, llm:)
       depends_on "weaviate-ruby"
       require "weaviate"

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.5.3"
+  VERSION = "0.5.4"
 end

data/lib/langchain.rb

@@ -62,6 +62,7 @@ module Langchain
 
   autoload :Loader, "langchain/loader"
   autoload :Data, "langchain/data"
+  autoload :Chat, "langchain/chat"
   autoload :DependencyHelper, "langchain/dependency_helper"
 
   module Agent
@@ -92,12 +93,18 @@ module Langchain
   end
 
   module Utils
-    autoload :TokenLengthValidator, "langchain/utils/token_length_validator"
+    module TokenLength
+      class TokenLimitExceeded < StandardError; end
+
+      autoload :OpenAIValidator, "langchain/utils/token_length/openai_validator"
+      autoload :GooglePalmValidator, "langchain/utils/token_length/google_palm_validator"
+    end
   end
 
   module Vectorsearch
     autoload :Base, "langchain/vectorsearch/base"
     autoload :Chroma, "langchain/vectorsearch/chroma"
+    autoload :Hnswlib, "langchain/vectorsearch/hnswlib"
     autoload :Milvus, "langchain/vectorsearch/milvus"
     autoload :Pinecone, "langchain/vectorsearch/pinecone"
     autoload :Pgvector, "langchain/vectorsearch/pgvector"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - Andrei Bondarev
@@ -192,6 +192,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: hnswlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: hugging-face
   requirement: !ruby/object:Gem::Requirement
@@ -432,6 +446,7 @@ files:
 - lib/langchain/agent/sql_query_agent/sql_query_agent.rb
 - lib/langchain/agent/sql_query_agent/sql_query_agent_answer_prompt.json
 - lib/langchain/agent/sql_query_agent/sql_query_agent_sql_prompt.json
+- lib/langchain/chat.rb
 - lib/langchain/data.rb
 - lib/langchain/dependency_helper.rb
 - lib/langchain/llm/ai21.rb
@@ -462,9 +477,11 @@ files:
 - lib/langchain/tool/ruby_code_interpreter.rb
 - lib/langchain/tool/serp_api.rb
 - lib/langchain/tool/wikipedia.rb
-- lib/langchain/utils/token_length_validator.rb
+- lib/langchain/utils/token_length/google_palm_validator.rb
+- lib/langchain/utils/token_length/openai_validator.rb
 - lib/langchain/vectorsearch/base.rb
 - lib/langchain/vectorsearch/chroma.rb
+- lib/langchain/vectorsearch/hnswlib.rb
 - lib/langchain/vectorsearch/milvus.rb
 - lib/langchain/vectorsearch/pgvector.rb
 - lib/langchain/vectorsearch/pinecone.rb
@@ -495,7 +512,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: Build LLM-backed Ruby applications with Ruby's LangChain

data/lib/langchain/utils/token_length_validator.rb

@@ -1,89 +0,0 @@
-# frozen_string_literal: true
-
-require "tiktoken_ruby"
-
-module Langchain
-  module Utils
-    class TokenLimitExceeded < StandardError; end
-
-    class TokenLengthValidator
-      #
-      # This class is meant to validate the length of the text passed in to OpenAI's API.
-      # It is used to validate the token length before the API call is made
-      #
-      TOKEN_LIMITS = {
-        # Source:
-        # https://platform.openai.com/docs/api-reference/embeddings
-        # https://platform.openai.com/docs/models/gpt-4
-        "text-embedding-ada-002" => 8191,
-        "gpt-3.5-turbo" => 4096,
-        "gpt-3.5-turbo-0301" => 4096,
-        "text-davinci-003" => 4097,
-        "text-davinci-002" => 4097,
-        "code-davinci-002" => 8001,
-        "gpt-4" => 8192,
-        "gpt-4-0314" => 8192,
-        "gpt-4-32k" => 32768,
-        "gpt-4-32k-0314" => 32768,
-        "text-curie-001" => 2049,
-        "text-babbage-001" => 2049,
-        "text-ada-001" => 2049,
-        "davinci" => 2049,
-        "curie" => 2049,
-        "babbage" => 2049,
-        "ada" => 2049
-      }.freeze
-
-      # GOOGLE_PALM_TOKEN_LIMITS = {
-      #   "chat-bison-001" => {
-      #     "inputTokenLimit"=>4096,
-      #     "outputTokenLimit"=>1024
-      #   },
-      #   "text-bison-001" => {
-      #     "inputTokenLimit"=>8196,
-      #     "outputTokenLimit"=>1024
-      #   },
-      #   "embedding-gecko-001" => {
-      #     "inputTokenLimit"=>1024
-      #   }
-      # }.freeze
-
-      #
-      # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
-      #
-      # @param content [String | Array<String>] The text or array of texts to validate
-      # @param model_name [String] The model name to validate against
-      # @return [Integer] Whether the text is valid or not
-      # @raise [TokenLimitExceeded] If the text is too long
-      #
-      def self.validate_max_tokens!(content, model_name)
-        text_token_length = if content.is_a?(Array)
-          content.sum { |item| token_length(item.to_json, model_name) }
-        else
-          token_length(content, model_name)
-        end
-
-        max_tokens = TOKEN_LIMITS[model_name] - text_token_length
-
-        # Raise an error even if whole prompt is equal to the model's token limit (max_tokens == 0) since not response will be returned
-        if max_tokens <= 0
-          raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS[model_name]} tokens, but the given text is #{text_token_length} tokens long."
-        end
-
-        max_tokens
-      end
-
-      #
-      # Calculate token length for a given text and model name
-      #
-      # @param text [String] The text to validate
-      # @param model_name [String] The model name to validate against
-      # @return [Integer] The token length of the text
-      #
-      def self.token_length(text, model_name)
-        encoder = Tiktoken.encoding_for_model(model_name)
-        encoder.encode(text).length
-      end
-    end
-  end
-end