langchainrb 0.5.2 → 0.5.4

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.4
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-08 00:00:00.000000000 Z
+date: 2023-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tiktoken_ruby
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.1.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: google_search_results
   requirement: !ruby/object:Gem::Requirement
@@ -192,6 +192,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: hnswlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: hugging-face
   requirement: !ruby/object:Gem::Requirement
@@ -432,6 +446,7 @@ files:
 - lib/langchain/agent/sql_query_agent/sql_query_agent.rb
 - lib/langchain/agent/sql_query_agent/sql_query_agent_answer_prompt.json
 - lib/langchain/agent/sql_query_agent/sql_query_agent_sql_prompt.json
+- lib/langchain/chat.rb
 - lib/langchain/data.rb
 - lib/langchain/dependency_helper.rb
 - lib/langchain/llm/ai21.rb
@@ -462,9 +477,11 @@ files:
 - lib/langchain/tool/ruby_code_interpreter.rb
 - lib/langchain/tool/serp_api.rb
 - lib/langchain/tool/wikipedia.rb
-- lib/langchain/utils/token_length_validator.rb
+- lib/langchain/utils/token_length/google_palm_validator.rb
+- lib/langchain/utils/token_length/openai_validator.rb
 - lib/langchain/vectorsearch/base.rb
 - lib/langchain/vectorsearch/chroma.rb
+- lib/langchain/vectorsearch/hnswlib.rb
 - lib/langchain/vectorsearch/milvus.rb
 - lib/langchain/vectorsearch/pgvector.rb
 - lib/langchain/vectorsearch/pinecone.rb

data/lib/langchain/utils/token_length_validator.rb

@@ -1,84 +0,0 @@
-# frozen_string_literal: true
-
-require "tiktoken_ruby"
-
-module Langchain
-  module Utils
-    class TokenLimitExceeded < StandardError; end
-
-    class TokenLengthValidator
-      #
-      # This class is meant to validate the length of the text passed in to OpenAI's API.
-      # It is used to validate the token length before the API call is made
-      #
-      TOKEN_LIMITS = {
-        # Source:
-        # https://platform.openai.com/docs/api-reference/embeddings
-        # https://platform.openai.com/docs/models/gpt-4
-        "text-embedding-ada-002" => 8191,
-        "gpt-3.5-turbo" => 4096,
-        "gpt-3.5-turbo-0301" => 4096,
-        "text-davinci-003" => 4097,
-        "text-davinci-002" => 4097,
-        "code-davinci-002" => 8001,
-        "gpt-4" => 8192,
-        "gpt-4-0314" => 8192,
-        "gpt-4-32k" => 32768,
-        "gpt-4-32k-0314" => 32768,
-        "text-curie-001" => 2049,
-        "text-babbage-001" => 2049,
-        "text-ada-001" => 2049,
-        "davinci" => 2049,
-        "curie" => 2049,
-        "babbage" => 2049,
-        "ada" => 2049
-      }.freeze
-
-      # GOOGLE_PALM_TOKEN_LIMITS = {
-      #   "chat-bison-001" => {
-      #     "inputTokenLimit"=>4096,
-      #     "outputTokenLimit"=>1024
-      #   },
-      #   "text-bison-001" => {
-      #     "inputTokenLimit"=>8196,
-      #     "outputTokenLimit"=>1024
-      #   },
-      #   "embedding-gecko-001" => {
-      #     "inputTokenLimit"=>1024
-      #   }
-      # }.freeze
-
-      #
-      # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
-      #
-      # @param text [String] The text to validate
-      # @param model_name [String] The model name to validate against
-      # @return [Integer] Whether the text is valid or not
-      # @raise [TokenLimitExceeded] If the text is too long
-      #
-      def self.validate_max_tokens!(text, model_name)
-        text_token_length = token_length(text, model_name)
-        max_tokens = TOKEN_LIMITS[model_name] - text_token_length
-
-        # Raise an error even if whole prompt is equal to the model's token limit (max_tokens == 0) since not response will be returned
-        if max_tokens <= 0
-          raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS[model_name]} tokens, but the given text is #{text_token_length} tokens long."
-        end
-
-        max_tokens
-      end
-
-      #
-      # Calculate token length for a given text and model name
-      #
-      # @param text [String] The text to validate
-      # @param model_name [String] The model name to validate against
-      # @return [Integer] The token length of the text
-      #
-      def self.token_length(text, model_name)
-        encoder = Tiktoken.encoding_for_model(model_name)
-        encoder.encode(text).length
-      end
-    end
-  end
-end