langchainrb 0.5.3 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ff52d2013c31933c6c12e3b824db979122decf0f7a4be2a99ec22da22ecb4f3
-  data.tar.gz: f632e80bd634e1ff899216498a13e8dc24073d435754077612f8753f51c9fbcf
+  metadata.gz: 9781999daf45e5fedb0c7a905268866fbefd4581fd35a1a512ebb5844598f2c7
+  data.tar.gz: 93e2161a331151218cb94706827ab1ca2d94cb363613a5117ef7ce0c36cb9469
 SHA512:
-  metadata.gz: e79f3c438f18b90d6f926088a613dfafdf7009e3c9d768adcb4ba097f688731073af1a983ddbeb754e479d82ca34492c085a05d9bb6d2ddcd4996a5be418d03c
-  data.tar.gz: 32682afc3ce3d3ce63d351ff10695d97f46f19b725f975ed6901b4970f6254a95c50fc3575bfc3a3003c95c0cd5a955e1ccdbb17e062cbcf65c1fa9f541a286a
+  metadata.gz: 58255ecc90b645cf6b276bee83a385d567122fb8b90ad7a075f4b2ec90ba2f6871156c6e5762b0bd67b259371d1c35b3973fd6b4e631be858ea8a368aac163b7
+  data.tar.gz: ffcaba1dc980b1f3175269a223ebe522bae1a3e17931b1f911adf387f8fc4a32692d45712ab0032e5a9c273ba304d97ca0ce2086978381d17e716efa1941072a

data/.env.example

@@ -5,6 +5,7 @@ HUGGING_FACE_API_KEY=
 MILVUS_URL=
 OPENAI_API_KEY=
 GOOGLE_PALM_API_KEY=
+OPEN_WEATHER_API_KEY=
 PINECONE_API_KEY=
 PINECONE_ENVIRONMENT=
 POSTGRES_URL=

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [0.5.5] - 2023-06-12
+- [BREAKING] Rename `Langchain::Chat` to `Langchain::Conversation`
+- 🛠️ Tools
+  - Introducing `Langchain::Tool::Weather`, a tool that calls Open Weather API to retrieve the current weather
+
+## [0.5.4] - 2023-06-10
+- 🔍 Vectorsearch
+  - Introducing support for HNSWlib
+- Improved and new `Langchain::Chat` interface that persists chat history in memory
+
 ## [0.5.3] - 2023-06-09
 - 🗣️ LLMs
   - Chat message history support for Langchain::LLM::GooglePalm and Langchain::LLM::OpenAI

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.5.3)
+    langchainrb (0.5.5)
       colorize (~> 0.8.1)
       tiktoken_ruby (~> 0.0.5)
 
@@ -135,6 +135,8 @@ GEM
       activesupport (>= 3.0)
       graphql
     hashery (2.1.2)
+    hashie (5.0.0)
+    hnswlib (0.8.1)
     httparty (0.21.0)
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
@@ -166,6 +168,11 @@ GEM
       racc (~> 1.4)
     nokogiri (1.14.3-x86_64-linux)
       racc (~> 1.4)
+    open-weather-ruby-client (0.3.0)
+      activesupport
+      faraday (>= 1.0.0)
+      faraday_middleware
+      hashie
     parallel (1.23.0)
     parser (3.2.2.1)
       ast (~> 2.4.1)
@@ -312,10 +319,12 @@ DEPENDENCIES
   eqn (~> 1.6.5)
   google_palm_api (~> 0.1.1)
   google_search_results (~> 2.0.0)
+  hnswlib (~> 0.8.1)
   hugging-face (~> 0.3.4)
   langchainrb!
   milvus (~> 0.9.0)
   nokogiri (~> 1.13)
+  open-weather-ruby-client (~> 0.3.0)
   pdf-reader (~> 1.4)
   pg (~> 1.5)
   pgvector (~> 0.2)

data/README.md

@@ -34,6 +34,7 @@ require "langchain"
 | Database | Querying           | Storage | Schema Management | Backups | Rails Integration |
 | -------- |:------------------:| -------:| -----------------:| -------:| -----------------:|
 | [Chroma](https://trychroma.com/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Hnswlib](https://github.com/nmslib/hnswlib/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Milvus](https://milvus.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Pinecone](https://www.pinecone.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Pgvector](https://github.com/pgvector/pgvector) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
@@ -56,11 +57,12 @@ client = Langchain::Vectorsearch::Weaviate.new(
 )
 
 # You can instantiate any other supported vector search database:
+client = Langchain::Vectorsearch::Chroma.new(...) # `gem "chroma-db", "~> 0.3.0"`
+client = Langchain::Vectorsearch::Hnswlib.new(...) # `gem "hnswlib", "~> 0.8.1"`
 client = Langchain::Vectorsearch::Milvus.new(...) # `gem "milvus", "~> 0.9.0"`
-client = Langchain::Vectorsearch::Qdrant.new(...) # `gem"qdrant-ruby", "~> 0.9.0"`
 client = Langchain::Vectorsearch::Pinecone.new(...) # `gem "pinecone", "~> 0.1.6"`
-client = Langchain::Vectorsearch::Chroma.new(...) # `gem "chroma-db", "~> 0.3.0"`
 client = Langchain::Vectorsearch::Pgvector.new(...) # `gem "pgvector", "~> 0.2"`
+client = Langchain::Vectorsearch::Qdrant.new(...) # `gem"qdrant-ruby", "~> 0.9.0"`
 ```
 
 ```ruby
@@ -307,6 +309,7 @@ agent.run(question: "How many users have a name with length greater than 5 in th
 | "database"   | Useful for querying a SQL database |                                                               | `gem "sequel", "~> 5.68.0"`                   |
 | "ruby_code_interpreter" | Interprets Ruby expressions             |                                                               | `gem "safe_ruby", "~> 1.0.4"`             |
 | "search"     | A wrapper around Google Search                     | `ENV["SERPAPI_API_KEY"]` (https://serpapi.com/manage-api-key) | `gem "google_search_results", "~> 2.0.0"` |
+| "weather"  | Calls Open Weather API to retrieve the current weather        |      `ENV["OPEN_WEATHER_API_KEY]` (https://home.openweathermap.org/api_keys)               | `gem "open-weather-ruby-client", "~> 0.3.0"`    |
 | "wikipedia"  | Calls Wikipedia API to retrieve the summary        |                                                               | `gem "wikipedia-client", "~> 1.17.0"`     |
 
 #### Loaders 🚚

data/lib/langchain/agent/chain_of_thought_agent/chain_of_thought_agent.rb

@@ -101,7 +101,7 @@ module Langchain::Agent
         tool_names: "[#{tool_list.join(", ")}]",
         tools: tools.map do |tool|
           tool_name = tool.tool_name
-          tool_description = tool.class.const_get(:DESCRIPTION)
+          tool_description = tool.tool_description
           "#{tool_name}: #{tool_description}"
         end.join("\n")
       )

data/lib/langchain/conversation.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Langchain
+  #
+  # A high-level API for running a conversation with an LLM.
+  # Currently supports: OpenAI and Google PaLM LLMs.
+  #
+  # Usage:
+  #     llm = Langchain::LLM::OpenAI.new(api_key: "YOUR_API_KEY")
+  #     chat = Langchain::Conversation.new(llm: llm)
+  #     chat.set_context("You are a chatbot from the future")
+  #     chat.message("Tell me about future technologies")
+  #
+  class Conversation
+    attr_reader :context, :examples, :messages
+
+    # Intialize Conversation with a LLM
+    #
+    # @param llm [Object] The LLM to use for the conversation
+    # @param options [Hash] Options to pass to the LLM, like temperature, top_k, etc.
+    # @return [Langchain::Conversation] The Langchain::Conversation instance
+    def initialize(llm:, **options)
+      @llm = llm
+      @options = options
+      @context = nil
+      @examples = []
+      @messages = []
+    end
+
+    # Set the context of the conversation. Usually used to set the model's persona.
+    # @param message [String] The context of the conversation
+    def set_context(message)
+      @context = message
+    end
+
+    # Add examples to the conversation. Used to give the model a sense of the conversation.
+    # @param examples [Array<Hash>] The examples to add to the conversation
+    def add_examples(examples)
+      @examples.concat examples
+    end
+
+    # Message the model with a prompt and return the response.
+    # @param message [String] The prompt to message the model with
+    # @return [String] The response from the model
+    def message(message)
+      append_user_message(message)
+      response = llm_response(message)
+      append_ai_message(response)
+      response
+    end
+
+    private
+
+    def llm_response(prompt)
+      @llm.chat(messages: @messages, context: @context, examples: @examples, **@options)
+    end
+
+    def append_ai_message(message)
+      @messages << {role: "ai", content: message}
+    end
+
+    def append_user_message(message)
+      @messages << {role: "user", content: message}
+    end
+  end
+end

data/lib/langchain/llm/google_palm.rb

@@ -80,28 +80,31 @@ module Langchain::LLM
     # @param params extra parameters passed to GooglePalmAPI::Client#generate_chat_message
     # @return [String] The chat completion
     #
-    def chat(prompt: "", messages: [], **params)
+    def chat(prompt: "", messages: [], context: "", examples: [], **options)
       raise ArgumentError.new(":prompt or :messages argument is expected") if prompt.empty? && messages.empty?
 
-      messages << {author: "0", content: prompt} if !prompt.empty?
-
-      # TODO: Figure out how to introduce persisted conversations
       default_params = {
         temperature: DEFAULTS[:temperature],
-        messages: messages
+        context: context,
+        messages: compose_chat_messages(prompt: prompt, messages: messages),
+        examples: compose_examples(examples)
       }
 
-      if params[:stop_sequences]
-        default_params[:stop] = params.delete(:stop_sequences)
+      Langchain::Utils::TokenLength::GooglePalmValidator.validate_max_tokens!(self, default_params[:messages], "chat-bison-001")
+
+      if options[:stop_sequences]
+        default_params[:stop] = options.delete(:stop_sequences)
       end
 
-      if params[:max_tokens]
-        default_params[:max_output_tokens] = params.delete(:max_tokens)
+      if options[:max_tokens]
+        default_params[:max_output_tokens] = options.delete(:max_tokens)
       end
 
-      default_params.merge!(params)
+      default_params.merge!(options)
 
       response = client.generate_chat_message(**default_params)
+      raise "GooglePalm API returned an error: #{response}" if response.dig("error")
+
       response.dig("candidates", 0, "content")
     end
 
@@ -124,5 +127,39 @@ module Langchain::LLM
         max_tokens: 2048
       )
     end
+
+    private
+
+    def compose_chat_messages(prompt:, messages:)
+      history = []
+      history.concat transform_messages(messages) unless messages.empty?
+
+      unless prompt.empty?
+        if history.last && history.last[:role] == "user"
+          history.last[:content] += "\n#{prompt}"
+        else
+          history.append({author: "user", content: prompt})
+        end
+      end
+      history
+    end
+
+    def compose_examples(examples)
+      examples.each_slice(2).map do |example|
+        {
+          input: {content: example.first[:content]},
+          output: {content: example.last[:content]}
+        }
+      end
+    end
+
+    def transform_messages(messages)
+      messages.map do |message|
+        {
+          author: message[:role],
+          content: message[:content]
+        }
+      end
+    end
   end
 end

data/lib/langchain/llm/openai.rb

@@ -35,7 +35,7 @@ module Langchain::LLM
     def embed(text:, **params)
       parameters = {model: DEFAULTS[:embeddings_model_name], input: text}
 
-      Langchain::Utils::TokenLengthValidator.validate_max_tokens!(text, parameters[:model])
+      Langchain::Utils::TokenLength::OpenAIValidator.validate_max_tokens!(text, parameters[:model])
 
       response = client.embeddings(parameters: parameters.merge(params))
       response.dig("data").first.dig("embedding")
@@ -52,7 +52,7 @@ module Langchain::LLM
       parameters = compose_parameters DEFAULTS[:completion_model_name], params
 
       parameters[:prompt] = prompt
-      parameters[:max_tokens] = Langchain::Utils::TokenLengthValidator.validate_max_tokens!(prompt, parameters[:model])
+      parameters[:max_tokens] = Langchain::Utils::TokenLength::OpenAIValidator.validate_max_tokens!(prompt, parameters[:model])
 
       response = client.completions(parameters: parameters)
       response.dig("choices", 0, "text")
@@ -63,19 +63,22 @@ module Langchain::LLM
     #
     # @param prompt [String] The prompt to generate a chat completion for
     # @param messages [Array] The messages that have been sent in the conversation
-    # @param params extra parameters passed to OpenAI::Client#chat
+    # @param context [String] The context of the conversation
+    # @param examples [Array] Examples of messages provide model with
+    # @param options extra parameters passed to OpenAI::Client#chat
     # @return [String] The chat completion
     #
-    def chat(prompt: "", messages: [], **params)
+    def chat(prompt: "", messages: [], context: "", examples: [], **options)
       raise ArgumentError.new(":prompt or :messages argument is expected") if prompt.empty? && messages.empty?
 
-      messages << {role: "user", content: prompt} if !prompt.empty?
-
-      parameters = compose_parameters DEFAULTS[:chat_completion_model_name], params
-      parameters[:messages] = messages
-      parameters[:max_tokens] = validate_max_tokens(messages, parameters[:model])
+      parameters = compose_parameters DEFAULTS[:chat_completion_model_name], options
+      parameters[:messages] = compose_chat_messages(prompt: prompt, messages: messages, context: context, examples: examples)
+      parameters[:max_tokens] = validate_max_tokens(parameters[:messages], parameters[:model])
 
       response = client.chat(parameters: parameters)
+
+      raise "Chat completion failed: #{response}" if response.dig("error")
+
       response.dig("choices", 0, "message", "content")
     end
 
@@ -104,8 +107,40 @@ module Langchain::LLM
       default_params.merge(params)
     end
 
+    def compose_chat_messages(prompt:, messages:, context:, examples:)
+      history = []
+
+      history.concat transform_messages(examples) unless examples.empty?
+
+      history.concat transform_messages(messages) unless messages.empty?
+
+      unless context.nil? || context.empty?
+        history.reject! { |message| message[:role] == "system" }
+        history.prepend({role: "system", content: context})
+      end
+
+      unless prompt.empty?
+        if history.last && history.last[:role] == "user"
+          history.last[:content] += "\n#{prompt}"
+        else
+          history.append({role: "user", content: prompt})
+        end
+      end
+
+      history
+    end
+
+    def transform_messages(messages)
+      messages.map do |message|
+        {
+          content: message[:content],
+          role: (message[:role] == "ai") ? "assistant" : message[:role]
+        }
+      end
+    end
+
     def validate_max_tokens(messages, model)
-      Langchain::Utils::TokenLengthValidator.validate_max_tokens!(messages, model)
+      Langchain::Utils::TokenLength::OpenAIValidator.validate_max_tokens!(messages, model)
     end
   end
 end

data/lib/langchain/tool/base.rb

@@ -57,6 +57,15 @@ module Langchain::Tool
       self.class.const_get(:NAME)
     end
 
+    #
+    # Returns the DESCRIPTION constant of the tool
+    #
+    # @return [String] tool description
+    #
+    def tool_description
+      self.class.const_get(:DESCRIPTION)
+    end
+
     #
     # Sets the DESCRIPTION constant of the tool
     #

data/lib/langchain/tool/weather.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Langchain::Tool
+  class Weather < Base
+    #
+    # A weather tool that gets current weather data
+    #
+    # Current weather data is free for 1000 calls per day (https://home.openweathermap.org/api_keys)
+    # Forecast and historical data require registration with credit card, so not supported yet.
+    #
+    # Gem requirements:
+    #     gem "open-weather-ruby-client", "~> 0.3.0"
+    #     api_key: https://home.openweathermap.org/api_keys
+    #
+    # Usage:
+    #     weather = Langchain::Tool::Weather.new(api_key: "YOUR_API_KEY")
+    #     weather.execute(input: "Boston, MA; imperial")
+    #
+
+    NAME = "weather"
+
+    description <<~DESC
+      Useful for getting current weather data
+    
+      The input to this tool should be a city name followed by the units (imperial, metric, or standard)
+      Usage:
+        Action Input: St Louis, Missouri; metric
+        Action Input: Boston, Massachusetts; imperial
+        Action Input: Dubai, AE; imperial
+        Action Input: Kiev, Ukraine; metric
+    DESC
+
+    attr_reader :client, :units
+
+    #
+    # Initializes the Weather tool
+    #
+    # @param api_key [String] Open Weather API key
+    # @return [Langchain::Tool::Weather] Weather tool
+    #
+    def initialize(api_key:, units: "metric")
+      depends_on "open-weather-ruby-client"
+      require "open-weather-ruby-client"
+
+      OpenWeather::Client.configure do |config|
+        config.api_key = api_key
+        config.user_agent = "Langchainrb Ruby Client"
+      end
+
+      @client = OpenWeather::Client.new
+    end
+
+    # Returns current weather for a city
+    # @param input [String] comma separated city and unit (optional: imperial, metric, or standard)
+    # @return [String] Answer
+    def execute(input:)
+      Langchain.logger.info("[#{self.class.name}]".light_blue + ": Executing for \"#{input}\"")
+
+      input_array = input.split(";")
+      city, units = *input_array.map(&:strip)
+
+      data = client.current_weather(city: city, units: units)
+      weather = data.main.map { |key, value| "#{key} #{value}" }.join(", ")
+      "The current weather in #{data.name} is #{weather}"
+    end
+  end
+end

data/lib/langchain/utils/token_length/google_palm_validator.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Utils
+    module TokenLength
+      #
+      # This class is meant to validate the length of the text passed in to Google Palm's API.
+      # It is used to validate the token length before the API call is made
+      #
+      class GooglePalmValidator
+        TOKEN_LIMITS = {
+          # Source:
+          # This data can be pulled when `list_models()` method is called: https://github.com/andreibondarev/google_palm_api#usage
+
+          # chat-bison-001 is the only model that currently supports countMessageTokens functions
+          "chat-bison-001" => {
+            "input_token_limit" => 4000, # 4096 is the limit but the countMessageTokens does not return anything higher than 4000
+            "output_token_limit" => 1024
+          }
+          # "text-bison-001" => {
+          #   "input_token_limit" => 8196,
+          #   "output_token_limit" => 1024
+          # },
+          # "embedding-gecko-001" => {
+          #   "input_token_limit" => 1024
+          # }
+        }.freeze
+
+        #
+        # Validate the context length of the text
+        #
+        # @param content [String | Array<String>] The text or array of texts to validate
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] Whether the text is valid or not
+        # @raise [TokenLimitExceeded] If the text is too long
+        #
+        def self.validate_max_tokens!(google_palm_llm, content, model_name)
+          text_token_length = if content.is_a?(Array)
+            content.sum { |item| token_length(google_palm_llm, item.to_json, model_name) }
+          else
+            token_length(google_palm_llm, content, model_name)
+          end
+
+          leftover_tokens = TOKEN_LIMITS.dig(model_name, "input_token_limit") - text_token_length
+
+          # Raise an error even if whole prompt is equal to the model's token limit (leftover_tokens == 0)
+          if leftover_tokens <= 0
+            raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS.dig(model_name, "input_token_limit")} tokens, but the given text is #{text_token_length} tokens long."
+          end
+
+          leftover_tokens
+        end
+
+        #
+        # Calculate token length for a given text and model name
+        #
+        # @param llm [Langchain::LLM:GooglePalm] The Langchain::LLM:GooglePalm instance
+        # @param text [String] The text to calculate the token length for
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] The token length of the text
+        #
+        def self.token_length(llm, text, model_name = "chat-bison-001")
+          response = llm.client.count_message_tokens(model: model_name, prompt: text)
+          response.dig("tokenCount")
+        end
+      end
+    end
+  end
+end

data/lib/langchain/utils/token_length/openai_validator.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "tiktoken_ruby"
+
+module Langchain
+  module Utils
+    module TokenLength
+      #
+      # This class is meant to validate the length of the text passed in to OpenAI's API.
+      # It is used to validate the token length before the API call is made
+      #
+      class OpenAIValidator
+        TOKEN_LIMITS = {
+          # Source:
+          # https://platform.openai.com/docs/api-reference/embeddings
+          # https://platform.openai.com/docs/models/gpt-4
+          "text-embedding-ada-002" => 8191,
+          "gpt-3.5-turbo" => 4096,
+          "gpt-3.5-turbo-0301" => 4096,
+          "text-davinci-003" => 4097,
+          "text-davinci-002" => 4097,
+          "code-davinci-002" => 8001,
+          "gpt-4" => 8192,
+          "gpt-4-0314" => 8192,
+          "gpt-4-32k" => 32768,
+          "gpt-4-32k-0314" => 32768,
+          "text-curie-001" => 2049,
+          "text-babbage-001" => 2049,
+          "text-ada-001" => 2049,
+          "davinci" => 2049,
+          "curie" => 2049,
+          "babbage" => 2049,
+          "ada" => 2049
+        }.freeze
+
+        #
+        # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
+        #
+        # @param content [String | Array<String>] The text or array of texts to validate
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] Whether the text is valid or not
+        # @raise [TokenLimitExceeded] If the text is too long
+        #
+        def self.validate_max_tokens!(content, model_name)
+          text_token_length = if content.is_a?(Array)
+            content.sum { |item| token_length(item.to_json, model_name) }
+          else
+            token_length(content, model_name)
+          end
+
+          max_tokens = TOKEN_LIMITS[model_name] - text_token_length
+
+          # Raise an error even if whole prompt is equal to the model's token limit (max_tokens == 0) since not response will be returned
+          if max_tokens <= 0
+            raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS[model_name]} tokens, but the given text is #{text_token_length} tokens long."
+          end
+
+          max_tokens
+        end
+
+        #
+        # Calculate token length for a given text and model name
+        #
+        # @param text [String] The text to calculate the token length for
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] The token length of the text
+        #
+        def self.token_length(text, model_name)
+          encoder = Tiktoken.encoding_for_model(model_name)
+          encoder.encode(text).length
+        end
+      end
+    end
+  end
+end

data/lib/langchain/vectorsearch/hnswlib.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Langchain::Vectorsearch
+  class Hnswlib < Base
+    #
+    # Wrapper around HNSW (Hierarchical Navigable Small World) library.
+    # HNSWLib is an in-memory vectorstore that can be saved to a file on disk.
+    #
+    # Gem requirements:
+    #     gem "hnswlib", "~> 0.8.1"
+    #
+    # Usage:
+    #     hnsw = Langchain::Vectorsearch::Hnswlib.new(llm:, url:, index_name:)
+    #
+
+    attr_reader :client, :path_to_index
+
+    #
+    # Initialize the HNSW vector search
+    #
+    # @param llm [Object] The LLM client to use
+    # @param path_to_index [String] The local path to the index file, e.g.: "/storage/index.ann"
+    # @return [Langchain::Vectorsearch::Hnswlib] Class instance
+    #
+    def initialize(llm:, path_to_index:)
+      depends_on "hnswlib"
+      require "hnswlib"
+
+      super(llm: llm)
+
+      @client = ::Hnswlib::HierarchicalNSW.new(space: DEFAULT_METRIC, dim: llm.default_dimension)
+      @path_to_index = path_to_index
+
+      initialize_index
+    end
+
+    #
+    # Add a list of texts and corresponding IDs to the index
+    #
+    # @param texts [Array] The list of texts to add
+    # @param ids [Array] The list of corresponding IDs (integers) to the texts
+    # @return [Boolean] The response from the HNSW library
+    #
+    def add_texts(texts:, ids:)
+      resize_index(texts.size)
+
+      Array(texts).each_with_index do |text, i|
+        embedding = llm.embed(text: text)
+
+        client.add_point(embedding, ids[i])
+      end
+
+      client.save_index(path_to_index)
+    end
+
+    #
+    # Search for similar texts
+    #
+    # @param query [String] The text to search for
+    # @param k [Integer] The number of results to return
+    # @return [Array] Results in the format `[[id1, distance3], [id2, distance2]]`
+    #
+    def similarity_search(
+      query:,
+      k: 4
+    )
+      embedding = llm.embed(text: query)
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    #
+    # Search for the K nearest neighbors of a given vector
+    #
+    # @param embedding [Array] The embedding to search for
+    # @param k [Integer] The number of results to return
+    # @return [Array] Results in the format `[[id1, distance3], [id2, distance2]]`
+    #
+    def similarity_search_by_vector(
+      embedding:,
+      k: 4
+    )
+      client.search_knn(embedding, k)
+    end
+
+    private
+
+    #
+    # Optionally resizes the index if there's no space for new data
+    #
+    # @param num_of_elements_to_add [Integer] The number of elements to add to the index
+    #
+    def resize_index(num_of_elements_to_add)
+      current_count = client.current_count
+
+      if (current_count + num_of_elements_to_add) > client.max_elements
+        new_size = current_count + num_of_elements_to_add
+
+        client.resize_index(new_size)
+      end
+    end
+
+    #
+    # Loads or initializes the new index
+    #
+    def initialize_index
+      if File.exist?(path_to_index)
+        client.load_index(path_to_index)
+
+        Langchain.logger.info("[#{self.class.name}]".blue + ": Successfully loaded the index at \"#{path_to_index}\"")
+      else
+        # Default max_elements: 100, but we constantly resize the index as new data is written to it
+        client.init_index(max_elements: 100)
+
+        Langchain.logger.info("[#{self.class.name}]".blue + ": Creating a new index at \"#{path_to_index}\"")
+      end
+    end
+  end
+end

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.5.3"
+  VERSION = "0.5.5"
 end

data/lib/langchain.rb

@@ -62,6 +62,7 @@ module Langchain
 
   autoload :Loader, "langchain/loader"
   autoload :Data, "langchain/data"
+  autoload :Conversation, "langchain/conversation"
   autoload :DependencyHelper, "langchain/dependency_helper"
 
   module Agent
@@ -75,6 +76,7 @@ module Langchain
     autoload :Calculator, "langchain/tool/calculator"
     autoload :RubyCodeInterpreter, "langchain/tool/ruby_code_interpreter"
     autoload :SerpApi, "langchain/tool/serp_api"
+    autoload :Weather, "langchain/tool/weather"
     autoload :Wikipedia, "langchain/tool/wikipedia"
     autoload :Database, "langchain/tool/database"
   end
@@ -92,12 +94,18 @@ module Langchain
   end
 
   module Utils
-    autoload :TokenLengthValidator, "langchain/utils/token_length_validator"
+    module TokenLength
+      class TokenLimitExceeded < StandardError; end
+
+      autoload :OpenAIValidator, "langchain/utils/token_length/openai_validator"
+      autoload :GooglePalmValidator, "langchain/utils/token_length/google_palm_validator"
+    end
   end
 
   module Vectorsearch
     autoload :Base, "langchain/vectorsearch/base"
     autoload :Chroma, "langchain/vectorsearch/chroma"
+    autoload :Hnswlib, "langchain/vectorsearch/hnswlib"
     autoload :Milvus, "langchain/vectorsearch/milvus"
     autoload :Pinecone, "langchain/vectorsearch/pinecone"
     autoload :Pgvector, "langchain/vectorsearch/pgvector"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.5
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-10 00:00:00.000000000 Z
+date: 2023-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tiktoken_ruby
@@ -192,6 +192,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: hnswlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: hugging-face
   requirement: !ruby/object:Gem::Requirement
@@ -234,6 +248,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: open-weather-ruby-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -432,6 +460,7 @@ files:
 - lib/langchain/agent/sql_query_agent/sql_query_agent.rb
 - lib/langchain/agent/sql_query_agent/sql_query_agent_answer_prompt.json
 - lib/langchain/agent/sql_query_agent/sql_query_agent_sql_prompt.json
+- lib/langchain/conversation.rb
 - lib/langchain/data.rb
 - lib/langchain/dependency_helper.rb
 - lib/langchain/llm/ai21.rb
@@ -461,10 +490,13 @@ files:
 - lib/langchain/tool/database.rb
 - lib/langchain/tool/ruby_code_interpreter.rb
 - lib/langchain/tool/serp_api.rb
+- lib/langchain/tool/weather.rb
 - lib/langchain/tool/wikipedia.rb
-- lib/langchain/utils/token_length_validator.rb
+- lib/langchain/utils/token_length/google_palm_validator.rb
+- lib/langchain/utils/token_length/openai_validator.rb
 - lib/langchain/vectorsearch/base.rb
 - lib/langchain/vectorsearch/chroma.rb
+- lib/langchain/vectorsearch/hnswlib.rb
 - lib/langchain/vectorsearch/milvus.rb
 - lib/langchain/vectorsearch/pgvector.rb
 - lib/langchain/vectorsearch/pinecone.rb
@@ -480,6 +512,7 @@ metadata:
   homepage_uri: https://rubygems.org/gems/langchainrb
   source_code_uri: https://github.com/andreibondarev/langchainrb
   changelog_uri: https://github.com/andreibondarev/langchainrb/CHANGELOG.md
+  documentation_uri: https://rubydoc.info/gems/langchainrb
 post_install_message:
 rdoc_options: []
 require_paths:

data/lib/langchain/utils/token_length_validator.rb

@@ -1,89 +0,0 @@
-# frozen_string_literal: true
-
-require "tiktoken_ruby"
-
-module Langchain
-  module Utils
-    class TokenLimitExceeded < StandardError; end
-
-    class TokenLengthValidator
-      #
-      # This class is meant to validate the length of the text passed in to OpenAI's API.
-      # It is used to validate the token length before the API call is made
-      #
-      TOKEN_LIMITS = {
-        # Source:
-        # https://platform.openai.com/docs/api-reference/embeddings
-        # https://platform.openai.com/docs/models/gpt-4
-        "text-embedding-ada-002" => 8191,
-        "gpt-3.5-turbo" => 4096,
-        "gpt-3.5-turbo-0301" => 4096,
-        "text-davinci-003" => 4097,
-        "text-davinci-002" => 4097,
-        "code-davinci-002" => 8001,
-        "gpt-4" => 8192,
-        "gpt-4-0314" => 8192,
-        "gpt-4-32k" => 32768,
-        "gpt-4-32k-0314" => 32768,
-        "text-curie-001" => 2049,
-        "text-babbage-001" => 2049,
-        "text-ada-001" => 2049,
-        "davinci" => 2049,
-        "curie" => 2049,
-        "babbage" => 2049,
-        "ada" => 2049
-      }.freeze
-
-      # GOOGLE_PALM_TOKEN_LIMITS = {
-      #   "chat-bison-001" => {
-      #     "inputTokenLimit"=>4096,
-      #     "outputTokenLimit"=>1024
-      #   },
-      #   "text-bison-001" => {
-      #     "inputTokenLimit"=>8196,
-      #     "outputTokenLimit"=>1024
-      #   },
-      #   "embedding-gecko-001" => {
-      #     "inputTokenLimit"=>1024
-      #   }
-      # }.freeze
-
-      #
-      # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
-      #
-      # @param content [String | Array<String>] The text or array of texts to validate
-      # @param model_name [String] The model name to validate against
-      # @return [Integer] Whether the text is valid or not
-      # @raise [TokenLimitExceeded] If the text is too long
-      #
-      def self.validate_max_tokens!(content, model_name)
-        text_token_length = if content.is_a?(Array)
-          content.sum { |item| token_length(item.to_json, model_name) }
-        else
-          token_length(content, model_name)
-        end
-
-        max_tokens = TOKEN_LIMITS[model_name] - text_token_length
-
-        # Raise an error even if whole prompt is equal to the model's token limit (max_tokens == 0) since not response will be returned
-        if max_tokens <= 0
-          raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS[model_name]} tokens, but the given text is #{text_token_length} tokens long."
-        end
-
-        max_tokens
-      end
-
-      #
-      # Calculate token length for a given text and model name
-      #
-      # @param text [String] The text to validate
-      # @param model_name [String] The model name to validate against
-      # @return [Integer] The token length of the text
-      #
-      def self.token_length(text, model_name)
-        encoder = Tiktoken.encoding_for_model(model_name)
-        encoder.encode(text).length
-      end
-    end
-  end
-end