langchainrb 0.5.5 → 0.5.6

data/lib/langchain/prompt/base.rb

@@ -45,11 +45,11 @@ module Langchain::Prompt
     end
 
     #
-    # Save the object to a file in JSON format.
+    # Save the object to a file in JSON or YAML format.
     #
     # @param file_path [String, Pathname] The path to the file to save the object to
     #
-    # @raise [ArgumentError] If file_path doesn't end with .json
+    # @raise [ArgumentError] If file_path doesn't end with .json or .yaml or .yml
     #
     # @return [void]
     #

data/lib/langchain/tool/base.rb

@@ -9,7 +9,7 @@ module Langchain::Tool
   #
   # - {Langchain::Tool::Calculator}: Calculate the result of a math expression
   # - {Langchain::Tool::RubyCodeInterpretor}: Runs ruby code
-  # - {Langchain::Tool::Search}: search on Google (via SerpAPI)
+  # - {Langchain::Tool::GoogleSearch}: search on Google (via SerpAPI)
   # - {Langchain::Tool::Wikipedia}: search on Wikipedia
   #
   # == Usage
@@ -30,13 +30,13 @@ module Langchain::Tool
   #     agent = Langchain::Agent::ChainOfThoughtAgent.new(
   #       llm: :openai, # or :cohere, :hugging_face, :google_palm or :replicate
   #       llm_api_key: ENV["OPENAI_API_KEY"],
-  #       tools: ["search", "calculator", "wikipedia"]
+  #       tools: ["google_search", "calculator", "wikipedia"]
   #     )
   #
   # 4. Confirm that the Agent is using the Tools you passed in:
   #
   #     agent.tools
-  #     # => ["search", "calculator", "wikipedia"]
+  #     # => ["google_search", "calculator", "wikipedia"]
   #
   # == Adding Tools
   #
@@ -57,6 +57,12 @@ module Langchain::Tool
       self.class.const_get(:NAME)
     end
 
+    def self.logger_options
+      {
+        color: :light_blue
+      }
+    end
+
     #
     # Returns the DESCRIPTION constant of the tool
     #

data/lib/langchain/tool/calculator.rb

@@ -28,14 +28,14 @@ module Langchain::Tool
     # @param input [String] math expression
     # @return [String] Answer
     def execute(input:)
-      Langchain.logger.info("[#{self.class.name}]".light_blue + ": Executing \"#{input}\"")
+      Langchain.logger.info("Executing \"#{input}\"", for: self.class)
 
       Eqn::Calculator.calc(input)
     rescue Eqn::ParseError, Eqn::NoVariableValueError
       # Sometimes the input is not a pure math expression, e.g: "12F in Celsius"
       # We can use the google answer box to evaluate this expression
       # TODO: Figure out to find a better way to evaluate these language expressions.
-      hash_results = Langchain::Tool::SerpApi
+      hash_results = Langchain::Tool::GoogleSearch
         .new(api_key: ENV["SERPAPI_API_KEY"])
         .execute_search(input: input)
       hash_results.dig(:answer_box, :to) ||

data/lib/langchain/tool/database.rb

@@ -39,7 +39,7 @@ module Langchain::Tool
     # @return [String] schema
     #
     def schema
-      Langchain.logger.info("[#{self.class.name}]".light_blue + ": Dumping schema")
+      Langchain.logger.info("Dumping schema", for: self.class)
       db.dump_schema_migration(same_db: true, indexes: false) unless db.adapter_scheme == :mock
     end
 
@@ -50,11 +50,11 @@ module Langchain::Tool
     # @return [Array] results
     #
     def execute(input:)
-      Langchain.logger.info("[#{self.class.name}]".light_blue + ": Executing \"#{input}\"")
+      Langchain.logger.info("Executing \"#{input}\"", for: self.class)
 
       db[input].to_a
     rescue Sequel::DatabaseError => e
-      Langchain.logger.error("[#{self.class.name}]".light_red + ": #{e.message}")
+      Langchain.logger.error(e.message, for: self.class)
     end
   end
 end

data/lib/langchain/tool/{serp_api.rb → google_search.rb}

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
 module Langchain::Tool
-  class SerpApi < Base
+  class GoogleSearch < Base
     #
-    # Wrapper around SerpAPI
+    # Wrapper around Google Serp SPI
     #
     # Gem requirements: gem "google_search_results", "~> 2.0.0"
     #
     # Usage:
-    # search = Langchain::Tool::SerpApi.new(api_key: "YOUR_API_KEY")
+    # search = Langchain::Tool::GoogleSearch.new(api_key: "YOUR_API_KEY")
     # search.execute(input: "What is the capital of France?")
     #
 
-    NAME = "search"
+    NAME = "google_search"
 
     description <<~DESC
       A wrapper around Google Search.
@@ -26,10 +26,10 @@ module Langchain::Tool
     attr_reader :api_key
 
     #
-    # Initializes the SerpAPI tool
+    # Initializes the Google Search tool
     #
-    # @param api_key [String] SerpAPI API key
-    # @return [Langchain::Tool::SerpApi] SerpAPI tool
+    # @param api_key [String] Search API key
+    # @return [Langchain::Tool::GoogleSearch] Google search tool
     #
     def initialize(api_key:)
       depends_on "google_search_results"
@@ -54,7 +54,7 @@ module Langchain::Tool
     # @return [String] Answer
     #
     def execute(input:)
-      Langchain.logger.info("[#{self.class.name}]".light_blue + ": Executing \"#{input}\"")
+      Langchain.logger.info("Executing \"#{input}\"", for: self.class)
 
       hash_results = execute_search(input: input)
 
@@ -72,7 +72,7 @@ module Langchain::Tool
     # @return [Hash] hash_results JSON
     #
     def execute_search(input:)
-      GoogleSearch
+      ::GoogleSearch
         .new(q: input, serp_api_key: api_key)
         .get_hash
     end

data/lib/langchain/tool/ruby_code_interpreter.rb

@@ -21,7 +21,7 @@ module Langchain::Tool
     # @param input [String] ruby code expression
     # @return [String] Answer
     def execute(input:)
-      Langchain.logger.info("[#{self.class.name}]".light_blue + ": Executing \"#{input}\"")
+      Langchain.logger.info("Executing \"#{input}\"", for: self.class)
 
       safe_eval(input)
     end

data/lib/langchain/tool/weather.rb

@@ -21,7 +21,7 @@ module Langchain::Tool
 
     description <<~DESC
       Useful for getting current weather data
-    
+
       The input to this tool should be a city name followed by the units (imperial, metric, or standard)
       Usage:
         Action Input: St Louis, Missouri; metric
@@ -54,7 +54,7 @@ module Langchain::Tool
     # @param input [String] comma separated city and unit (optional: imperial, metric, or standard)
     # @return [String] Answer
     def execute(input:)
-      Langchain.logger.info("[#{self.class.name}]".light_blue + ": Executing for \"#{input}\"")
+      Langchain.logger.info("Executing for \"#{input}\"", for: self.class)
 
       input_array = input.split(";")
       city, units = *input_array.map(&:strip)

data/lib/langchain/tool/wikipedia.rb

@@ -26,7 +26,7 @@ module Langchain::Tool
     # @param input [String] search query
     # @return [String] Answer
     def execute(input:)
-      Langchain.logger.info("[#{self.class.name}]".light_blue + ": Executing \"#{input}\"")
+      Langchain.logger.info("Executing \"#{input}\"", for: self.class)
 
       page = ::Wikipedia.find(input)
       # It would be nice to figure out a way to provide page.content but the LLM token limit is an issue

data/lib/langchain/utils/token_length/base_validator.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Utils
+    module TokenLength
+      #
+      # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
+      #
+      # @param content [String | Array<String>] The text or array of texts to validate
+      # @param model_name [String] The model name to validate against
+      # @return [Integer] Whether the text is valid or not
+      # @raise [TokenLimitExceeded] If the text is too long
+      #
+      class BaseValidator
+        def self.validate_max_tokens!(content, model_name, options = {})
+          text_token_length = if content.is_a?(Array)
+            content.sum { |item| token_length(item.to_json, model_name, options) }
+          else
+            token_length(content, model_name, options)
+          end
+
+          leftover_tokens = token_limit(model_name) - text_token_length
+
+          # Raise an error even if whole prompt is equal to the model's token limit (leftover_tokens == 0)
+          if leftover_tokens <= 0
+            raise limit_exceeded_exception(token_limit(model_name), text_token_length)
+          end
+
+          leftover_tokens
+        end
+
+        def self.limit_exceeded_exception(limit, length)
+          TokenLimitExceeded.new("This model's maximum context length is #{limit} tokens, but the given text is #{length} tokens long.", length - limit)
+        end
+      end
+    end
+  end
+end

data/lib/langchain/utils/token_length/google_palm_validator.rb

@@ -7,7 +7,7 @@ module Langchain
       # This class is meant to validate the length of the text passed in to Google Palm's API.
       # It is used to validate the token length before the API call is made
       #
-      class GooglePalmValidator
+      class GooglePalmValidator < BaseValidator
         TOKEN_LIMITS = {
           # Source:
           # This data can be pulled when `list_models()` method is called: https://github.com/andreibondarev/google_palm_api#usage
@@ -26,43 +26,23 @@ module Langchain
           # }
         }.freeze
 
-        #
-        # Validate the context length of the text
-        #
-        # @param content [String | Array<String>] The text or array of texts to validate
-        # @param model_name [String] The model name to validate against
-        # @return [Integer] Whether the text is valid or not
-        # @raise [TokenLimitExceeded] If the text is too long
-        #
-        def self.validate_max_tokens!(google_palm_llm, content, model_name)
-          text_token_length = if content.is_a?(Array)
-            content.sum { |item| token_length(google_palm_llm, item.to_json, model_name) }
-          else
-            token_length(google_palm_llm, content, model_name)
-          end
-
-          leftover_tokens = TOKEN_LIMITS.dig(model_name, "input_token_limit") - text_token_length
-
-          # Raise an error even if whole prompt is equal to the model's token limit (leftover_tokens == 0)
-          if leftover_tokens <= 0
-            raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS.dig(model_name, "input_token_limit")} tokens, but the given text is #{text_token_length} tokens long."
-          end
-
-          leftover_tokens
-        end
-
         #
         # Calculate token length for a given text and model name
         #
-        # @param llm [Langchain::LLM:GooglePalm] The Langchain::LLM:GooglePalm instance
         # @param text [String] The text to calculate the token length for
         # @param model_name [String] The model name to validate against
+        # @param options [Hash] the options to create a message with
+        # @option options [Langchain::LLM:GooglePalm] :llm The Langchain::LLM:GooglePalm instance
         # @return [Integer] The token length of the text
         #
-        def self.token_length(llm, text, model_name = "chat-bison-001")
-          response = llm.client.count_message_tokens(model: model_name, prompt: text)
+        def self.token_length(text, model_name = "chat-bison-001", options)
+          response = options[:llm].client.count_message_tokens(model: model_name, prompt: text)
           response.dig("tokenCount")
         end
+
+        def self.token_limit(model_name)
+          TOKEN_LIMITS.dig(model_name, "input_token_limit")
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/openai_validator.rb

@@ -9,7 +9,7 @@ module Langchain
       # This class is meant to validate the length of the text passed in to OpenAI's API.
       # It is used to validate the token length before the API call is made
       #
-      class OpenAIValidator
+      class OpenAIValidator < BaseValidator
         TOKEN_LIMITS = {
           # Source:
           # https://platform.openai.com/docs/api-reference/embeddings
@@ -17,6 +17,9 @@ module Langchain
           "text-embedding-ada-002" => 8191,
           "gpt-3.5-turbo" => 4096,
           "gpt-3.5-turbo-0301" => 4096,
+          "gpt-3.5-turbo-0613" => 4096,
+          "gpt-3.5-turbo-16k" => 16384,
+          "gpt-3.5-turbo-16k-0613" => 16384,
           "text-davinci-003" => 4097,
           "text-davinci-002" => 4097,
           "code-davinci-002" => 8001,
@@ -24,6 +27,7 @@ module Langchain
           "gpt-4-0314" => 8192,
           "gpt-4-32k" => 32768,
           "gpt-4-32k-0314" => 32768,
+          "gpt-4-32k-0613" => 32768,
           "text-curie-001" => 2049,
           "text-babbage-001" => 2049,
           "text-ada-001" => 2049,
@@ -33,31 +37,6 @@ module Langchain
           "ada" => 2049
         }.freeze
 
-        #
-        # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
-        #
-        # @param content [String | Array<String>] The text or array of texts to validate
-        # @param model_name [String] The model name to validate against
-        # @return [Integer] Whether the text is valid or not
-        # @raise [TokenLimitExceeded] If the text is too long
-        #
-        def self.validate_max_tokens!(content, model_name)
-          text_token_length = if content.is_a?(Array)
-            content.sum { |item| token_length(item.to_json, model_name) }
-          else
-            token_length(content, model_name)
-          end
-
-          max_tokens = TOKEN_LIMITS[model_name] - text_token_length
-
-          # Raise an error even if whole prompt is equal to the model's token limit (max_tokens == 0) since not response will be returned
-          if max_tokens <= 0
-            raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS[model_name]} tokens, but the given text is #{text_token_length} tokens long."
-          end
-
-          max_tokens
-        end
-
         #
         # Calculate token length for a given text and model name
         #
@@ -65,10 +44,14 @@ module Langchain
         # @param model_name [String] The model name to validate against
         # @return [Integer] The token length of the text
         #
-        def self.token_length(text, model_name)
+        def self.token_length(text, model_name, options = {})
           encoder = Tiktoken.encoding_for_model(model_name)
           encoder.encode(text).length
         end
+
+        def self.token_limit(model_name)
+          TOKEN_LIMITS[model_name]
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/token_limit_exceeded.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Utils
+    module TokenLength
+      class TokenLimitExceeded < StandardError
+        attr_reader :token_overflow
+
+        def initialize(message = "", token_overflow = 0)
+          super message
+
+          @token_overflow = token_overflow
+        end
+      end
+    end
+  end
+end

data/lib/langchain/vectorsearch/base.rb

@@ -155,5 +155,11 @@ module Langchain::Vectorsearch
 
       add_texts(texts: texts)
     end
+
+    def self.logger_options
+      {
+        color: :blue
+      }
+    end
   end
 end

data/lib/langchain/vectorsearch/chroma.rb

@@ -8,7 +8,7 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "chroma-db", "~> 0.3.0"
     #
     # Usage:
-    # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, api_key: nil)
+    # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, llm_api_key:, api_key: nil)
     #
 
     # Initialize the Chroma client

data/lib/langchain/vectorsearch/hnswlib.rb

@@ -110,12 +110,12 @@ module Langchain::Vectorsearch
       if File.exist?(path_to_index)
         client.load_index(path_to_index)
 
-        Langchain.logger.info("[#{self.class.name}]".blue + ": Successfully loaded the index at \"#{path_to_index}\"")
+        Langchain.logger.info("Successfully loaded the index at \"#{path_to_index}\"", for: self.class)
       else
         # Default max_elements: 100, but we constantly resize the index as new data is written to it
         client.init_index(max_elements: 100)
 
-        Langchain.logger.info("[#{self.class.name}]".blue + ": Creating a new index at \"#{path_to_index}\"")
+        Langchain.logger.info("Creating a new index at \"#{path_to_index}\"", for: self.class)
       end
     end
   end

data/lib/langchain/vectorsearch/milvus.rb

@@ -8,17 +8,9 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "milvus", "~> 0.9.0"
     #
     # Usage:
-    # milvus = Langchain::Vectorsearch::Milvus.new(url:, index_name:, llm:)
+    # milvus = Langchain::Vectorsearch::Milvus.new(url:, index_name:, llm:, llm_api_key:)
     #
 
-    #
-    # Initialize the Milvus client
-    #
-    # @param url [String] The URL of the Milvus server
-    # @param api_key [String] The API key to use
-    # @param index_name [String] The name of the index to use
-    # @param llm [Object] The LLM client to use
-    #
     def initialize(url:, index_name:, llm:, api_key: nil)
       depends_on "milvus"
       require "milvus"
@@ -29,11 +21,6 @@ module Langchain::Vectorsearch
       super(llm: llm)
     end
 
-    #
-    # Add a list of texts to the index
-    #
-    # @param texts [Array] The list of texts to add
-    #
     def add_texts(texts:)
       client.entities.insert(
         collection_name: index_name,

data/lib/langchain/vectorsearch/pgvector.rb

@@ -8,7 +8,7 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "pgvector", "~> 0.2"
     #
     # Usage:
-    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:)
+    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:, llm_api_key:)
     #
 
     # The operators supported by the PostgreSQL vector search adapter
@@ -20,14 +20,10 @@ module Langchain::Vectorsearch
 
     attr_reader :operator, :quoted_table_name
 
-    #
-    # Initialize the PostgreSQL client
-    #
     # @param url [String] The URL of the PostgreSQL database
     # @param index_name [String] The name of the table to use for the index
     # @param llm [Object] The LLM client to use
     # @param api_key [String] The API key for the Vectorsearch DB (not used for PostgreSQL)
-    #
     def initialize(url:, index_name:, llm:, api_key: nil)
       require "pg"
       require "pgvector"

data/lib/langchain/vectorsearch/pinecone.rb

@@ -8,17 +8,14 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "pinecone", "~> 0.1.6"
     #
     # Usage:
-    # pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:)
+    # pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
-    #
     # Initialize the Pinecone client
-    #
     # @param environment [String] The environment to use
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
-    #
     def initialize(environment:, api_key:, index_name:, llm:)
       depends_on "pinecone"
       require "pinecone"

data/lib/langchain/vectorsearch/qdrant.rb

@@ -8,17 +8,14 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "qdrant-ruby", "~> 0.9.0"
     #
     # Usage:
-    # qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:)
+    # qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
-    #
     # Initialize the Qdrant client
-    #
     # @param url [String] The URL of the Qdrant server
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
-    #
     def initialize(url:, api_key:, index_name:, llm:)
       depends_on "qdrant-ruby"
       require "qdrant"

data/lib/langchain/vectorsearch/weaviate.rb

@@ -8,17 +8,14 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "weaviate-ruby", "~> 0.8.0"
     #
     # Usage:
-    # weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:)
+    # weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
-    #
     # Initialize the Weaviate adapter
-    #
     # @param url [String] The URL of the Weaviate instance
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
-    #
     def initialize(url:, api_key:, index_name:, llm:)
       depends_on "weaviate-ruby"
       require "weaviate"

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.5.5"
+  VERSION = "0.5.6"
 end

data/lib/langchain.rb

@@ -46,36 +46,48 @@ require_relative "./langchain/version"
 #
 # LangChain.rb uses standard logging mechanisms and defaults to :debug level. Most messages are at info level, but we will add debug or warn statements as needed. To show all log messages:
 #
-#     Langchain.logger.level = :info
+# Langchain.logger.level = :info
 module Langchain
+  autoload :Loader, "langchain/loader"
+  autoload :Data, "langchain/data"
+  autoload :Conversation, "langchain/conversation"
+  autoload :DependencyHelper, "langchain/dependency_helper"
+  autoload :ContextualLogger, "langchain/contextual_logger"
+
   class << self
-    # @return [Logger]
-    attr_accessor :logger
+    # @return [ContextualLogger]
+    attr_reader :logger
+
+    # @param logger [Logger]
+    # @return [ContextualLogger]
+    def logger=(logger)
+      @logger = ContextualLogger.new(logger)
+    end
 
     # @return [Pathname]
     attr_reader :root
   end
 
-  @logger ||= ::Logger.new($stdout, level: :warn, formatter: ->(severity, datetime, progname, msg) { "[LangChain.rb]".yellow + " #{msg}\n" })
+  self.logger ||= ::Logger.new($stdout, level: :warn)
 
   @root = Pathname.new(__dir__)
 
-  autoload :Loader, "langchain/loader"
-  autoload :Data, "langchain/data"
-  autoload :Conversation, "langchain/conversation"
-  autoload :DependencyHelper, "langchain/dependency_helper"
-
   module Agent
     autoload :Base, "langchain/agent/base"
     autoload :ChainOfThoughtAgent, "langchain/agent/chain_of_thought_agent/chain_of_thought_agent.rb"
     autoload :SQLQueryAgent, "langchain/agent/sql_query_agent/sql_query_agent.rb"
   end
 
+  module Chunker
+    autoload :Base, "langchain/chunker/base"
+    autoload :Text, "langchain/chunker/text"
+  end
+
   module Tool
     autoload :Base, "langchain/tool/base"
     autoload :Calculator, "langchain/tool/calculator"
     autoload :RubyCodeInterpreter, "langchain/tool/ruby_code_interpreter"
-    autoload :SerpApi, "langchain/tool/serp_api"
+    autoload :GoogleSearch, "langchain/tool/google_search"
     autoload :Weather, "langchain/tool/weather"
     autoload :Wikipedia, "langchain/tool/wikipedia"
     autoload :Database, "langchain/tool/database"
@@ -95,8 +107,8 @@ module Langchain
 
   module Utils
     module TokenLength
-      class TokenLimitExceeded < StandardError; end
-
+      autoload :BaseValidator, "langchain/utils/token_length/base_validator"
+      autoload :TokenLimitExceeded, "langchain/utils/token_length/token_limit_exceeded"
       autoload :OpenAIValidator, "langchain/utils/token_length/openai_validator"
       autoload :GooglePalmValidator, "langchain/utils/token_length/google_palm_validator"
     end
@@ -130,4 +142,8 @@ module Langchain
     autoload :PromptTemplate, "langchain/prompt/prompt_template"
     autoload :FewShotPromptTemplate, "langchain/prompt/few_shot_prompt_template"
   end
+
+  module Errors
+    class BaseError < StandardError; end
+  end
 end