langchainrb 0.5.5 → 0.5.7

data/lib/langchain/utils/token_length/openai_validator.rb

@@ -9,7 +9,7 @@ module Langchain
       # This class is meant to validate the length of the text passed in to OpenAI's API.
       # It is used to validate the token length before the API call is made
       #
-      class OpenAIValidator
+      class OpenAIValidator < BaseValidator
         TOKEN_LIMITS = {
           # Source:
           # https://platform.openai.com/docs/api-reference/embeddings
@@ -17,6 +17,9 @@ module Langchain
           "text-embedding-ada-002" => 8191,
           "gpt-3.5-turbo" => 4096,
           "gpt-3.5-turbo-0301" => 4096,
+          "gpt-3.5-turbo-0613" => 4096,
+          "gpt-3.5-turbo-16k" => 16384,
+          "gpt-3.5-turbo-16k-0613" => 16384,
           "text-davinci-003" => 4097,
           "text-davinci-002" => 4097,
           "code-davinci-002" => 8001,
@@ -24,6 +27,7 @@ module Langchain
           "gpt-4-0314" => 8192,
           "gpt-4-32k" => 32768,
           "gpt-4-32k-0314" => 32768,
+          "gpt-4-32k-0613" => 32768,
           "text-curie-001" => 2049,
           "text-babbage-001" => 2049,
           "text-ada-001" => 2049,
@@ -33,31 +37,6 @@ module Langchain
           "ada" => 2049
         }.freeze
 
-        #
-        # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
-        #
-        # @param content [String | Array<String>] The text or array of texts to validate
-        # @param model_name [String] The model name to validate against
-        # @return [Integer] Whether the text is valid or not
-        # @raise [TokenLimitExceeded] If the text is too long
-        #
-        def self.validate_max_tokens!(content, model_name)
-          text_token_length = if content.is_a?(Array)
-            content.sum { |item| token_length(item.to_json, model_name) }
-          else
-            token_length(content, model_name)
-          end
-
-          max_tokens = TOKEN_LIMITS[model_name] - text_token_length
-
-          # Raise an error even if whole prompt is equal to the model's token limit (max_tokens == 0) since not response will be returned
-          if max_tokens <= 0
-            raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS[model_name]} tokens, but the given text is #{text_token_length} tokens long."
-          end
-
-          max_tokens
-        end
-
         #
         # Calculate token length for a given text and model name
         #
@@ -65,10 +44,14 @@ module Langchain
         # @param model_name [String] The model name to validate against
         # @return [Integer] The token length of the text
         #
-        def self.token_length(text, model_name)
+        def self.token_length(text, model_name, options = {})
           encoder = Tiktoken.encoding_for_model(model_name)
           encoder.encode(text).length
         end
+
+        def self.token_limit(model_name)
+          TOKEN_LIMITS[model_name]
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/token_limit_exceeded.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Utils
+    module TokenLength
+      class TokenLimitExceeded < StandardError
+        attr_reader :token_overflow
+
+        def initialize(message = "", token_overflow = 0)
+          super message
+
+          @token_overflow = token_overflow
+        end
+      end
+    end
+  end
+end

data/lib/langchain/vectorsearch/base.rb

@@ -155,5 +155,11 @@ module Langchain::Vectorsearch
 
       add_texts(texts: texts)
     end
+
+    def self.logger_options
+      {
+        color: :blue
+      }
+    end
   end
 end

data/lib/langchain/vectorsearch/chroma.rb

@@ -8,7 +8,7 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "chroma-db", "~> 0.3.0"
     #
     # Usage:
-    # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, api_key: nil)
+    # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, llm_api_key:, api_key: nil)
     #
 
     # Initialize the Chroma client

data/lib/langchain/vectorsearch/hnswlib.rb

@@ -110,12 +110,12 @@ module Langchain::Vectorsearch
       if File.exist?(path_to_index)
         client.load_index(path_to_index)
 
-        Langchain.logger.info("[#{self.class.name}]".blue + ": Successfully loaded the index at \"#{path_to_index}\"")
+        Langchain.logger.info("Successfully loaded the index at \"#{path_to_index}\"", for: self.class)
       else
         # Default max_elements: 100, but we constantly resize the index as new data is written to it
         client.init_index(max_elements: 100)
 
-        Langchain.logger.info("[#{self.class.name}]".blue + ": Creating a new index at \"#{path_to_index}\"")
+        Langchain.logger.info("Creating a new index at \"#{path_to_index}\"", for: self.class)
       end
     end
   end

data/lib/langchain/vectorsearch/milvus.rb

@@ -8,17 +8,9 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "milvus", "~> 0.9.0"
     #
     # Usage:
-    # milvus = Langchain::Vectorsearch::Milvus.new(url:, index_name:, llm:)
+    # milvus = Langchain::Vectorsearch::Milvus.new(url:, index_name:, llm:, llm_api_key:)
     #
 
-    #
-    # Initialize the Milvus client
-    #
-    # @param url [String] The URL of the Milvus server
-    # @param api_key [String] The API key to use
-    # @param index_name [String] The name of the index to use
-    # @param llm [Object] The LLM client to use
-    #
     def initialize(url:, index_name:, llm:, api_key: nil)
       depends_on "milvus"
       require "milvus"
@@ -29,11 +21,6 @@ module Langchain::Vectorsearch
       super(llm: llm)
     end
 
-    #
-    # Add a list of texts to the index
-    #
-    # @param texts [Array] The list of texts to add
-    #
     def add_texts(texts:)
       client.entities.insert(
         collection_name: index_name,

data/lib/langchain/vectorsearch/pgvector.rb

@@ -8,7 +8,7 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "pgvector", "~> 0.2"
     #
     # Usage:
-    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:)
+    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:, llm_api_key:)
     #
 
     # The operators supported by the PostgreSQL vector search adapter
@@ -20,14 +20,10 @@ module Langchain::Vectorsearch
 
     attr_reader :operator, :quoted_table_name
 
-    #
-    # Initialize the PostgreSQL client
-    #
     # @param url [String] The URL of the PostgreSQL database
     # @param index_name [String] The name of the table to use for the index
     # @param llm [Object] The LLM client to use
     # @param api_key [String] The API key for the Vectorsearch DB (not used for PostgreSQL)
-    #
     def initialize(url:, index_name:, llm:, api_key: nil)
       require "pg"
       require "pgvector"

data/lib/langchain/vectorsearch/pinecone.rb

@@ -8,17 +8,14 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "pinecone", "~> 0.1.6"
     #
     # Usage:
-    # pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:)
+    # pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
-    #
     # Initialize the Pinecone client
-    #
     # @param environment [String] The environment to use
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
-    #
     def initialize(environment:, api_key:, index_name:, llm:)
       depends_on "pinecone"
       require "pinecone"

data/lib/langchain/vectorsearch/qdrant.rb

@@ -8,17 +8,14 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "qdrant-ruby", "~> 0.9.0"
     #
     # Usage:
-    # qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:)
+    # qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
-    #
     # Initialize the Qdrant client
-    #
     # @param url [String] The URL of the Qdrant server
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
-    #
     def initialize(url:, api_key:, index_name:, llm:)
       depends_on "qdrant-ruby"
       require "qdrant"

data/lib/langchain/vectorsearch/weaviate.rb

@@ -8,17 +8,14 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "weaviate-ruby", "~> 0.8.0"
     #
     # Usage:
-    # weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:)
+    # weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:, llm_api_key:)
     #
 
-    #
     # Initialize the Weaviate adapter
-    #
     # @param url [String] The URL of the Weaviate instance
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
-    #
     def initialize(url:, api_key:, index_name:, llm:)
       depends_on "weaviate-ruby"
       require "weaviate"

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.5.5"
+  VERSION = "0.5.7"
 end

data/lib/langchain.rb

@@ -46,36 +46,48 @@ require_relative "./langchain/version"
 #
 # LangChain.rb uses standard logging mechanisms and defaults to :debug level. Most messages are at info level, but we will add debug or warn statements as needed. To show all log messages:
 #
-#     Langchain.logger.level = :info
+# Langchain.logger.level = :info
 module Langchain
+  autoload :Loader, "langchain/loader"
+  autoload :Data, "langchain/data"
+  autoload :Conversation, "langchain/conversation"
+  autoload :DependencyHelper, "langchain/dependency_helper"
+  autoload :ContextualLogger, "langchain/contextual_logger"
+
   class << self
-    # @return [Logger]
-    attr_accessor :logger
+    # @return [ContextualLogger]
+    attr_reader :logger
+
+    # @param logger [Logger]
+    # @return [ContextualLogger]
+    def logger=(logger)
+      @logger = ContextualLogger.new(logger)
+    end
 
     # @return [Pathname]
     attr_reader :root
   end
 
-  @logger ||= ::Logger.new($stdout, level: :warn, formatter: ->(severity, datetime, progname, msg) { "[LangChain.rb]".yellow + " #{msg}\n" })
+  self.logger ||= ::Logger.new($stdout, level: :warn)
 
   @root = Pathname.new(__dir__)
 
-  autoload :Loader, "langchain/loader"
-  autoload :Data, "langchain/data"
-  autoload :Conversation, "langchain/conversation"
-  autoload :DependencyHelper, "langchain/dependency_helper"
-
   module Agent
     autoload :Base, "langchain/agent/base"
     autoload :ChainOfThoughtAgent, "langchain/agent/chain_of_thought_agent/chain_of_thought_agent.rb"
     autoload :SQLQueryAgent, "langchain/agent/sql_query_agent/sql_query_agent.rb"
   end
 
+  module Chunker
+    autoload :Base, "langchain/chunker/base"
+    autoload :Text, "langchain/chunker/text"
+  end
+
   module Tool
     autoload :Base, "langchain/tool/base"
     autoload :Calculator, "langchain/tool/calculator"
     autoload :RubyCodeInterpreter, "langchain/tool/ruby_code_interpreter"
-    autoload :SerpApi, "langchain/tool/serp_api"
+    autoload :GoogleSearch, "langchain/tool/google_search"
     autoload :Weather, "langchain/tool/weather"
     autoload :Wikipedia, "langchain/tool/wikipedia"
     autoload :Database, "langchain/tool/database"
@@ -95,8 +107,8 @@ module Langchain
 
   module Utils
     module TokenLength
-      class TokenLimitExceeded < StandardError; end
-
+      autoload :BaseValidator, "langchain/utils/token_length/base_validator"
+      autoload :TokenLimitExceeded, "langchain/utils/token_length/token_limit_exceeded"
       autoload :OpenAIValidator, "langchain/utils/token_length/openai_validator"
       autoload :GooglePalmValidator, "langchain/utils/token_length/google_palm_validator"
     end
@@ -130,4 +142,8 @@ module Langchain
     autoload :PromptTemplate, "langchain/prompt/prompt_template"
     autoload :FewShotPromptTemplate, "langchain/prompt/few_shot_prompt_template"
   end
+
+  module Errors
+    class BaseError < StandardError; end
+  end
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.5.7
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-12 00:00:00.000000000 Z
+date: 2023-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: tiktoken_ruby
+  name: baran
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.5
+        version: 0.1.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.5
+        version: 0.1.6
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.1
+- !ruby/object:Gem::Dependency
+  name: tiktoken_ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
 - !ruby/object:Gem::Dependency
   name: dotenv-rails
   requirement: !ruby/object:Gem::Requirement
@@ -456,10 +470,13 @@ files:
 - lib/langchain.rb
 - lib/langchain/agent/base.rb
 - lib/langchain/agent/chain_of_thought_agent/chain_of_thought_agent.rb
-- lib/langchain/agent/chain_of_thought_agent/chain_of_thought_agent_prompt.json
+- lib/langchain/agent/chain_of_thought_agent/chain_of_thought_agent_prompt.yaml
 - lib/langchain/agent/sql_query_agent/sql_query_agent.rb
-- lib/langchain/agent/sql_query_agent/sql_query_agent_answer_prompt.json
-- lib/langchain/agent/sql_query_agent/sql_query_agent_sql_prompt.json
+- lib/langchain/agent/sql_query_agent/sql_query_agent_answer_prompt.yaml
+- lib/langchain/agent/sql_query_agent/sql_query_agent_sql_prompt.yaml
+- lib/langchain/chunker/base.rb
+- lib/langchain/chunker/text.rb
+- lib/langchain/contextual_logger.rb
 - lib/langchain/conversation.rb
 - lib/langchain/data.rb
 - lib/langchain/dependency_helper.rb
@@ -469,7 +486,7 @@ files:
 - lib/langchain/llm/google_palm.rb
 - lib/langchain/llm/hugging_face.rb
 - lib/langchain/llm/openai.rb
-- lib/langchain/llm/prompts/summarize_template.json
+- lib/langchain/llm/prompts/summarize_template.yaml
 - lib/langchain/llm/replicate.rb
 - lib/langchain/loader.rb
 - lib/langchain/processors/base.rb
@@ -488,12 +505,14 @@ files:
 - lib/langchain/tool/base.rb
 - lib/langchain/tool/calculator.rb
 - lib/langchain/tool/database.rb
+- lib/langchain/tool/google_search.rb
 - lib/langchain/tool/ruby_code_interpreter.rb
-- lib/langchain/tool/serp_api.rb
 - lib/langchain/tool/weather.rb
 - lib/langchain/tool/wikipedia.rb
+- lib/langchain/utils/token_length/base_validator.rb
 - lib/langchain/utils/token_length/google_palm_validator.rb
 - lib/langchain/utils/token_length/openai_validator.rb
+- lib/langchain/utils/token_length/token_limit_exceeded.rb
 - lib/langchain/vectorsearch/base.rb
 - lib/langchain/vectorsearch/chroma.rb
 - lib/langchain/vectorsearch/hnswlib.rb
@@ -528,7 +547,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: Build LLM-backed Ruby applications with Ruby's LangChain

data/lib/langchain/agent/chain_of_thought_agent/chain_of_thought_agent_prompt.json

@@ -1,10 +0,0 @@
-{
-  "_type": "prompt",
-  "template": "Today is {date} and you can use tools to get new information. Answer the following questions as best you can using the following tools:\n\n{tools}\n\nUse the following format:\n\nQuestion: the input question you must answer\nThought: you should always think about what to do\nAction: the action to take, should be one of {tool_names}\nAction Input: the input to the action\nObservation: the result of the action\n... (this Thought/Action/Action Input/Observation can repeat N times)\nThought: I now know the final answer\nFinal Answer: the final answer to the original input question\n\nBegin!\n\nQuestion: {question}\nThought:",
-  "input_variables": [
-    "date",
-    "question",
-    "tools",
-    "tool_names"
-  ]
-}

data/lib/langchain/agent/sql_query_agent/sql_query_agent_answer_prompt.json

@@ -1,10 +0,0 @@
-{
-  "_type": "prompt",
-  "template":
-    "Given an input question and results of a SQL query, look at the results and return the answer. Use the following format:\nQuestion: {question}\nThe SQL query: {sql_query}\nResult of the SQLQuery: {results}\nFinal answer: Final answer here",
-  "input_variables": [
-    "question",
-    "sql_query",
-    "results"
-  ]
-}

data/lib/langchain/agent/sql_query_agent/sql_query_agent_sql_prompt.json

@@ -1,10 +0,0 @@
-{
-  "_type": "prompt",
-   "template":
-    "Given an input question, create a syntactically correct {dialect} query to run, then return the query in valid SQL.\nNever query for all the columns from a specific table, only ask for a the few relevant columns given the question.\nPay attention to use only the column names that you can see in the schema description. Be careful to not query for columns that do not exist. Pay attention to which column is in which table. Also, qualify column names with the table name when needed.\nOnly use the tables listed below.\n{schema}\nUse the following format:\nQuestion: {question}\nSQLQuery:",
-  "input_variables": [
-    "dialect",
-    "schema",
-    "question"
-  ]
-}

data/lib/langchain/llm/prompts/summarize_template.json

@@ -1,5 +0,0 @@
-{
-  "_type": "prompt",
-  "input_variables": ["text"],
-  "template": "Write a concise summary of the following:\n\n{text}\n\nCONCISE SUMMARY:"
-}