langchainrb 0.5.0 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0a2fe8026e861c9d97465bce7da08a0b077492d6f7cf8fb42c45dbfdfe6749f
-  data.tar.gz: c04099c44a847bd9c05e8594859f92ca1f54d338c463ce59a375c2cb9731b1ad
+  metadata.gz: d36de4206b792714ba9b6773c03272e9638b14caf7140e0bc00c3e767aa5fdef
+  data.tar.gz: 819fab9de55a34e4e6dc865febc19bb9979df55fa8fc6a753774cf1961c40103
 SHA512:
-  metadata.gz: dec375b2b7cae377cf31f3f8ed0a6ac9d79215c945e7c0da78ed1fbad3c502ecfcc5ce5318c55a9a634db88fea1f5fbbeed7a0f7dc6ab8096c909e0a3ff02154
-  data.tar.gz: 558a0f6ddf90ad044f9e2cc7c6ca678958472748d2e430a3cbb4308290898b9b22a204a94a5b5943f06137f7da5238d038a65b8c79d96f8a3705499d95cfb597
+  metadata.gz: 6e180b41bbca96bd5523c276923f223bbebe470314086c6a909df440890793bcc70dbd66ecf59bf5d0fd52426650cc5d2684c56cc8fc643209cc1679527cbef4
+  data.tar.gz: af5db76c2b22b5c7bdc1170de437921e8464a16566f46a5cad465d69e6da47c97a82f7331a5ea5747840e58acc71463aa8456b03e9bc8851efda7b734e5d23cc

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [0.5.2] - 2023-06-07
+- 🗣️ LLMs
+  - Auto-calculate the max_tokens: setting to be passed on to OpenAI
+
+## [0.5.1] - 2023-06-06
+- 🛠️ Tools
+  - Modified Tool usage. Agents now accept Tools instances instead of Tool strings.
+
 ## [0.5.0] - 2023-06-05
 - [BREAKING] LLMs are now passed as objects to Vectorsearch classes instead of `llm: :name, llm_api_key:` previously
 - 📋 Prompts

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.5.0)
+    langchainrb (0.5.2)
       colorize (~> 0.8.1)
       tiktoken_ruby (~> 0.0.5)
 

data/README.md

@@ -256,7 +256,15 @@ Agents are semi-autonomous bots that can respond to user questions and use avail
 Add `gem "ruby-openai"`, `gem "eqn"`, and `gem "google_search_results"` to your Gemfile
 
 ```ruby
-agent = Langchain::Agent::ChainOfThoughtAgent.new(llm: Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"]), tools: ['search', 'calculator'])
+search_tool = Langchain::Tool::SerpApi.new(api_key: ENV["SERPAPI_API_KEY"])
+calculator = Langchain::Tool::Calculator.new
+
+openai = Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
+
+agent = Langchain::Agent::ChainOfThoughtAgent.new(
+  llm: openai,
+  tools: [search_tool, calculator]
+)
 
 agent.tools
 # => ["search", "calculator"]
@@ -271,11 +279,12 @@ agent.run(question: "How many full soccer fields would be needed to cover the di
 Add `gem "sequel"` to your Gemfile
 
 ```ruby
-agent = Langchain::Agent::SQLQueryAgent.new(llm: Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"]), db_connection_string: "postgres://user:password@localhost:5432/db_name")
+database = Langchain::Tool::Database.new(connection_string: "postgres://user:password@localhost:5432/db_name")
 
+agent = Langchain::Agent::SQLQueryAgent.new(llm: Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"]), db: database)
 ```
 ```ruby
-agent.ask(question: "How many users have a name with length greater than 5 in the users table?")
+agent.run(question: "How many users have a name with length greater than 5 in the users table?")
 #=> "14 users have a name with length greater than 5 in the users table."
 ```
 

data/examples/create_and_manage_few_shot_prompt_templates.rb

@@ -1,10 +1,10 @@
 require "langchain"
 
 # Create a prompt with a few shot examples
-prompt = Prompt::FewShotPromptTemplate.new(
+prompt = Langchain::Prompt::FewShotPromptTemplate.new(
   prefix: "Write antonyms for the following words.",
   suffix: "Input: {adjective}\nOutput:",
-  example_prompt: Prompt::PromptTemplate.new(
+  example_prompt: Langchain::Prompt::PromptTemplate.new(
     input_variables: ["input", "output"],
     template: "Input: {input}\nOutput: {output}"
   ),
@@ -32,5 +32,5 @@ prompt.format(adjective: "good")
 prompt.save(file_path: "spec/fixtures/prompt/few_shot_prompt_template.json")
 
 # Loading a new prompt template using a JSON file
-prompt = Prompt.load_from_path(file_path: "spec/fixtures/prompt/few_shot_prompt_template.json")
+prompt = Langchain::Prompt.load_from_path(file_path: "spec/fixtures/prompt/few_shot_prompt_template.json")
 prompt.prefix # "Write antonyms for the following words."

data/examples/create_and_manage_prompt_templates.rb

@@ -1,15 +1,15 @@
 require "langchain"
 
 # Create a prompt with one input variable
-prompt = Prompt::PromptTemplate.new(template: "Tell me a {adjective} joke.", input_variables: ["adjective"])
+prompt = Langchain::Prompt::PromptTemplate.new(template: "Tell me a {adjective} joke.", input_variables: ["adjective"])
 prompt.format(adjective: "funny") # "Tell me a funny joke."
 
 # Create a prompt with multiple input variables
-prompt = Prompt::PromptTemplate.new(template: "Tell me a {adjective} joke about {content}.", input_variables: ["adjective", "content"])
+prompt = Langchain::Prompt::PromptTemplate.new(template: "Tell me a {adjective} joke about {content}.", input_variables: ["adjective", "content"])
 prompt.format(adjective: "funny", content: "chickens") # "Tell me a funny joke about chickens."
 
 # Creating a PromptTemplate using just a prompt and no input_variables
-prompt = Prompt::PromptTemplate.from_template("Tell me a {adjective} joke about {content}.")
+prompt = Langchain::Prompt::PromptTemplate.from_template("Tell me a {adjective} joke about {content}.")
 prompt.input_variables # ["adjective", "content"]
 prompt.format(adjective: "funny", content: "chickens") # "Tell me a funny joke about chickens."
 
@@ -17,5 +17,9 @@ prompt.format(adjective: "funny", content: "chickens") # "Tell me a funny joke a
 prompt.save(file_path: "spec/fixtures/prompt/prompt_template.json")
 
 # Loading a new prompt template using a JSON file
-prompt = Prompt.load_from_path(file_path: "spec/fixtures/prompt/prompt_template.json")
+prompt = Langchain::Prompt.load_from_path(file_path: "spec/fixtures/prompt/prompt_template.json")
+prompt.input_variables # ["adjective", "content"]
+
+# Loading a new prompt template using a YAML file
+prompt = Langchain::Prompt.load_from_path(file_path: "spec/fixtures/prompt/prompt_template.yaml")
 prompt.input_variables # ["adjective", "content"]

data/examples/pdf_store_and_query_with_chroma.rb

@@ -4,7 +4,7 @@ require "langchain"
 # or add `gem "chroma-db", "~> 0.3.0"` to your Gemfile
 
 # Instantiate the Chroma client
-chroma = Vectorsearch::Chroma.new(
+chroma = Langchain::Vectorsearch::Chroma.new(
   url: ENV["CHROMA_URL"],
   index_name: "documents",
   llm: Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])

data/examples/store_and_query_with_pinecone.rb

@@ -4,7 +4,7 @@ require "langchain"
 # or add `gem "pinecone"` to your Gemfile
 
 # Instantiate the Qdrant client
-pinecone = Vectorsearch::Pinecone.new(
+pinecone = Langchain::Vectorsearch::Pinecone.new(
   environment: ENV["PINECONE_ENVIRONMENT"],
   api_key: ENV["PINECONE_API_KEY"],
   index_name: "recipes",
@@ -37,7 +37,7 @@ pinecone.ask(
 )
 
 # Generate your an embedding and search by it
-openai = LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
+openai = Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
 embedding = openai.embed(text: "veggie")
 
 pinecone.similarity_search_by_vector(

data/examples/store_and_query_with_qdrant.rb

@@ -4,7 +4,7 @@ require "langchain"
 # or add `gem "qdrant-ruby"` to your Gemfile
 
 # Instantiate the Qdrant client
-qdrant = Vectorsearch::Qdrant.new(
+qdrant = Langchain::Vectorsearch::Qdrant.new(
   url: ENV["QDRANT_URL"],
   api_key: ENV["QDRANT_API_KEY"],
   index_name: "recipes",

data/examples/store_and_query_with_weaviate.rb

@@ -4,7 +4,7 @@ require "langchain"
 # or add `gem "weaviate-ruby"` to your Gemfile
 
 # Instantiate the Weaviate client
-weaviate = Vectorsearch::Weaviate.new(
+weaviate = Langchain::Vectorsearch::Weaviate.new(
   url: ENV["WEAVIATE_URL"],
   api_key: ENV["WEAVIATE_API_KEY"],
   index_name: "Recipes",

data/lib/langchain/agent/chain_of_thought_agent/chain_of_thought_agent.rb

@@ -39,11 +39,8 @@ module Langchain::Agent
 
       loop do
         Langchain.logger.info("[#{self.class.name}]".red + ": Sending the prompt to the #{llm.class} LLM")
-        response = llm.complete(
-          prompt: prompt,
-          stop_sequences: ["Observation:"],
-          max_tokens: 500
-        )
+
+        response = llm.complete(prompt: prompt, stop_sequences: ["Observation:"])
 
         # Append the response to the prompt
         prompt += response
@@ -55,10 +52,11 @@ module Langchain::Agent
           # Find the input to the action in the "Action Input: [action_input]" format
           action_input = response.match(/Action Input: "?(.*)"?/)&.send(:[], -1)
 
-          # Retrieve the Tool::[ToolName] class and call `execute`` with action_input as the input
-          tool = Langchain::Tool.const_get(Langchain::Tool::Base::TOOLS[action.strip])
-          Langchain.logger.info("[#{self.class.name}]".red + ": Invoking \"#{tool}\" Tool with \"#{action_input}\"")
+          # Find the Tool and call `execute`` with action_input as the input
+          tool = tools.find { |tool| tool.tool_name == action.strip }
+          Langchain.logger.info("[#{self.class.name}]".red + ": Invoking \"#{tool.class}\" Tool with \"#{action_input}\"")
 
+          # Call `execute` with action_input as the input
           result = tool.execute(input: action_input)
 
           # Append the Observation to the prompt
@@ -81,12 +79,16 @@ module Langchain::Agent
     # @param tools [Array] Tools to use
     # @return [String] Prompt
     def create_prompt(question:, tools:)
+      tool_list = tools.map(&:tool_name)
+
       prompt_template.format(
         date: Date.today.strftime("%B %d, %Y"),
         question: question,
-        tool_names: "[#{tools.join(", ")}]",
+        tool_names: "[#{tool_list.join(", ")}]",
         tools: tools.map do |tool|
-          "#{tool}: #{Langchain::Tool.const_get(Langchain::Tool::Base::TOOLS[tool]).const_get(:DESCRIPTION)}"
+          tool_name = tool.tool_name
+          tool_description = tool.class.const_get(:DESCRIPTION)
+          "#{tool_name}: #{tool_description}"
         end.join("\n")
       )
     end

data/lib/langchain/agent/sql_query_agent/sql_query_agent.rb

@@ -4,26 +4,30 @@ module Langchain::Agent
   class SQLQueryAgent < Base
     attr_reader :llm, :db, :schema
 
+    #
     # Initializes the Agent
     #
     # @param llm [Object] The LLM client to use
-    # @param db_connection_string [String] Database connection info
-    def initialize(llm:, db_connection_string:)
+    # @param db [Object] Database connection info
+    #
+    def initialize(llm:, db:)
       @llm = llm
-      @db = Langchain::Tool::Database.new(db_connection_string)
+      @db = db
       @schema = @db.schema
     end
 
+    #
     # Ask a question and get an answer
     #
     # @param question [String] Question to ask the LLM/Database
     # @return [String] Answer to the question
-    def ask(question:)
+    #
+    def run(question:)
       prompt = create_prompt_for_sql(question: question)
 
       # Get the SQL string to execute
       Langchain.logger.info("[#{self.class.name}]".red + ":  Passing the inital prompt to the #{llm.class} LLM")
-      sql_string = llm.complete(prompt: prompt, max_tokens: 500)
+      sql_string = llm.complete(prompt: prompt)
 
       # Execute the SQL string and collect the results
       Langchain.logger.info("[#{self.class.name}]".red + ":  Passing the SQL to the Database: #{sql_string}")
@@ -32,7 +36,7 @@ module Langchain::Agent
       # Pass the results and get the LLM to synthesize the answer to the question
       Langchain.logger.info("[#{self.class.name}]".red + ":  Passing the synthesize prompt to the #{llm.class} LLM with results: #{results}")
       prompt2 = create_prompt_for_answer(question: question, sql_query: sql_string, results: results)
-      llm.complete(prompt: prompt2, max_tokens: 500)
+      llm.complete(prompt: prompt2)
     end
 
     private

data/lib/langchain/llm/openai.rb

@@ -35,7 +35,7 @@ module Langchain::LLM
     def embed(text:, **params)
       parameters = {model: DEFAULTS[:embeddings_model_name], input: text}
 
-      Langchain::Utils::TokenLengthValidator.validate!(text, parameters[:model])
+      Langchain::Utils::TokenLengthValidator.validate_max_tokens!(text, parameters[:model])
 
       response = client.embeddings(parameters: parameters.merge(params))
       response.dig("data").first.dig("embedding")
@@ -50,9 +50,8 @@ module Langchain::LLM
     def complete(prompt:, **params)
       parameters = compose_parameters DEFAULTS[:completion_model_name], params
 
-      Langchain::Utils::TokenLengthValidator.validate!(prompt, parameters[:model])
-
       parameters[:prompt] = prompt
+      parameters[:max_tokens] = Langchain::Utils::TokenLengthValidator.validate_max_tokens!(prompt, parameters[:model])
 
       response = client.completions(parameters: parameters)
       response.dig("choices", 0, "text")
@@ -67,9 +66,8 @@ module Langchain::LLM
     def chat(prompt:, **params)
       parameters = compose_parameters DEFAULTS[:chat_completion_model_name], params
 
-      Langchain::Utils::TokenLengthValidator.validate!(prompt, parameters[:model])
-
       parameters[:messages] = [{role: "user", content: prompt}]
+      parameters[:max_tokens] = Langchain::Utils::TokenLengthValidator.validate_max_tokens!(prompt, parameters[:model])
 
       response = client.chat(parameters: parameters)
       response.dig("choices", 0, "message", "content")
@@ -87,12 +85,7 @@ module Langchain::LLM
       )
       prompt = prompt_template.format(text: text)
 
-      complete(
-        prompt: prompt,
-        temperature: DEFAULTS[:temperature],
-        # Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
-        max_tokens: 2048
-      )
+      complete(prompt: prompt, temperature: DEFAULTS[:temperature])
     end
 
     private

data/lib/langchain/tool/base.rb

@@ -6,47 +6,59 @@ module Langchain::Tool
 
     # How to add additional Tools?
     # 1. Create a new file in lib/tool/your_tool_name.rb
-    # 2. Add your tool to the TOOLS hash below
-    #   "your_tool_name" => "Tool::YourToolName"
-    # 3. Implement `self.execute(input:)` method in your tool class
-    # 4. Add your tool to the README.md
+    # 2. Create a class in the file that inherits from Langchain::Tool::Base
+    # 3. Add `NAME=` and `DESCRIPTION=` constants in your Tool class
+    # 4. Implement `execute(input:)` method in your tool class
+    # 5. Add your tool to the README.md
 
-    TOOLS = {
-      "calculator" => "Langchain::Tool::Calculator",
-      "search" => "Langchain::Tool::SerpApi",
-      "wikipedia" => "Langchain::Tool::Wikipedia",
-      "database" => "Langchain::Tool::Database"
-    }
+    #
+    # Returns the NAME constant of the tool
+    #
+    # @return [String] tool name
+    #
+    def tool_name
+      self.class.const_get(:NAME)
+    end
 
+    #
+    # Sets the DESCRIPTION constant of the tool
+    #
+    # @param value [String] tool description
+    #
     def self.description(value)
       const_set(:DESCRIPTION, value.tr("\n", " ").strip)
     end
 
+    #
     # Instantiates and executes the tool and returns the answer
+    #
     # @param input [String] input to the tool
     # @return [String] answer
+    #
     def self.execute(input:)
       new.execute(input: input)
     end
 
+    #
     # Executes the tool and returns the answer
+    #
     # @param input [String] input to the tool
     # @return [String] answer
+    #
     def execute(input:)
       raise NotImplementedError, "Your tool must implement the `#execute(input:)` method that returns a string"
     end
 
     #
-    # Validates the list of strings (tools) are all supported or raises an error
-    # @param tools [Array<String>] list of tools to be used
+    # Validates the list of tools or raises an error
+    # @param tools [Array<Langchain::Tool>] list of tools to be used
     #
     # @raise [ArgumentError] If any of the tools are not supported
     #
     def self.validate_tools!(tools:)
-      unrecognized_tools = tools - Langchain::Tool::Base::TOOLS.keys
-
-      if unrecognized_tools.any?
-        raise ArgumentError, "Unrecognized Tools: #{unrecognized_tools}"
+      # Check if the tool count is equal to unique tool count
+      if tools.count != tools.map(&:tool_name).uniq.count
+        raise ArgumentError, "Either tools are not unique or are conflicting with each other"
       end
     end
   end

data/lib/langchain/tool/calculator.rb

@@ -8,9 +8,10 @@ module Langchain::Tool
     # Gem requirements:
     #   gem "eqn", "~> 1.6.5"
     #   gem "google_search_results", "~> 2.0.0"
-    # ENV requirements: ENV["SERPAPI_API_KEY"]
     #
 
+    NAME = "calculator"
+
     description <<~DESC
       Useful for getting the result of a math expression.
 
@@ -33,8 +34,12 @@ module Langchain::Tool
     rescue Eqn::ParseError, Eqn::NoVariableValueError
       # Sometimes the input is not a pure math expression, e.g: "12F in Celsius"
       # We can use the google answer box to evaluate this expression
-      hash_results = Langchain::Tool::SerpApi.execute_search(input: input)
-      hash_results.dig(:answer_box, :to)
+      # TODO: Figure out to find a better way to evaluate these language expressions.
+      hash_results = Langchain::Tool::SerpApi
+        .new(api_key: ENV["SERPAPI_API_KEY"])
+        .execute_search(input: input)
+      hash_results.dig(:answer_box, :to) ||
+        hash_results.dig(:answer_box, :result)
     end
   end
 end

data/lib/langchain/tool/database.rb

@@ -6,40 +6,55 @@ module Langchain::Tool
     # Gem requirements: gem "sequel", "~> 5.68.0"
     #
 
+    NAME = "database"
+
     description <<~DESC
       Useful for getting the result of a database query.
 
       The input to this tool should be valid SQL.
     DESC
 
+    attr_reader :db
+
+    #
     # Establish a database connection
-    # @param db_connection_string [String] Database connection info, e.g. 'postgres://user:password@localhost:5432/db_name'
-    def initialize(db_connection_string)
+    #
+    # @param connection_string [String] Database connection info, e.g. 'postgres://user:password@localhost:5432/db_name'
+    # @return [Database] Database object
+    #
+    def initialize(connection_string:)
       depends_on "sequel"
       require "sequel"
       require "sequel/extensions/schema_dumper"
 
-      raise StandardError, "db_connection_string parameter cannot be blank" if db_connection_string.empty?
+      raise StandardError, "connection_string parameter cannot be blank" if connection_string.empty?
 
-      @db = Sequel.connect(db_connection_string)
+      @db = Sequel.connect(connection_string)
       @db.extension :schema_dumper
     end
 
+    #
+    # Returns the database schema
+    #
+    # @return [String] schema
+    #
     def schema
       Langchain.logger.info("[#{self.class.name}]".light_blue + ": Dumping schema")
-      @db.dump_schema_migration(same_db: true, indexes: false) unless @db.adapter_scheme == :mock
+      db.dump_schema_migration(same_db: true, indexes: false) unless db.adapter_scheme == :mock
     end
 
+    #
     # Evaluates a sql expression
+    #
     # @param input [String] sql expression
     # @return [Array] results
+    #
     def execute(input:)
       Langchain.logger.info("[#{self.class.name}]".light_blue + ": Executing \"#{input}\"")
-      begin
-        @db[input].to_a
-      rescue Sequel::DatabaseError => e
-        Langchain.logger.error("[#{self.class.name}]".light_red + ": #{e.message}")
-      end
+
+      db[input].to_a
+    rescue Sequel::DatabaseError => e
+      Langchain.logger.error("[#{self.class.name}]".light_red + ": #{e.message}")
     end
   end
 end

data/lib/langchain/tool/ruby_code_interpreter.rb

@@ -7,7 +7,7 @@ module Langchain::Tool
     #
     # Gem requirements: gem "safe_ruby", "~> 1.0.4"
     #
-
+    NAME = "ruby_code_interpreter"
     description <<~DESC
       A Ruby code interpreter. Use this to execute ruby expressions. Input should be a valid ruby expression. If you want to see the output of the tool, make sure to return a value.
     DESC

data/lib/langchain/tool/serp_api.rb

@@ -6,8 +6,13 @@ module Langchain::Tool
     # Wrapper around SerpAPI
     #
     # Gem requirements: gem "google_search_results", "~> 2.0.0"
-    # ENV requirements: ENV["SERPAPI_API_KEY"] # https://serpapi.com/manage-api-key)
     #
+    # Usage:
+    # search = Langchain::Tool::SerpApi.new(api_key: "YOUR_API_KEY")
+    # search.execute(input: "What is the capital of France?")
+    #
+
+    NAME = "search"
 
     description <<~DESC
       A wrapper around Google Search.
@@ -18,39 +23,57 @@ module Langchain::Tool
       Input should be a search query.
     DESC
 
-    def initialize
+    attr_reader :api_key
+
+    #
+    # Initializes the SerpAPI tool
+    #
+    # @param api_key [String] SerpAPI API key
+    # @return [Langchain::Tool::SerpApi] SerpAPI tool
+    #
+    def initialize(api_key:)
       depends_on "google_search_results"
       require "google_search_results"
+      @api_key = api_key
     end
 
+    #
     # Executes Google Search and returns hash_results JSON
+    #
     # @param input [String] search query
     # @return [Hash] hash_results JSON
-
+    #
     def self.execute_search(input:)
       new.execute_search(input: input)
     end
 
-    # Executes Google Search and returns hash_results JSON
+    #
+    # Executes Google Search and returns the result
+    #
     # @param input [String] search query
     # @return [String] Answer
-    # TODO: Glance at all of the fields that langchain Python looks through: https://github.com/hwchase17/langchain/blob/v0.0.166/langchain/utilities/serpapi.py#L128-L156
-    # We may need to do the same thing here.
+    #
     def execute(input:)
       Langchain.logger.info("[#{self.class.name}]".light_blue + ": Executing \"#{input}\"")
 
       hash_results = execute_search(input: input)
 
+      # TODO: Glance at all of the fields that langchain Python looks through: https://github.com/hwchase17/langchain/blob/v0.0.166/langchain/utilities/serpapi.py#L128-L156
+      # We may need to do the same thing here.
       hash_results.dig(:answer_box, :answer) ||
         hash_results.dig(:answer_box, :snippet) ||
         hash_results.dig(:organic_results, 0, :snippet)
     end
 
+    #
+    # Executes Google Search and returns hash_results JSON
+    #
+    # @param input [String] search query
+    # @return [Hash] hash_results JSON
+    #
     def execute_search(input:)
-      GoogleSearch.new(
-        q: input,
-        serp_api_key: ENV["SERPAPI_API_KEY"]
-      )
+      GoogleSearch
+        .new(q: input, serp_api_key: api_key)
         .get_hash
     end
   end

data/lib/langchain/tool/wikipedia.rb

@@ -7,7 +7,7 @@ module Langchain::Tool
     #
     # Gem requirements: gem "wikipedia-client", "~> 1.17.0"
     #
-
+    NAME = "wikipedia"
     description <<~DESC
       A wrapper around Wikipedia.
 

data/lib/langchain/utils/token_length_validator.rb

@@ -34,23 +34,50 @@ module Langchain
         "ada" => 2049
       }.freeze
 
+      # GOOGLE_PALM_TOKEN_LIMITS = {
+      #   "chat-bison-001" => {
+      #     "inputTokenLimit"=>4096,
+      #     "outputTokenLimit"=>1024
+      #   },
+      #   "text-bison-001" => {
+      #     "inputTokenLimit"=>8196,
+      #     "outputTokenLimit"=>1024
+      #   },
+      #   "embedding-gecko-001" => {
+      #     "inputTokenLimit"=>1024
+      #   }
+      # }.freeze
+
       #
-      # Validate the length of the text passed in to OpenAI's API
+      # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
       #
       # @param text [String] The text to validate
       # @param model_name [String] The model name to validate against
-      # @return [Boolean] Whether the text is valid or not
+      # @return [Integer] Whether the text is valid or not
       # @raise [TokenLimitExceeded] If the text is too long
       #
-      def self.validate!(text, model_name)
-        encoder = Tiktoken.encoding_for_model(model_name)
-        token_length = encoder.encode(text).length
+      def self.validate_max_tokens!(text, model_name)
+        text_token_length = token_length(text, model_name)
+        max_tokens = TOKEN_LIMITS[model_name] - text_token_length
 
-        if token_length > TOKEN_LIMITS[model_name]
-          raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS[model_name]} tokens, but the given text is #{token_length} tokens long."
+        # Raise an error even if whole prompt is equal to the model's token limit (max_tokens == 0) since not response will be returned
+        if max_tokens <= 0
+          raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS[model_name]} tokens, but the given text is #{text_token_length} tokens long."
         end
 
-        true
+        max_tokens
+      end
+
+      #
+      # Calculate token length for a given text and model name
+      #
+      # @param text [String] The text to validate
+      # @param model_name [String] The model name to validate against
+      # @return [Integer] The token length of the text
+      #
+      def self.token_length(text, model_name)
+        encoder = Tiktoken.encoding_for_model(model_name)
+        encoder.encode(text).length
       end
     end
   end

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.5.0"
+  VERSION = "0.5.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.2
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-05 00:00:00.000000000 Z
+date: 2023-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tiktoken_ruby