langchainrb 0.8.1 → 0.9.0

data/lib/langchain/tool/base.rb

@@ -91,6 +91,30 @@ module Langchain::Tool
       new.execute(input: input)
     end
 
+    # Returns the tool as an OpenAI tool
+    #
+    # @return [Hash] tool as an OpenAI tool
+    def to_openai_tool
+      # TODO: This is hardcoded to def execute(input:) found in each tool, needs to be dynamic.
+      {
+        type: "function",
+        function: {
+          name: name,
+          description: description,
+          parameters: {
+            type: "object",
+            properties: {
+              input: {
+                type: "string",
+                description: "Input to the tool"
+              }
+            },
+            required: ["input"]
+          }
+        }
+      }
+    end
+
     #
     # Executes the tool and returns the answer
     #

data/lib/langchain/tool/google_search.rb

@@ -17,10 +17,7 @@ module Langchain::Tool
     description <<~DESC
       A wrapper around SerpApi's Google Search API.
 
-      Useful for when you need to answer questions about current events.
-      Always one of the first options when you need to find information on internet.
-
-      Input should be a search query.
+      Useful for when you need to answer questions about current events. Always one of the first options when you need to find information on internet. Input should be a search query.
     DESC
 
     attr_reader :api_key

data/lib/langchain/utils/token_length/ai21_validator.rb

@@ -22,8 +22,8 @@ module Langchain
         # @param model_name [String] The model name to validate against
         # @return [Integer] The token length of the text
         #
-        def self.token_length(text, model_name, client)
-          res = client.tokenize(text)
+        def self.token_length(text, model_name, options = {})
+          res = options[:llm].tokenize(text)
           res.dig(:tokens).length
         end
 
@@ -31,6 +31,10 @@ module Langchain
           TOKEN_LIMITS[model_name]
         end
         singleton_class.alias_method :completion_token_limit, :token_limit
+
+        def self.token_length_from_messages(messages, model_name, options)
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/base_validator.rb

@@ -14,7 +14,7 @@ module Langchain
       class BaseValidator
         def self.validate_max_tokens!(content, model_name, options = {})
           text_token_length = if content.is_a?(Array)
-            content.sum { |item| token_length(item.to_json, model_name, options) }
+            token_length_from_messages(content, model_name, options)
           else
             token_length(content, model_name, options)
           end

data/lib/langchain/utils/token_length/cohere_validator.rb

@@ -30,8 +30,8 @@ module Langchain
         # @param model_name [String] The model name to validate against
         # @return [Integer] The token length of the text
         #
-        def self.token_length(text, model_name, client)
-          res = client.tokenize(text: text)
+        def self.token_length(text, model_name, options = {})
+          res = options[:llm].tokenize(text: text)
           res["tokens"].length
         end
 
@@ -39,6 +39,10 @@ module Langchain
           TOKEN_LIMITS[model_name]
         end
         singleton_class.alias_method :completion_token_limit, :token_limit
+
+        def self.token_length_from_messages(messages, model_name, options)
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/google_palm_validator.rb

@@ -35,7 +35,7 @@ module Langchain
         # @option options [Langchain::LLM:GooglePalm] :llm The Langchain::LLM:GooglePalm instance
         # @return [Integer] The token length of the text
         #
-        def self.token_length(text, model_name = "chat-bison-001", options)
+        def self.token_length(text, model_name = "chat-bison-001", options = {})
           response = options[:llm].client.count_message_tokens(model: model_name, prompt: text)
 
           raise Langchain::LLM::ApiError.new(response["error"]["message"]) unless response["error"].nil?
@@ -43,6 +43,10 @@ module Langchain
           response.dig("tokenCount")
         end
 
+        def self.token_length_from_messages(messages, model_name, options = {})
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
+
         def self.token_limit(model_name)
           TOKEN_LIMITS.dig(model_name, "input_token_limit")
         end

data/lib/langchain/utils/token_length/openai_validator.rb

@@ -75,6 +75,47 @@ module Langchain
           max_tokens = super(content, model_name, options)
           [options[:max_tokens], max_tokens].reject(&:nil?).min
         end
+
+        # Copied from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+        # Return the number of tokens used by a list of messages
+        #
+        # @param messages [Array<Hash>] The messages to calculate the token length for
+        # @param model [String] The model name to validate against
+        # @return [Integer] The token length of the messages
+        #
+        def self.token_length_from_messages(messages, model_name, options = {})
+          encoding = Tiktoken.encoding_for_model(model_name)
+
+          if ["gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613", "gpt-4-0314", "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613"].include?(model_name)
+            tokens_per_message = 3
+            tokens_per_name = 1
+          elsif model_name == "gpt-3.5-turbo-0301"
+            tokens_per_message = 4  # every message follows {role/name}\n{content}\n
+            tokens_per_name = -1  # if there's a name, the role is omitted
+          elsif model_name.include?("gpt-3.5-turbo")
+            # puts "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613."
+            return token_length_from_messages(messages, "gpt-3.5-turbo-0613", options)
+          elsif model_name.include?("gpt-4")
+            # puts "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613."
+            return token_length_from_messages(messages, "gpt-4-0613", options)
+          else
+            raise NotImplementedError.new(
+              "token_length_from_messages() is not implemented for model #{model_name}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."
+            )
+          end
+
+          num_tokens = 0
+          messages.each do |message|
+            num_tokens += tokens_per_message
+            message.each do |key, value|
+              num_tokens += encoding.encode(value).length
+              num_tokens += tokens_per_name if ["name", :name].include?(key)
+            end
+          end
+
+          num_tokens += 3  # every reply is primed with assistant
+          num_tokens
+        end
       end
     end
   end

data/lib/langchain/vectorsearch/base.rb

@@ -175,13 +175,13 @@ module Langchain::Vectorsearch
       prompt_template.format(question: question, context: context)
     end
 
-    def add_data(paths:)
+    def add_data(paths:, options: {}, chunker: Langchain::Chunker::Text)
       raise ArgumentError, "Paths must be provided" if Array(paths).empty?
 
       texts = Array(paths)
         .flatten
         .map do |path|
-          data = Langchain::Loader.new(path)&.load&.chunks
+          data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
           data.map { |chunk| chunk.text }
         end
 

data/lib/langchain/vectorsearch/epsilla.rb

@@ -36,7 +36,11 @@ module Langchain::Vectorsearch
         status_code, response = @client.database.load_db(db_name, db_path)
 
         if status_code != 200
-          if status_code == 500 && response["message"].include?("already loaded")
+          if status_code == 409 || (status_code == 500 && response["message"].include?("already loaded"))
+            # When db is already loaded, Epsilla may return HTTP 409 Conflict.
+            # This behavior is changed in https://github.com/epsilla-cloud/vectordb/pull/95
+            # Old behavior (HTTP 500) is preserved for backwards compatibility.
+            # It does not prevent us from using the db.
             Langchain.logger.info("Database already loaded")
           else
             raise "Failed to load database: #{response}"

data/lib/langchain/vectorsearch/pinecone.rb

@@ -64,13 +64,13 @@ module Langchain::Vectorsearch
       index.upsert(vectors: vectors, namespace: namespace)
     end
 
-    def add_data(paths:, namespace: "")
+    def add_data(paths:, namespace: "", options: {}, chunker: Langchain::Chunker::Text)
       raise ArgumentError, "Paths must be provided" if Array(paths).empty?
 
       texts = Array(paths)
         .flatten
         .map do |path|
-          data = Langchain::Loader.new(path)&.load&.chunks
+          data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
           data.map { |chunk| chunk.text }
         end
 

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.8.1"
+  VERSION = "0.9.0"
 end

data/lib/langchain.rb

@@ -24,6 +24,7 @@ loader.inflector.inflect(
   "sql_query_agent" => "SQLQueryAgent"
 )
 loader.collapse("#{__dir__}/langchain/llm/response")
+loader.collapse("#{__dir__}/langchain/assistants")
 loader.setup
 
 # Langchain.rb a is library for building LLM-backed Ruby applications. It is an abstraction layer that sits on top of the emerging AI-related tools that makes it easy for developers to consume and string those services together.
@@ -82,7 +83,7 @@ module Langchain
     attr_reader :root
   end
 
-  self.logger ||= ::Logger.new($stdout, level: :warn)
+  self.logger ||= ::Logger.new($stdout, level: :debug)
 
   @root = Pathname.new(__dir__)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-12-07 00:00:00.000000000 Z
+date: 2024-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: baran
@@ -534,14 +534,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 6.1.0
+        version: 6.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 6.1.0
+        version: 6.3.0
 - !ruby/object:Gem::Dependency
   name: safe_ruby
   requirement: !ruby/object:Gem::Requirement
@@ -616,8 +616,12 @@ files:
 - lib/langchain/agent/sql_query_agent.rb
 - lib/langchain/agent/sql_query_agent/sql_query_agent_answer_prompt.yaml
 - lib/langchain/agent/sql_query_agent/sql_query_agent_sql_prompt.yaml
+- lib/langchain/assistants/assistant.rb
+- lib/langchain/assistants/message.rb
+- lib/langchain/assistants/thread.rb
 - lib/langchain/chunk.rb
 - lib/langchain/chunker/base.rb
+- lib/langchain/chunker/markdown.rb
 - lib/langchain/chunker/prompts/semantic_prompt_template.yml
 - lib/langchain/chunker/recursive_text.rb
 - lib/langchain/chunker/semantic.rb
@@ -677,6 +681,7 @@ files:
 - lib/langchain/processors/html.rb
 - lib/langchain/processors/json.rb
 - lib/langchain/processors/jsonl.rb
+- lib/langchain/processors/markdown.rb
 - lib/langchain/processors/pdf.rb
 - lib/langchain/processors/text.rb
 - lib/langchain/processors/xlsx.rb
@@ -736,7 +741,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: Build LLM-backed Ruby applications with Ruby's LangChain