langchainrb 0.7.5 → 0.12.0

data/lib/langchain/tool/file_system/file_system.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Langchain::Tool
+  class FileSystem < Base
+    #
+    # A tool that wraps the Ruby file system classes.
+    #
+    # Usage:
+    #    file_system = Langchain::Tool::FileSystem.new
+    #
+    NAME = "file_system"
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
+
+    def list_directory(directory_path:)
+      Dir.entries(directory_path)
+    rescue Errno::ENOENT
+      "No such directory: #{directory_path}"
+    end
+
+    def read_file(file_path:)
+      File.read(file_path)
+    rescue Errno::ENOENT
+      "No such file: #{file_path}"
+    end
+
+    def write_to_file(file_path:, content:)
+      File.write(file_path, content)
+    rescue Errno::EACCES
+      "Permission denied: #{file_path}"
+    end
+  end
+end

data/lib/langchain/tool/google_search/google_search.json

@@ -0,0 +1,19 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "google_search-execute",
+      "description": "Executes Google Search and returns the result",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "search query"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/{google_search.rb → google_search/google_search.rb}

@@ -5,23 +5,15 @@ module Langchain::Tool
     #
     # Wrapper around SerpApi's Google Search API
     #
-    # Gem requirements: gem "google_search_results", "~> 2.0.0"
+    # Gem requirements:
+    #     gem "google_search_results", "~> 2.0.0"
     #
     # Usage:
-    # search = Langchain::Tool::GoogleSearch.new(api_key: "YOUR_API_KEY")
-    # search.execute(input: "What is the capital of France?")
+    #     search = Langchain::Tool::GoogleSearch.new(api_key: "YOUR_API_KEY")
+    #     search.execute(input: "What is the capital of France?")
     #
-
     NAME = "google_search"
-
-    description <<~DESC
-      A wrapper around SerpApi's Google Search API.
-
-      Useful for when you need to answer questions about current events.
-      Always one of the first options when you need to find information on internet.
-
-      Input should be a search query.
-    DESC
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
 
     attr_reader :api_key
 
@@ -47,12 +39,10 @@ module Langchain::Tool
       new.execute_search(input: input)
     end
 
-    #
     # Executes Google Search and returns the result
     #
     # @param input [String] search query
     # @return [String] Answer
-    #
     def execute(input:)
       Langchain.logger.info("Executing \"#{input}\"", for: self.class)
 

data/lib/langchain/tool/ruby_code_interpreter/ruby_code_interpreter.json

@@ -0,0 +1,19 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "ruby_code_interpreter-execute",
+      "description": "Executes Ruby code in a sandboxes environment.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "ruby code expression"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/{ruby_code_interpreter.rb → ruby_code_interpreter/ruby_code_interpreter.rb}

@@ -5,12 +5,14 @@ module Langchain::Tool
     #
     # A tool that execute Ruby code in a sandboxed environment.
     #
-    # Gem requirements: gem "safe_ruby", "~> 1.0.4"
+    # Gem requirements:
+    #     gem "safe_ruby", "~> 1.0.4"
+    #
+    # Usage:
+    #    interpreter = Langchain::Tool::RubyCodeInterpreter.new
     #
     NAME = "ruby_code_interpreter"
-    description <<~DESC
-      A Ruby code interpreter. Use this to execute ruby expressions. Input should be a valid ruby expression. If you want to see the output of the tool, make sure to return a value.
-    DESC
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
 
     def initialize(timeout: 30)
       depends_on "safe_ruby"
@@ -18,6 +20,8 @@ module Langchain::Tool
       @timeout = timeout
     end
 
+    # Executes Ruby code in a sandboxes environment.
+    #
     # @param input [String] ruby code expression
     # @return [String] Answer
     def execute(input:)

data/lib/langchain/tool/vectorsearch/vectorsearch.json

@@ -0,0 +1,24 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "vectorsearch-similarity_search",
+      "description": "Vectorsearch: Retrieves relevant document for the query",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "query": {
+            "type": "string",
+            "description": "Query to find similar documents for"
+          },
+          "k": {
+            "type": "integer",
+            "description": "Number of similar documents to retrieve",
+            "default": 4
+          }
+        },
+        "required": ["query"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/vectorsearch/vectorsearch.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Langchain::Tool
+  class Vectorsearch < Base
+    #
+    # A tool wraps vectorsearch classes
+    #
+    # Usage:
+    #    # Initialize the LLM that will be used to generate embeddings
+    #    ollama = Langchain::LLM::Ollama.new(url: ENV["OLLAMA_URL"]
+    #    chroma = Langchain::Vectorsearch::Chroma.new(url: ENV["CHROMA_URL"], index_name: "my_index", llm: ollama)
+    #
+    #    # This tool can now be used by the Assistant
+    #    vectorsearch_tool = Langchain::Tool::Vectorsearch.new(vectorsearch: chroma)
+    #
+    NAME = "vectorsearch"
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
+
+    attr_reader :vectorsearch
+
+    # Initializes the Vectorsearch tool
+    #
+    # @param vectorsearch [Langchain::Vectorsearch::Base] Vectorsearch instance to use
+    def initialize(vectorsearch:)
+      @vectorsearch = vectorsearch
+    end
+
+    # Executes the vector search and returns the results
+    #
+    # @param query [String] The query to search for
+    # @param k [Integer] The number of results to return
+    def similarity_search(query:, k: 4)
+      vectorsearch.similarity_search(query:, k: 4)
+    end
+  end
+end

data/lib/langchain/tool/weather/weather.json

@@ -0,0 +1,19 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "weather-execute",
+      "description": "Returns current weather for a city",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "comma separated city and unit (optional: imperial, metric, or standard)"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/{weather.rb → weather/weather.rb}

@@ -13,31 +13,18 @@ module Langchain::Tool
     #     api_key: https://home.openweathermap.org/api_keys
     #
     # Usage:
-    #     weather = Langchain::Tool::Weather.new(api_key: "YOUR_API_KEY")
+    #     weather = Langchain::Tool::Weather.new(api_key: ENV["OPEN_WEATHER_API_KEY"])
     #     weather.execute(input: "Boston, MA; imperial")
     #
-
     NAME = "weather"
-
-    description <<~DESC
-      Useful for getting current weather data
-
-      The input to this tool should be a city name followed by the units (imperial, metric, or standard)
-      Usage:
-        Action Input: St Louis, Missouri; metric
-        Action Input: Boston, Massachusetts; imperial
-        Action Input: Dubai, AE; imperial
-        Action Input: Kiev, Ukraine; metric
-    DESC
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
 
     attr_reader :client, :units
 
-    #
     # Initializes the Weather tool
     #
     # @param api_key [String] Open Weather API key
     # @return [Langchain::Tool::Weather] Weather tool
-    #
     def initialize(api_key:, units: "metric")
       depends_on "open-weather-ruby-client"
       require "open-weather-ruby-client"
@@ -51,6 +38,7 @@ module Langchain::Tool
     end
 
     # Returns current weather for a city
+    #
     # @param input [String] comma separated city and unit (optional: imperial, metric, or standard)
     # @return [String] Answer
     def execute(input:)

data/lib/langchain/tool/wikipedia/wikipedia.json

@@ -0,0 +1,19 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "wikipedia-execute",
+      "description": "Executes Wikipedia API search and returns the answer",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "search query"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/{wikipedia.rb → wikipedia/wikipedia.rb}

@@ -5,23 +5,23 @@ module Langchain::Tool
     #
     # Tool that adds the capability to search using the Wikipedia API
     #
-    # Gem requirements: gem "wikipedia-client", "~> 1.17.0"
+    # Gem requirements:
+    #     gem "wikipedia-client", "~> 1.17.0"
+    #
+    # Usage:
+    #     weather = Langchain::Tool::Wikipedia.new
+    #     weather.execute(input: "The Roman Empire")
     #
     NAME = "wikipedia"
-    description <<~DESC
-      A wrapper around Wikipedia.
-
-      Useful for when you need to answer general questions about
-      people, places, companies, facts, historical events, or other subjects.
-
-      Input should be a search query.
-    DESC
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
 
+    # Initializes the Wikipedia tool
     def initialize
       depends_on "wikipedia-client", req: "wikipedia"
     end
 
     # Executes Wikipedia API search and returns the answer
+    #
     # @param input [String] search query
     # @return [String] Answer
     def execute(input:)

data/lib/langchain/utils/token_length/ai21_validator.rb

@@ -22,8 +22,8 @@ module Langchain
         # @param model_name [String] The model name to validate against
         # @return [Integer] The token length of the text
         #
-        def self.token_length(text, model_name, client)
-          res = client.tokenize(text)
+        def self.token_length(text, model_name, options = {})
+          res = options[:llm].tokenize(text)
           res.dig(:tokens).length
         end
 
@@ -31,6 +31,10 @@ module Langchain
           TOKEN_LIMITS[model_name]
         end
         singleton_class.alias_method :completion_token_limit, :token_limit
+
+        def self.token_length_from_messages(messages, model_name, options)
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/base_validator.rb

@@ -14,7 +14,7 @@ module Langchain
       class BaseValidator
         def self.validate_max_tokens!(content, model_name, options = {})
           text_token_length = if content.is_a?(Array)
-            content.sum { |item| token_length(item.to_json, model_name, options) }
+            token_length_from_messages(content, model_name, options)
           else
             token_length(content, model_name, options)
           end

data/lib/langchain/utils/token_length/cohere_validator.rb

@@ -30,8 +30,8 @@ module Langchain
         # @param model_name [String] The model name to validate against
         # @return [Integer] The token length of the text
         #
-        def self.token_length(text, model_name, client)
-          res = client.tokenize(text: text)
+        def self.token_length(text, model_name, options = {})
+          res = options[:llm].tokenize(text: text)
           res["tokens"].length
         end
 
@@ -39,6 +39,10 @@ module Langchain
           TOKEN_LIMITS[model_name]
         end
         singleton_class.alias_method :completion_token_limit, :token_limit
+
+        def self.token_length_from_messages(messages, model_name, options)
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/google_palm_validator.rb

@@ -35,7 +35,7 @@ module Langchain
         # @option options [Langchain::LLM:GooglePalm] :llm The Langchain::LLM:GooglePalm instance
         # @return [Integer] The token length of the text
         #
-        def self.token_length(text, model_name = "chat-bison-001", options)
+        def self.token_length(text, model_name = "chat-bison-001", options = {})
           response = options[:llm].client.count_message_tokens(model: model_name, prompt: text)
 
           raise Langchain::LLM::ApiError.new(response["error"]["message"]) unless response["error"].nil?
@@ -43,6 +43,10 @@ module Langchain
           response.dig("tokenCount")
         end
 
+        def self.token_length_from_messages(messages, model_name, options = {})
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
+
         def self.token_limit(model_name)
           TOKEN_LIMITS.dig(model_name, "input_token_limit")
         end

data/lib/langchain/utils/token_length/openai_validator.rb

@@ -15,17 +15,23 @@ module Langchain
           # Source:
           # https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
           "gpt-4-1106-preview" => 4096,
-          "gpt-4-vision-preview" => 4096
+          "gpt-4-vision-preview" => 4096,
+          "gpt-3.5-turbo-1106" => 4096
         }
 
+        # NOTE: The gpt-4-turbo-preview is an alias that will always point to the latest GPT 4 Turbo preview
+        #   the future previews may have a different token limit!
         TOKEN_LIMITS = {
           # Source:
           # https://platform.openai.com/docs/api-reference/embeddings
           # https://platform.openai.com/docs/models/gpt-4
+          "text-embedding-3-large" => 8191,
+          "text-embedding-3-small" => 8191,
           "text-embedding-ada-002" => 8191,
           "gpt-3.5-turbo" => 4096,
           "gpt-3.5-turbo-0301" => 4096,
           "gpt-3.5-turbo-0613" => 4096,
+          "gpt-3.5-turbo-1106" => 16385,
           "gpt-3.5-turbo-16k" => 16384,
           "gpt-3.5-turbo-16k-0613" => 16384,
           "text-davinci-003" => 4097,
@@ -38,6 +44,8 @@ module Langchain
           "gpt-4-32k-0314" => 32768,
           "gpt-4-32k-0613" => 32768,
           "gpt-4-1106-preview" => 128000,
+          "gpt-4-turbo-preview" => 128000,
+          "gpt-4-0125-preview" => 128000,
           "gpt-4-vision-preview" => 128000,
           "text-curie-001" => 2049,
           "text-babbage-001" => 2049,
@@ -56,6 +64,11 @@ module Langchain
         # @return [Integer] The token length of the text
         #
         def self.token_length(text, model_name, options = {})
+          # tiktoken-ruby doesn't support text-embedding-3-large or text-embedding-3-small yet
+          if ["text-embedding-3-large", "text-embedding-3-small"].include?(model_name)
+            model_name = "text-embedding-ada-002"
+          end
+
           encoder = Tiktoken.encoding_for_model(model_name)
           encoder.encode(text).length
         end
@@ -73,6 +86,47 @@ module Langchain
           max_tokens = super(content, model_name, options)
           [options[:max_tokens], max_tokens].reject(&:nil?).min
         end
+
+        # Copied from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+        # Return the number of tokens used by a list of messages
+        #
+        # @param messages [Array<Hash>] The messages to calculate the token length for
+        # @param model [String] The model name to validate against
+        # @return [Integer] The token length of the messages
+        #
+        def self.token_length_from_messages(messages, model_name, options = {})
+          encoding = Tiktoken.encoding_for_model(model_name)
+
+          if ["gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613", "gpt-4-0314", "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613"].include?(model_name)
+            tokens_per_message = 3
+            tokens_per_name = 1
+          elsif model_name == "gpt-3.5-turbo-0301"
+            tokens_per_message = 4  # every message follows {role/name}\n{content}\n
+            tokens_per_name = -1  # if there's a name, the role is omitted
+          elsif model_name.include?("gpt-3.5-turbo")
+            # puts "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613."
+            return token_length_from_messages(messages, "gpt-3.5-turbo-0613", options)
+          elsif model_name.include?("gpt-4")
+            # puts "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613."
+            return token_length_from_messages(messages, "gpt-4-0613", options)
+          else
+            raise NotImplementedError.new(
+              "token_length_from_messages() is not implemented for model #{model_name}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."
+            )
+          end
+
+          num_tokens = 0
+          messages.each do |message|
+            num_tokens += tokens_per_message
+            message.each do |key, value|
+              num_tokens += encoding.encode(value).length
+              num_tokens += tokens_per_name if ["name", :name].include?(key)
+            end
+          end
+
+          num_tokens += 3  # every reply is primed with assistant
+          num_tokens
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/token_limit_exceeded.rb

@@ -7,7 +7,7 @@ module Langchain
         attr_reader :token_overflow
 
         def initialize(message = "", token_overflow = 0)
-          super message
+          super(message)
 
           @token_overflow = token_overflow
         end

data/lib/langchain/vectorsearch/base.rb

@@ -7,6 +7,7 @@ module Langchain::Vectorsearch
   # == Available vector databases
   #
   # - {Langchain::Vectorsearch::Chroma}
+  # - {Langchain::Vectorsearch::Epsilla}
   # - {Langchain::Vectorsearch::Elasticsearch}
   # - {Langchain::Vectorsearch::Hnswlib}
   # - {Langchain::Vectorsearch::Milvus}
@@ -29,10 +30,11 @@ module Langchain::Vectorsearch
   #     )
   #
   #     # You can instantiate other supported vector databases the same way:
+  #     epsilla  = Langchain::Vectorsearch::Epsilla.new(...)
   #     milvus   = Langchain::Vectorsearch::Milvus.new(...)
   #     qdrant   = Langchain::Vectorsearch::Qdrant.new(...)
   #     pinecone = Langchain::Vectorsearch::Pinecone.new(...)
-  #     chrome   = Langchain::Vectorsearch::Chroma.new(...)
+  #     chroma   = Langchain::Vectorsearch::Chroma.new(...)
   #     pgvector = Langchain::Vectorsearch::Pgvector.new(...)
   #
   # == Schema Creation
@@ -122,6 +124,11 @@ module Langchain::Vectorsearch
       raise NotImplementedError, "#{self.class.name} does not support updating texts"
     end
 
+    # Method supported by Vectorsearch DB to delete a list of texts from the index
+    def remove_texts(...)
+      raise NotImplementedError, "#{self.class.name} does not support deleting texts"
+    end
+
     # Method supported by Vectorsearch DB to search for similar texts in the index
     def similarity_search(...)
       raise NotImplementedError, "#{self.class.name} does not support similarity search"
@@ -134,7 +141,7 @@ module Langchain::Vectorsearch
     # @param k [Integer] The number of results to return
     # @return [String] Response
     def similarity_search_with_hyde(query:, k: 4)
-      hyde_completion = llm.complete(prompt: generate_hyde_prompt(question: query))
+      hyde_completion = llm.complete(prompt: generate_hyde_prompt(question: query)).completion
       similarity_search(query: hyde_completion, k: k)
     end
 
@@ -173,13 +180,13 @@ module Langchain::Vectorsearch
       prompt_template.format(question: question, context: context)
     end
 
-    def add_data(paths:)
+    def add_data(paths:, options: {}, chunker: Langchain::Chunker::Text)
       raise ArgumentError, "Paths must be provided" if Array(paths).empty?
 
       texts = Array(paths)
         .flatten
         .map do |path|
-          data = Langchain::Loader.new(path)&.load&.chunks
+          data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
           data.map { |chunk| chunk.text }
         end
 

data/lib/langchain/vectorsearch/chroma.rb

@@ -60,6 +60,13 @@ module Langchain::Vectorsearch
       collection.update(embeddings)
     end
 
+    # Remove a list of texts from the index
+    # @param ids [Array<String>] The list of ids to remove
+    # @return [Hash] The response from the server
+    def remove_texts(ids:)
+      collection.delete(ids)
+    end
+
     # Create the collection with the default schema
     # @return [::Chroma::Resources::Collection] Created collection
     def create_default_schema
@@ -126,7 +133,9 @@ module Langchain::Vectorsearch
 
       prompt = generate_rag_prompt(question: question, context: context)
 
-      response = llm.chat(prompt: prompt, &block)
+      messages = [{role: "user", content: prompt}]
+      response = llm.chat(messages: messages, &block)
+
       response.context = context
       response
     end