langchainrb 0.9.2 → 0.9.4

data/lib/langchain/processors/eml.rb

@@ -0,0 +1,65 @@
+require "mail"
+require "uri"
+
+module Langchain
+  module Processors
+    class Eml < Base
+      EXTENSIONS = [".eml"]
+      CONTENT_TYPES = ["message/rfc822"]
+
+      def initialize(*)
+        depends_on "mail"
+      end
+
+      # Parse the document and return the cleaned text
+      # @param [File] data
+      # @return [String]
+      def parse(data)
+        mail = Mail.read(data.path)
+        text_content = extract_text_content(mail)
+        clean_content(text_content)
+      end
+
+      private
+
+      # Extract text content from the email, preferring plaintext over HTML
+      def extract_text_content(mail)
+        text_content = ""
+        text_content += "From: #{mail.from}\n" \
+                        "To: #{mail.to}\n" \
+                        "Cc: #{mail.cc}\n" \
+                        "Bcc: #{mail.bcc}\n" \
+                        "Subject: #{mail.subject}\n" \
+                        "Date: #{mail.date}\n\n"
+        if mail.multipart?
+          mail.parts.each do |part|
+            if part.content_type.start_with?("text/plain")
+              text_content += part.body.decoded.force_encoding("UTF-8").strip + "\n"
+            elsif part.content_type.start_with?("multipart/alternative", "multipart/mixed")
+              text_content += extract_text_content(part) + "\n" # Recursively extract from multipart
+            elsif part.content_type.start_with?("message/rfc822")
+              # Handle embedded .eml parts as separate emails
+              embedded_mail = Mail.read_from_string(part.body.decoded)
+              text_content += "--- Begin Embedded Email ---\n"
+              text_content += extract_text_content(embedded_mail) + "\n"
+              text_content += "--- End Embedded Email ---\n"
+            end
+          end
+        elsif mail.content_type.start_with?("text/plain")
+          text_content = mail.body.decoded.force_encoding("UTF-8").strip
+        end
+        text_content
+      end
+
+      # Clean and format the extracted content
+      def clean_content(content)
+        content
+          .gsub(/\[cid:[^\]]+\]/, "") # Remove embedded image references
+          .gsub(URI::DEFAULT_PARSER.make_regexp(%w[http https])) { |match| "<#{match}>" } # Format URLs
+          .gsub(/\r\n?/, "\n") # Normalize line endings to Unix style
+          .gsub(/[\u200B-\u200D\uFEFF]/, "") # Remove zero width spaces and similar characters
+          .gsub(/<\/?[^>]+>/, "") # Remove any HTML tags that might have sneaked in
+      end
+    end
+  end
+end

data/lib/langchain/tool/base.rb

@@ -48,11 +48,9 @@ module Langchain::Tool
   class Base
     include Langchain::DependencyHelper
 
-    #
     # Returns the NAME constant of the tool
     #
     # @return [String] tool name
-    #
     def name
       self.class.const_get(:NAME)
     end
@@ -63,59 +61,37 @@ module Langchain::Tool
       }
     end
 
-    #
     # Returns the DESCRIPTION constant of the tool
     #
     # @return [String] tool description
-    #
     def description
       self.class.const_get(:DESCRIPTION)
     end
 
-    #
     # Sets the DESCRIPTION constant of the tool
     #
     # @param value [String] tool description
-    #
     def self.description(value)
       const_set(:DESCRIPTION, value.tr("\n", " ").strip)
     end
 
-    #
     # Instantiates and executes the tool and returns the answer
     #
     # @param input [String] input to the tool
     # @return [String] answer
-    #
     def self.execute(input:)
+      warn "DEPRECATED: `#{self}.execute` is deprecated, and will be removed in the next major version."
+
       new.execute(input: input)
     end
 
-    # Returns the tool as an OpenAI tool
+    # Returns the tool as a list of OpenAI formatted functions
     #
     # @return [Hash] tool as an OpenAI tool
-    def to_openai_tool
-      # TODO: This is hardcoded to def execute(input:) found in each tool, needs to be dynamic.
-      {
-        type: "function",
-        function: {
-          name: name,
-          description: description,
-          parameters: {
-            type: "object",
-            properties: {
-              input: {
-                type: "string",
-                description: "Input to the tool"
-              }
-            },
-            required: ["input"]
-          }
-        }
-      }
+    def to_openai_tools
+      method_annotations
     end
 
-    #
     # Executes the tool and returns the answer
     #
     # @param input [String] input to the tool
@@ -125,12 +101,21 @@ module Langchain::Tool
       raise NotImplementedError, "Your tool must implement the `#execute(input:)` method that returns a string"
     end
 
-    #
+    # Return tool's method annotations as JSON
+    #
+    # @return [Hash] Tool's method annotations
+    def method_annotations
+      JSON.parse(
+        File.read(
+          self.class.const_get(:ANNOTATIONS_PATH)
+        )
+      )
+    end
+
     # Validates the list of tools or raises an error
-    # @param tools [Array<Langchain::Tool>] list of tools to be used
     #
+    # @param tools [Array<Langchain::Tool>] list of tools to be used
     # @raise [ArgumentError] If any of the tools are not supported
-    #
     def self.validate_tools!(tools:)
       # Check if the tool count is equal to unique tool count
       if tools.count != tools.map(&:name).uniq.count

data/lib/langchain/tool/calculator/calculator.json

@@ -0,0 +1,19 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "calculator-execute",
+      "description": "Evaluates a pure math expression or if equation contains non-math characters (e.g.: \"12F in Celsius\") then it uses the google search calculator to evaluate the expression",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "math expression"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/{calculator.rb → calculator/calculator.rb}

@@ -6,11 +6,14 @@ module Langchain::Tool
     # A calculator tool that falls back to the Google calculator widget
     #
     # Gem requirements:
-    #   gem "eqn", "~> 1.6.5"
-    #   gem "google_search_results", "~> 2.0.0"
+    #     gem "eqn", "~> 1.6.5"
+    #     gem "google_search_results", "~> 2.0.0"
+    #
+    # Usage:
+    #     calculator = Langchain::Tool::Calculator.new
     #
-
     NAME = "calculator"
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
 
     description <<~DESC
       Useful for getting the result of a math expression.
@@ -27,8 +30,8 @@ module Langchain::Tool
       depends_on "eqn"
     end
 
-    # Evaluates a pure math expression or if equation contains non-math characters (e.g.: "12F in Celsius") then
-    # it uses the google search calculator to evaluate the expression
+    # Evaluates a pure math expression or if equation contains non-math characters (e.g.: "12F in Celsius") then it uses the google search calculator to evaluate the expression
+    #
     # @param input [String] math expression
     # @return [String] Answer
     def execute(input:)

data/lib/langchain/tool/database/database.json

@@ -0,0 +1,46 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "database-describe_tables",
+      "description": "Database Tool: Returns the schema for a list of tables",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "tables": {
+            "type": "string",
+            "description": "The tables to describe."
+          }
+        },
+        "required": ["tables"]
+      }
+    }
+  }, {
+    "type": "function",
+    "function": {
+      "name": "database-list_tables",
+      "description": "Database Tool: Returns a list of tables in the database",
+      "parameters": {
+        "type": "object",
+        "properties": {},
+        "required": []
+      }
+    }
+  }, {
+    "type": "function",
+    "function": {
+      "name": "database-execute",
+      "description": "Database Tool: Executes a SQL query and returns the results",
+      "parameters": { 
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "SQL query to be executed"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/database/database.rb

@@ -0,0 +1,105 @@
+module Langchain::Tool
+  class Database < Base
+    #
+    # Connects to a database, executes SQL queries, and outputs DB schema for Agents to use
+    #
+    # Gem requirements:
+    #     gem "sequel", "~> 5.68.0"
+    #
+    # Usage:
+    #     database = Langchain::Tool::Database.new(connection_string: "postgres://user:password@localhost:5432/db_name")
+    #
+    NAME = "database"
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
+
+    description <<~DESC
+      Useful for getting the result of a database query.
+
+      The input to this tool should be valid SQL.
+    DESC
+
+    attr_reader :db, :requested_tables, :excluded_tables
+
+    # Establish a database connection
+    #
+    # @param connection_string [String] Database connection info, e.g. 'postgres://user:password@localhost:5432/db_name'
+    # @param tables [Array<Symbol>] The tables to use. Will use all if empty.
+    # @param except_tables [Array<Symbol>] The tables to exclude. Will exclude none if empty.
+    # @return [Database] Database object
+    def initialize(connection_string:, tables: [], exclude_tables: [])
+      depends_on "sequel"
+
+      raise StandardError, "connection_string parameter cannot be blank" if connection_string.empty?
+
+      @db = Sequel.connect(connection_string)
+      @requested_tables = tables
+      @excluded_tables = exclude_tables
+    end
+
+    # Database Tool: Returns a list of tables in the database
+    def list_tables
+      db.tables
+    end
+
+    # Database Tool: Returns the schema for a list of tables
+    #
+    # @param tables [String] The tables to describe.
+    # @return [String] Database schema for the tables
+    def describe_tables(tables:)
+      schema = ""
+      tables.split(",").each do |table|
+        describe_table(table, schema)
+      end
+      schema
+    end
+
+    # Database Tool: Returns the database schema
+    #
+    # @return [String] Database schema
+    def dump_schema
+      Langchain.logger.info("Dumping schema tables and keys", for: self.class)
+      schema = ""
+      db.tables.each do |table|
+        describe_table(table, schema)
+      end
+      schema
+    end
+
+    def describe_table(table, schema)
+      primary_key_columns = []
+      primary_key_column_count = db.schema(table).count { |column| column[1][:primary_key] == true }
+
+      schema << "CREATE TABLE #{table}(\n"
+      db.schema(table).each do |column|
+        schema << "#{column[0]} #{column[1][:type]}"
+        if column[1][:primary_key] == true
+          schema << " PRIMARY KEY" if primary_key_column_count == 1
+        else
+          primary_key_columns << column[0]
+        end
+        schema << ",\n" unless column == db.schema(table).last && primary_key_column_count == 1
+      end
+      if primary_key_column_count > 1
+        schema << "PRIMARY KEY (#{primary_key_columns.join(",")})"
+      end
+      db.foreign_key_list(table).each do |fk|
+        schema << ",\n" if fk == db.foreign_key_list(table).first
+        schema << "FOREIGN KEY (#{fk[:columns][0]}) REFERENCES #{fk[:table]}(#{fk[:key][0]})"
+        schema << ",\n" unless fk == db.foreign_key_list(table).last
+      end
+      schema << ");\n"
+    end
+
+    # Database Tool: Executes a SQL query and returns the results
+    #
+    # @param input [String] SQL query to be executed
+    # @return [Array] Results from the SQL query
+    def execute(input:)
+      Langchain.logger.info("Executing \"#{input}\"", for: self.class)
+
+      db[input].to_a
+    rescue Sequel::DatabaseError => e
+      Langchain.logger.error(e.message, for: self.class)
+    end
+  end
+end

data/lib/langchain/tool/google_search/google_search.json

@@ -0,0 +1,19 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "google_search-execute",
+      "description": "Executes Google Search and returns the result",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "search query"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/{google_search.rb → google_search/google_search.rb}

@@ -5,14 +5,15 @@ module Langchain::Tool
     #
     # Wrapper around SerpApi's Google Search API
     #
-    # Gem requirements: gem "google_search_results", "~> 2.0.0"
+    # Gem requirements:
+    #     gem "google_search_results", "~> 2.0.0"
     #
     # Usage:
-    # search = Langchain::Tool::GoogleSearch.new(api_key: "YOUR_API_KEY")
-    # search.execute(input: "What is the capital of France?")
+    #     search = Langchain::Tool::GoogleSearch.new(api_key: "YOUR_API_KEY")
+    #     search.execute(input: "What is the capital of France?")
     #
-
     NAME = "google_search"
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
 
     description <<~DESC
       A wrapper around SerpApi's Google Search API.
@@ -44,12 +45,10 @@ module Langchain::Tool
       new.execute_search(input: input)
     end
 
-    #
     # Executes Google Search and returns the result
     #
     # @param input [String] search query
     # @return [String] Answer
-    #
     def execute(input:)
       Langchain.logger.info("Executing \"#{input}\"", for: self.class)
 

data/lib/langchain/tool/ruby_code_interpreter/ruby_code_interpreter.json

@@ -0,0 +1,19 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "ruby_code_interpreter-execute",
+      "description": "Executes Ruby code in a sandboxes environment.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "ruby code expression"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/{ruby_code_interpreter.rb → ruby_code_interpreter/ruby_code_interpreter.rb}

@@ -5,9 +5,15 @@ module Langchain::Tool
     #
     # A tool that execute Ruby code in a sandboxed environment.
     #
-    # Gem requirements: gem "safe_ruby", "~> 1.0.4"
+    # Gem requirements:
+    #     gem "safe_ruby", "~> 1.0.4"
+    #
+    # Usage:
+    #    interpreter = Langchain::Tool::RubyCodeInterpreter.new
     #
     NAME = "ruby_code_interpreter"
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
+
     description <<~DESC
       A Ruby code interpreter. Use this to execute ruby expressions. Input should be a valid ruby expression. If you want to see the output of the tool, make sure to return a value.
     DESC
@@ -18,6 +24,8 @@ module Langchain::Tool
       @timeout = timeout
     end
 
+    # Executes Ruby code in a sandboxes environment.
+    #
     # @param input [String] ruby code expression
     # @return [String] Answer
     def execute(input:)

data/lib/langchain/tool/weather/weather.json

@@ -0,0 +1,19 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "weather-execute",
+      "description": "Returns current weather for a city",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "comma separated city and unit (optional: imperial, metric, or standard)"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/{weather.rb → weather/weather.rb}

@@ -13,11 +13,11 @@ module Langchain::Tool
     #     api_key: https://home.openweathermap.org/api_keys
     #
     # Usage:
-    #     weather = Langchain::Tool::Weather.new(api_key: "YOUR_API_KEY")
+    #     weather = Langchain::Tool::Weather.new(api_key: ENV["OPEN_WEATHER_API_KEY"])
     #     weather.execute(input: "Boston, MA; imperial")
     #
-
     NAME = "weather"
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
 
     description <<~DESC
       Useful for getting current weather data
@@ -32,12 +32,10 @@ module Langchain::Tool
 
     attr_reader :client, :units
 
-    #
     # Initializes the Weather tool
     #
     # @param api_key [String] Open Weather API key
     # @return [Langchain::Tool::Weather] Weather tool
-    #
     def initialize(api_key:, units: "metric")
       depends_on "open-weather-ruby-client"
       require "open-weather-ruby-client"
@@ -51,6 +49,7 @@ module Langchain::Tool
     end
 
     # Returns current weather for a city
+    #
     # @param input [String] comma separated city and unit (optional: imperial, metric, or standard)
     # @return [String] Answer
     def execute(input:)

data/lib/langchain/tool/wikipedia/wikipedia.json

@@ -0,0 +1,19 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "wikipedia-execute",
+      "description": "Executes Wikipedia API search and returns the answer",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "search query"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/{wikipedia.rb → wikipedia/wikipedia.rb}

@@ -5,9 +5,16 @@ module Langchain::Tool
     #
     # Tool that adds the capability to search using the Wikipedia API
     #
-    # Gem requirements: gem "wikipedia-client", "~> 1.17.0"
+    # Gem requirements:
+    #     gem "wikipedia-client", "~> 1.17.0"
+    #
+    # Usage:
+    #     weather = Langchain::Tool::Wikipedia.new
+    #     weather.execute(input: "The Roman Empire")
     #
     NAME = "wikipedia"
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
+
     description <<~DESC
       A wrapper around Wikipedia.
 
@@ -17,11 +24,13 @@ module Langchain::Tool
       Input should be a search query.
     DESC
 
+    # Initializes the Wikipedia tool
     def initialize
       depends_on "wikipedia-client", req: "wikipedia"
     end
 
     # Executes Wikipedia API search and returns the answer
+    #
     # @param input [String] search query
     # @return [String] Answer
     def execute(input:)

data/lib/langchain/vectorsearch/base.rb

@@ -136,7 +136,7 @@ module Langchain::Vectorsearch
     # @param k [Integer] The number of results to return
     # @return [String] Response
     def similarity_search_with_hyde(query:, k: 4)
-      hyde_completion = llm.complete(prompt: generate_hyde_prompt(question: query))
+      hyde_completion = llm.complete(prompt: generate_hyde_prompt(question: query)).completion
       similarity_search(query: hyde_completion, k: k)
     end
 

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.9.2"
+  VERSION = "0.9.4"
 end

data/lib/langchain.rb

@@ -25,6 +25,13 @@ loader.inflector.inflect(
 )
 loader.collapse("#{__dir__}/langchain/llm/response")
 loader.collapse("#{__dir__}/langchain/assistants")
+
+loader.collapse("#{__dir__}/langchain/tool/calculator")
+loader.collapse("#{__dir__}/langchain/tool/database")
+loader.collapse("#{__dir__}/langchain/tool/google_search")
+loader.collapse("#{__dir__}/langchain/tool/ruby_code_interpreter")
+loader.collapse("#{__dir__}/langchain/tool/weather")
+loader.collapse("#{__dir__}/langchain/tool/wikipedia")
 loader.setup
 
 # Langchain.rb a is library for building LLM-backed Ruby applications. It is an abstraction layer that sits on top of the emerging AI-related tools that makes it easy for developers to consume and string those services together.
@@ -65,7 +72,7 @@ loader.setup
 #
 # = Logging
 #
-# LangChain.rb uses standard logging mechanisms and defaults to :debug level. Most messages are at info level, but we will add debug or warn statements as needed. To show all log messages:
+# Langchain.rb uses standard logging mechanisms and defaults to :debug level. Most messages are at info level, but we will add debug or warn statements as needed. To show all log messages:
 #
 # Langchain.logger.level = :info
 module Langchain