langchainrb 0.7.5 → 0.12.0

data/lib/langchain/output_parsers/output_fixing_parser.rb

@@ -6,13 +6,11 @@ module Langchain::OutputParsers
   class OutputFixingParser < Base
     attr_reader :llm, :parser, :prompt
 
-    #
     # Initializes a new instance of the class.
     #
     # @param llm [Langchain::LLM] The LLM used in the fixing process
     # @param parser [Langchain::OutputParsers] The parser originally used which resulted in parsing error
     # @param prompt [Langchain::Prompt::PromptTemplate]
-    #
     def initialize(llm:, parser:, prompt:)
       raise ArgumentError.new("llm must be an instance of Langchain::LLM got: #{llm.class}") unless llm.is_a?(Langchain::LLM::Base)
       raise ArgumentError.new("parser must be an instance of Langchain::OutputParsers got #{parser.class}") unless parser.is_a?(Langchain::OutputParsers::Base)
@@ -30,17 +28,14 @@ module Langchain::OutputParsers
       }
     end
 
-    #
     # calls get_format_instructions on the @parser
     #
     # @return [String] Instructions for how the output of a language model should be formatted
     # according to the @schema.
-    #
     def get_format_instructions
       parser.get_format_instructions
     end
 
-    #
     # Parse the output of an LLM call, if fails with OutputParserException
     # then call the LLM with a fix prompt in an attempt to get the correctly
     # formatted response
@@ -48,21 +43,20 @@ module Langchain::OutputParsers
     # @param completion [String] Text output from the LLM call
     #
     # @return [Object] object that is succesfully parsed by @parser.parse
-    #
     def parse(completion)
       parser.parse(completion)
     rescue OutputParserException => e
       new_completion = llm.chat(
-        prompt: prompt.format(
-          instructions: parser.get_format_instructions,
-          completion: completion,
-          error: e
-        )
-      )
+        messages: [{role: "user",
+                    content: prompt.format(
+                      instructions: parser.get_format_instructions,
+                      completion: completion,
+                      error: e
+                    )}]
+      ).completion
       parser.parse(new_completion)
     end
 
-    #
     # Creates a new instance of the class using the given JSON::Schema.
     #
     # @param llm [Langchain::LLM] The LLM used in the fixing process
@@ -70,7 +64,6 @@ module Langchain::OutputParsers
     # @param prompt [Langchain::Prompt::PromptTemplate]
     #
     # @return [Object] A new instance of the class
-    #
     def self.from_llm(llm:, parser:, prompt: nil)
       new(llm: llm, parser: parser, prompt: prompt || naive_fix_prompt)
     end

data/lib/langchain/output_parsers/structured_output_parser.rb

@@ -5,15 +5,12 @@ require "json-schema"
 
 module Langchain::OutputParsers
   # = Structured Output Parser
-  #
   class StructuredOutputParser < Base
     attr_reader :schema
 
-    #
     # Initializes a new instance of the class.
     #
     # @param schema [JSON::Schema] The json schema
-    #
     def initialize(schema:)
       @schema = validate_schema!(schema)
     end
@@ -25,24 +22,20 @@ module Langchain::OutputParsers
       }
     end
 
-    #
     # Creates a new instance of the class using the given JSON::Schema.
     #
     # @param schema [JSON::Schema] The JSON::Schema to use
     #
     # @return [Object] A new instance of the class
-    #
     def self.from_json_schema(schema)
       new(schema: schema)
     end
 
-    #
     # Returns a string containing instructions for how the output of a language model should be formatted
     # according to the @schema.
     #
     # @return [String] Instructions for how the output of a language model should be formatted
     # according to the @schema.
-    #
     def get_format_instructions
       <<~INSTRUCTIONS
         You must format your output as a JSON value that adheres to a given "JSON Schema" instance.
@@ -62,13 +55,10 @@ module Langchain::OutputParsers
       INSTRUCTIONS
     end
 
-    #
     # Parse the output of an LLM call extracting an object that abides by the @schema
     #
     # @param text [String] Text output from the LLM call
-    #
     # @return [Object] object that abides by the @schema
-    #
     def parse(text)
       json = text.include?("```") ? text.strip.split(/```(?:json)?/)[1] : text.strip
       parsed = JSON.parse(json)

data/lib/langchain/processors/csv.rb

@@ -5,15 +5,26 @@ require "csv"
 module Langchain
   module Processors
     class CSV < Base
+      class InvalidChunkMode < StandardError; end
+
       EXTENSIONS = [".csv"]
       CONTENT_TYPES = ["text/csv"]
+      CHUNK_MODE = {
+        row: "row",
+        file: "file"
+      }
 
       # Parse the document and return the text
       # @param [File] data
-      # @return [Array of Hash]
+      # @return [String]
       def parse(data)
-        ::CSV.new(data.read, col_sep: separator).map do |row|
-          row.map(&:strip)
+        case chunk_mode
+        when CHUNK_MODE[:row]
+          chunk_row(data)
+        when CHUNK_MODE[:file]
+          chunk_file(data)
+        else
+          raise InvalidChunkMode
         end
       end
 
@@ -22,6 +33,29 @@ module Langchain
       def separator
         @options[:col_sep] || ","
       end
+
+      def chunk_mode
+        if @options[:chunk_mode].to_s.empty?
+          CHUNK_MODE[:row]
+        else
+          raise InvalidChunkMode unless CHUNK_MODE.value?(@options[:chunk_mode])
+
+          @options[:chunk_mode]
+        end
+      end
+
+      def chunk_row(data)
+        ::CSV.new(data.read, col_sep: separator).map do |row|
+          row
+            .compact
+            .map(&:strip)
+            .join(separator)
+        end.join("\n\n")
+      end
+
+      def chunk_file(data)
+        data.read
+      end
     end
   end
 end

data/lib/langchain/processors/eml.rb

@@ -0,0 +1,64 @@
+require "uri"
+
+module Langchain
+  module Processors
+    class Eml < Base
+      EXTENSIONS = [".eml"]
+      CONTENT_TYPES = ["message/rfc822"]
+
+      def initialize(*)
+        depends_on "mail"
+      end
+
+      # Parse the document and return the cleaned text
+      # @param [File] data
+      # @return [String]
+      def parse(data)
+        mail = Mail.read(data.path)
+        text_content = extract_text_content(mail)
+        clean_content(text_content)
+      end
+
+      private
+
+      # Extract text content from the email, preferring plaintext over HTML
+      def extract_text_content(mail)
+        text_content = ""
+        text_content += "From: #{mail.from}\n" \
+                        "To: #{mail.to}\n" \
+                        "Cc: #{mail.cc}\n" \
+                        "Bcc: #{mail.bcc}\n" \
+                        "Subject: #{mail.subject}\n" \
+                        "Date: #{mail.date}\n\n"
+        if mail.multipart?
+          mail.parts.each do |part|
+            if part.content_type.start_with?("text/plain")
+              text_content += part.body.decoded.force_encoding("UTF-8").strip + "\n"
+            elsif part.content_type.start_with?("multipart/alternative", "multipart/mixed")
+              text_content += extract_text_content(part) + "\n" # Recursively extract from multipart
+            elsif part.content_type.start_with?("message/rfc822")
+              # Handle embedded .eml parts as separate emails
+              embedded_mail = Mail.read_from_string(part.body.decoded)
+              text_content += "--- Begin Embedded Email ---\n"
+              text_content += extract_text_content(embedded_mail) + "\n"
+              text_content += "--- End Embedded Email ---\n"
+            end
+          end
+        elsif mail.content_type.start_with?("text/plain")
+          text_content = mail.body.decoded.force_encoding("UTF-8").strip
+        end
+        text_content
+      end
+
+      # Clean and format the extracted content
+      def clean_content(content)
+        content
+          .gsub(/\[cid:[^\]]+\]/, "") # Remove embedded image references
+          .gsub(URI::DEFAULT_PARSER.make_regexp(%w[http https])) { |match| "<#{match}>" } # Format URLs
+          .gsub(/\r\n?/, "\n") # Normalize line endings to Unix style
+          .gsub(/[\u200B-\u200D\uFEFF]/, "") # Remove zero width spaces and similar characters
+          .gsub(/<\/?[^>]+>/, "") # Remove any HTML tags that might have sneaked in
+      end
+    end
+  end
+end

data/lib/langchain/processors/markdown.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Processors
+    class Markdown < Base
+      EXTENSIONS = [".markdown", ".md"]
+      CONTENT_TYPES = ["text/markdown"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [String]
+      def parse(data)
+        data.read
+      end
+    end
+  end
+end

data/lib/langchain/processors/pptx.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Processors
+    class Pptx < Base
+      EXTENSIONS = [".pptx"]
+      CONTENT_TYPES = ["application/vnd.openxmlformats-officedocument.presentationml.presentation"]
+
+      def initialize(*)
+        depends_on "power_point_pptx"
+      end
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [String]
+      def parse(data)
+        presentation = PowerPointPptx::Document.open(data)
+
+        slides = presentation.slides
+        contents = slides.map(&:content)
+        text = contents.map do |sections|
+          sections.map(&:strip).join(" ")
+        end
+
+        text.join("\n\n")
+      end
+    end
+  end
+end

data/lib/langchain/prompt/loading.rb

@@ -33,7 +33,7 @@ module Langchain::Prompt
         when ".json"
           config = JSON.parse(File.read(file_path))
         when ".yaml", ".yml"
-          config = YAML.safe_load(File.read(file_path))
+          config = YAML.safe_load_file(file_path)
         else
           raise ArgumentError, "Got unsupported file type #{file_path.extname}"
         end

data/lib/langchain/tool/base.rb

@@ -9,6 +9,7 @@ module Langchain::Tool
   #
   # - {Langchain::Tool::Calculator}: calculate the result of a math expression
   # - {Langchain::Tool::Database}: executes SQL queries
+  # - {Langchain::Tool::FileSystem}: interacts with files
   # - {Langchain::Tool::GoogleSearch}: search on Google (via SerpAPI)
   # - {Langchain::Tool::RubyCodeInterpreter}: runs ruby code
   # - {Langchain::Tool::Weather}: gets current weather data
@@ -29,8 +30,9 @@ module Langchain::Tool
   #
   # 3. Pass the tools when Agent is instantiated.
   #
-  #     agent = Langchain::Agent::ReActAgent.new(
-  #       llm: Langchain::LLM::OpenAI.new(api_key: "YOUR_API_KEY"), # or other like Cohere, Hugging Face, Google Palm or Replicate
+  #     agent = Langchain::Assistant.new(
+  #       llm: Langchain::LLM::OpenAI.new(api_key: "YOUR_API_KEY"), # or other LLM that supports function calling (coming soon)
+  #       thread: Langchain::Thread.new,
   #       tools: [
   #         Langchain::Tool::GoogleSearch.new(api_key: "YOUR_API_KEY"),
   #         Langchain::Tool::Calculator.new,
@@ -42,17 +44,16 @@ module Langchain::Tool
   #
   # 1. Create a new file in lib/langchain/tool/your_tool_name.rb
   # 2. Create a class in the file that inherits from {Langchain::Tool::Base}
-  # 3. Add `NAME=` and `DESCRIPTION=` constants in your Tool class
-  # 4. Implement `execute(input:)` method in your tool class
-  # 5. Add your tool to the {file:README.md}
+  # 3. Add `NAME=` and `ANNOTATIONS_PATH=` constants in your Tool class
+  # 4. Implement various methods in your tool class
+  # 5. Create a sidecar .json file in the same directory as your tool file annotating the methods in the Open API format
+  # 6. Add your tool to the {file:README.md}
   class Base
     include Langchain::DependencyHelper
 
-    #
     # Returns the NAME constant of the tool
     #
     # @return [String] tool name
-    #
     def name
       self.class.const_get(:NAME)
     end
@@ -63,55 +64,22 @@ module Langchain::Tool
       }
     end
 
+    # Returns the tool as a list of OpenAI formatted functions
     #
-    # Returns the DESCRIPTION constant of the tool
-    #
-    # @return [String] tool description
-    #
-    def description
-      self.class.const_get(:DESCRIPTION)
+    # @return [Hash] tool as an OpenAI tool
+    def to_openai_tools
+      method_annotations
     end
 
-    #
-    # Sets the DESCRIPTION constant of the tool
-    #
-    # @param value [String] tool description
-    #
-    def self.description(value)
-      const_set(:DESCRIPTION, value.tr("\n", " ").strip)
-    end
-
-    #
-    # Instantiates and executes the tool and returns the answer
-    #
-    # @param input [String] input to the tool
-    # @return [String] answer
-    #
-    def self.execute(input:)
-      new.execute(input: input)
-    end
-
-    #
-    # Executes the tool and returns the answer
-    #
-    # @param input [String] input to the tool
-    # @return [String] answer
-    # @raise NotImplementedError when not implemented
-    def execute(input:)
-      raise NotImplementedError, "Your tool must implement the `#execute(input:)` method that returns a string"
-    end
-
-    #
-    # Validates the list of tools or raises an error
-    # @param tools [Array<Langchain::Tool>] list of tools to be used
-    #
-    # @raise [ArgumentError] If any of the tools are not supported
-    #
-    def self.validate_tools!(tools:)
-      # Check if the tool count is equal to unique tool count
-      if tools.count != tools.map(&:name).uniq.count
-        raise ArgumentError, "Either tools are not unique or are conflicting with each other"
-      end
+    # Return tool's method annotations as JSON
+    #
+    # @return [Hash] Tool's method annotations
+    def method_annotations
+      JSON.parse(
+        File.read(
+          self.class.const_get(:ANNOTATIONS_PATH)
+        )
+      )
     end
   end
 end

data/lib/langchain/tool/calculator/calculator.json

@@ -0,0 +1,19 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "calculator-execute",
+      "description": "Evaluates a pure math expression or if equation contains non-math characters (e.g.: \"12F in Celsius\") then it uses the google search calculator to evaluate the expression",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "math expression"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/{calculator.rb → calculator/calculator.rb}

@@ -6,29 +6,21 @@ module Langchain::Tool
     # A calculator tool that falls back to the Google calculator widget
     #
     # Gem requirements:
-    #   gem "eqn", "~> 1.6.5"
-    #   gem "google_search_results", "~> 2.0.0"
+    #     gem "eqn", "~> 1.6.5"
+    #     gem "google_search_results", "~> 2.0.0"
+    #
+    # Usage:
+    #     calculator = Langchain::Tool::Calculator.new
     #
-
     NAME = "calculator"
-
-    description <<~DESC
-      Useful for getting the result of a math expression.
-
-      The input to this tool should be a valid mathematical expression that could be executed by a simple calculator.
-      Usage:
-        Action Input: 1 + 1
-        Action Input: 3 * 2 / 4
-        Action Input: 9 - 7
-        Action Input: (4.1 + 2.3) / (2.0 - 5.6) * 3
-    DESC
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
 
     def initialize
       depends_on "eqn"
     end
 
-    # Evaluates a pure math expression or if equation contains non-math characters (e.g.: "12F in Celsius") then
-    # it uses the google search calculator to evaluate the expression
+    # Evaluates a pure math expression or if equation contains non-math characters (e.g.: "12F in Celsius") then it uses the google search calculator to evaluate the expression
+    #
     # @param input [String] math expression
     # @return [String] Answer
     def execute(input:)

data/lib/langchain/tool/database/database.json

@@ -0,0 +1,46 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "database-describe_tables",
+      "description": "Database Tool: Returns the schema for a list of tables",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "tables": {
+            "type": "string",
+            "description": "The tables to describe."
+          }
+        },
+        "required": ["tables"]
+      }
+    }
+  }, {
+    "type": "function",
+    "function": {
+      "name": "database-list_tables",
+      "description": "Database Tool: Returns a list of tables in the database",
+      "parameters": {
+        "type": "object",
+        "properties": {},
+        "required": []
+      }
+    }
+  }, {
+    "type": "function",
+    "function": {
+      "name": "database-execute",
+      "description": "Database Tool: Executes a SQL query and returns the results",
+      "parameters": { 
+        "type": "object",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "SQL query to be executed"
+          }
+        },
+        "required": ["input"]
+      }
+    }
+  }
+]

data/lib/langchain/tool/database/database.rb

@@ -0,0 +1,99 @@
+module Langchain::Tool
+  class Database < Base
+    #
+    # Connects to a database, executes SQL queries, and outputs DB schema for Agents to use
+    #
+    # Gem requirements:
+    #     gem "sequel", "~> 5.68.0"
+    #
+    # Usage:
+    #     database = Langchain::Tool::Database.new(connection_string: "postgres://user:password@localhost:5432/db_name")
+    #
+    NAME = "database"
+    ANNOTATIONS_PATH = Langchain.root.join("./langchain/tool/#{NAME}/#{NAME}.json").to_path
+
+    attr_reader :db, :requested_tables, :excluded_tables
+
+    # Establish a database connection
+    #
+    # @param connection_string [String] Database connection info, e.g. 'postgres://user:password@localhost:5432/db_name'
+    # @param tables [Array<Symbol>] The tables to use. Will use all if empty.
+    # @param except_tables [Array<Symbol>] The tables to exclude. Will exclude none if empty.
+    # @return [Database] Database object
+    def initialize(connection_string:, tables: [], exclude_tables: [])
+      depends_on "sequel"
+
+      raise StandardError, "connection_string parameter cannot be blank" if connection_string.empty?
+
+      @db = Sequel.connect(connection_string)
+      @requested_tables = tables
+      @excluded_tables = exclude_tables
+    end
+
+    # Database Tool: Returns a list of tables in the database
+    def list_tables
+      db.tables
+    end
+
+    # Database Tool: Returns the schema for a list of tables
+    #
+    # @param tables [String] The tables to describe.
+    # @return [String] Database schema for the tables
+    def describe_tables(tables:)
+      schema = ""
+      tables.split(",").each do |table|
+        describe_table(table, schema)
+      end
+      schema
+    end
+
+    # Database Tool: Returns the database schema
+    #
+    # @return [String] Database schema
+    def dump_schema
+      Langchain.logger.info("Dumping schema tables and keys", for: self.class)
+      schema = ""
+      db.tables.each do |table|
+        describe_table(table, schema)
+      end
+      schema
+    end
+
+    def describe_table(table, schema)
+      primary_key_columns = []
+      primary_key_column_count = db.schema(table).count { |column| column[1][:primary_key] == true }
+
+      schema << "CREATE TABLE #{table}(\n"
+      db.schema(table).each do |column|
+        schema << "#{column[0]} #{column[1][:type]}"
+        if column[1][:primary_key] == true
+          schema << " PRIMARY KEY" if primary_key_column_count == 1
+        else
+          primary_key_columns << column[0]
+        end
+        schema << ",\n" unless column == db.schema(table).last && primary_key_column_count == 1
+      end
+      if primary_key_column_count > 1
+        schema << "PRIMARY KEY (#{primary_key_columns.join(",")})"
+      end
+      db.foreign_key_list(table).each do |fk|
+        schema << ",\n" if fk == db.foreign_key_list(table).first
+        schema << "FOREIGN KEY (#{fk[:columns][0]}) REFERENCES #{fk[:table]}(#{fk[:key][0]})"
+        schema << ",\n" unless fk == db.foreign_key_list(table).last
+      end
+      schema << ");\n"
+    end
+
+    # Database Tool: Executes a SQL query and returns the results
+    #
+    # @param input [String] SQL query to be executed
+    # @return [Array] Results from the SQL query
+    def execute(input:)
+      Langchain.logger.info("Executing \"#{input}\"", for: self.class)
+
+      db[input].to_a
+    rescue Sequel::DatabaseError => e
+      Langchain.logger.error(e.message, for: self.class)
+    end
+  end
+end

data/lib/langchain/tool/file_system/file_system.json

@@ -0,0 +1,57 @@
+[
+  {
+    "type": "function",
+    "function": {
+      "name": "file_system-list_directory",
+      "description": "File System Tool: Lists out the content of a specified directory",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "directory_path": {
+            "type": "string",
+            "description": "Directory path to list"
+          }
+        },
+        "required": ["directory_path"]
+      }
+    }
+  },
+  {
+    "type": "function",
+    "function": {
+      "name": "file_system-read_file",
+      "description": "File System Tool: Reads the contents of a file",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "file_path": {
+            "type": "string",
+            "description": "Path to the file to read from"
+          }
+        },
+        "required": ["file_path"]
+      }
+    }
+  },
+  {
+    "type": "function",
+    "function": {
+      "name": "file_system-write_to_file",
+      "description": "File System Tool: Write content to a file",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "file_path": {
+            "type": "string",
+            "description": "Path to the file to write"
+          },
+          "content": {
+            "type": "string",
+            "description": "Content to write to the file"
+          }
+        },
+        "required": ["file_path", "content"]
+      }
+    }
+  }    
+]