langchainrb 0.4.1 → 0.5.0

data/lib/langchain/llm/openai.rb

@@ -2,6 +2,15 @@
 
 module Langchain::LLM
   class OpenAI < Base
+    #
+    # Wrapper around OpenAI APIs.
+    #
+    # Gem requirements: gem "ruby-openai", "~> 4.0.0"
+    #
+    # Usage:
+    # openai = Langchain::LLM::OpenAI.new(api_key:, llm_options: {})
+    #
+
     DEFAULTS = {
       temperature: 0.0,
       completion_model_name: "text-davinci-003",
@@ -10,12 +19,11 @@ module Langchain::LLM
       dimension: 1536
     }.freeze
 
-    def initialize(api_key:)
+    def initialize(api_key:, llm_options: {})
       depends_on "ruby-openai"
       require "openai"
 
-      # TODO: Add support to pass `organization_id:`
-      @client = ::OpenAI::Client.new(access_token: api_key)
+      @client = ::OpenAI::Client.new(access_token: api_key, **llm_options)
     end
 
     #
@@ -24,17 +32,12 @@ module Langchain::LLM
     # @param text [String] The text to generate an embedding for
     # @return [Array] The embedding
     #
-    def embed(text:)
-      model = DEFAULTS[:embeddings_model_name]
+    def embed(text:, **params)
+      parameters = {model: DEFAULTS[:embeddings_model_name], input: text}
 
-      Langchain::Utils::TokenLengthValidator.validate!(text, model)
+      Langchain::Utils::TokenLengthValidator.validate!(text, parameters[:model])
 
-      response = client.embeddings(
-        parameters: {
-          model: model,
-          input: text
-        }
-      )
+      response = client.embeddings(parameters: parameters.merge(params))
       response.dig("data").first.dig("embedding")
     end
 
@@ -45,23 +48,13 @@ module Langchain::LLM
     # @return [String] The completion
     #
     def complete(prompt:, **params)
-      model = DEFAULTS[:completion_model_name]
-
-      Langchain::Utils::TokenLengthValidator.validate!(prompt, model)
-
-      default_params = {
-        model: model,
-        temperature: DEFAULTS[:temperature],
-        prompt: prompt
-      }
+      parameters = compose_parameters DEFAULTS[:completion_model_name], params
 
-      if params[:stop_sequences]
-        default_params[:stop] = params.delete(:stop_sequences)
-      end
+      Langchain::Utils::TokenLengthValidator.validate!(prompt, parameters[:model])
 
-      default_params.merge!(params)
+      parameters[:prompt] = prompt
 
-      response = client.completions(parameters: default_params)
+      response = client.completions(parameters: parameters)
       response.dig("choices", 0, "text")
     end
 
@@ -72,24 +65,13 @@ module Langchain::LLM
     # @return [String] The chat completion
     #
     def chat(prompt:, **params)
-      model = DEFAULTS[:chat_completion_model_name]
+      parameters = compose_parameters DEFAULTS[:chat_completion_model_name], params
 
-      Langchain::Utils::TokenLengthValidator.validate!(prompt, model)
+      Langchain::Utils::TokenLengthValidator.validate!(prompt, parameters[:model])
 
-      default_params = {
-        model: model,
-        temperature: DEFAULTS[:temperature],
-        # TODO: Figure out how to introduce persisted conversations
-        messages: [{role: "user", content: prompt}]
-      }
-
-      if params[:stop_sequences]
-        default_params[:stop] = params.delete(:stop_sequences)
-      end
+      parameters[:messages] = [{role: "user", content: prompt}]
 
-      default_params.merge!(params)
-
-      response = client.chat(parameters: default_params)
+      response = client.chat(parameters: parameters)
       response.dig("choices", 0, "message", "content")
     end
 
@@ -112,5 +94,15 @@ module Langchain::LLM
         max_tokens: 2048
       )
     end
+
+    private
+
+    def compose_parameters(model, params)
+      default_params = {model: model, temperature: DEFAULTS[:temperature]}
+
+      default_params[:stop] = params.delete(:stop_sequences) if params[:stop_sequences]
+
+      default_params.merge(params)
+    end
   end
 end

data/lib/langchain/llm/replicate.rb

@@ -2,7 +2,11 @@
 
 module Langchain::LLM
   class Replicate < Base
+    #
     # Wrapper around Replicate.com LLM provider
+    #
+    # Gem requirements: gem "replicate-ruby", "~> 0.2.2"
+    #
     # Use it directly:
     # replicate = LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
     #
@@ -10,8 +14,7 @@ module Langchain::LLM
     # chroma = Vectorsearch::Chroma.new(
     #   url: ENV["CHROMA_URL"],
     #   index_name: "...",
-    #   llm: :replicate,
-    #   llm_api_key: ENV["REPLICATE_API_KEY"],
+    #   llm: Langchain::LLM::Replicate(api_key: ENV["REPLICATE_API_KEY"])
     # )
 
     DEFAULTS = {

data/lib/langchain/processors/base.rb

@@ -3,6 +3,8 @@
 module Langchain
   module Processors
     class Base
+      include Langchain::DependencyHelper
+
       EXTENSIONS = []
       CONTENT_TYPES = []
 

data/lib/langchain/processors/xlsx.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Processors
+    class Xlsx < Base
+      EXTENSIONS = [".xlsx", ".xlsm"].freeze
+      CONTENT_TYPES = ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"].freeze
+
+      def initialize(*)
+        depends_on "roo"
+        require "roo"
+      end
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [Array<Array<String>>] Array of rows, each row is an array of cells
+      def parse(data)
+        xlsx_file = Roo::Spreadsheet.open(data)
+        xlsx_file.each_with_pagename.flat_map do |_, sheet|
+          sheet.map do |row|
+            row.map { |i| i.to_s.strip }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/langchain/prompt/base.rb

@@ -2,6 +2,7 @@
 
 require "strscan"
 require "json"
+require "yaml"
 
 module Langchain::Prompt
   class Base
@@ -52,10 +53,13 @@ module Langchain::Prompt
       directory_path = save_path.dirname
       FileUtils.mkdir_p(directory_path) unless directory_path.directory?
 
-      if save_path.extname == ".json"
+      case save_path.extname
+      when ".json"
         File.write(file_path, to_h.to_json)
+      when ".yaml", ".yml"
+        File.write(file_path, to_h.to_yaml)
       else
-        raise ArgumentError, "#{file_path} must be json"
+        raise ArgumentError, "#{file_path} must be json or yaml file"
       end
     end
 
@@ -64,9 +68,9 @@ module Langchain::Prompt
     #
     # This method takes a template string and returns an array of input variable names
     # contained within the template. Input variables are defined as text enclosed in
-    # curly braces (e.g. "{variable_name}").
+    # curly braces (e.g. <code>\{variable_name\}</code>).
     #
-    # Content within two consecutive curly braces (e.g. "{{ignore_me}}) are ignored.
+    # Content within two consecutive curly braces (e.g. <code>\{\{ignore_me}}</code>) are ignored.
     #
     # @param template [String] The template string to extract variables from.
     #

data/lib/langchain/prompt/loading.rb

@@ -2,6 +2,8 @@
 
 require "strscan"
 require "pathname"
+require "json"
+require "yaml"
 
 module Langchain::Prompt
   TYPE_TO_LOADER = {
@@ -22,8 +24,11 @@ module Langchain::Prompt
     def load_from_path(file_path:)
       file_path = file_path.is_a?(String) ? Pathname.new(file_path) : file_path
 
-      if file_path.extname == ".json"
+      case file_path.extname
+      when ".json"
         config = JSON.parse(File.read(file_path))
+      when ".yaml", ".yml"
+        config = YAML.safe_load(File.read(file_path))
       else
         raise ArgumentError, "Got unsupported file type #{file_path.extname}"
       end

data/lib/langchain/prompt/prompt_template.rb

@@ -20,7 +20,7 @@ module Langchain::Prompt
     end
 
     #
-    # Format the prompt with the inputs. Double {{}} replaced with single {} to adhere to f-string spec.
+    # Format the prompt with the inputs. Double <code>{{}}</code> replaced with single <code>{}</code> to adhere to f-string spec.
     #
     # @param kwargs [Hash] Any arguments to be passed to the prompt template.
     # @return [String] A formatted string.

data/lib/langchain/tool/base.rb

@@ -2,6 +2,8 @@
 
 module Langchain::Tool
   class Base
+    include Langchain::DependencyHelper
+
     # How to add additional Tools?
     # 1. Create a new file in lib/tool/your_tool_name.rb
     # 2. Add your tool to the TOOLS hash below
@@ -12,7 +14,8 @@ module Langchain::Tool
     TOOLS = {
       "calculator" => "Langchain::Tool::Calculator",
       "search" => "Langchain::Tool::SerpApi",
-      "wikipedia" => "Langchain::Tool::Wikipedia"
+      "wikipedia" => "Langchain::Tool::Wikipedia",
+      "database" => "Langchain::Tool::Database"
     }
 
     def self.description(value)

data/lib/langchain/tool/calculator.rb

@@ -2,6 +2,15 @@
 
 module Langchain::Tool
   class Calculator < Base
+    #
+    # A calculator tool that falls back to the Google calculator widget
+    #
+    # Gem requirements:
+    #   gem "eqn", "~> 1.6.5"
+    #   gem "google_search_results", "~> 2.0.0"
+    # ENV requirements: ENV["SERPAPI_API_KEY"]
+    #
+
     description <<~DESC
       Useful for getting the result of a math expression.
 

data/lib/langchain/tool/database.rb

@@ -0,0 +1,45 @@
+module Langchain::Tool
+  class Database < Base
+    #
+    # Connects to a database, executes SQL queries, and outputs DB schema for Agents to use
+    #
+    # Gem requirements: gem "sequel", "~> 5.68.0"
+    #
+
+    description <<~DESC
+      Useful for getting the result of a database query.
+
+      The input to this tool should be valid SQL.
+    DESC
+
+    # Establish a database connection
+    # @param db_connection_string [String] Database connection info, e.g. 'postgres://user:password@localhost:5432/db_name'
+    def initialize(db_connection_string)
+      depends_on "sequel"
+      require "sequel"
+      require "sequel/extensions/schema_dumper"
+
+      raise StandardError, "db_connection_string parameter cannot be blank" if db_connection_string.empty?
+
+      @db = Sequel.connect(db_connection_string)
+      @db.extension :schema_dumper
+    end
+
+    def schema
+      Langchain.logger.info("[#{self.class.name}]".light_blue + ": Dumping schema")
+      @db.dump_schema_migration(same_db: true, indexes: false) unless @db.adapter_scheme == :mock
+    end
+
+    # Evaluates a sql expression
+    # @param input [String] sql expression
+    # @return [Array] results
+    def execute(input:)
+      Langchain.logger.info("[#{self.class.name}]".light_blue + ": Executing \"#{input}\"")
+      begin
+        @db[input].to_a
+      rescue Sequel::DatabaseError => e
+        Langchain.logger.error("[#{self.class.name}]".light_red + ": #{e.message}")
+      end
+    end
+  end
+end

data/lib/langchain/tool/ruby_code_interpreter.rb

@@ -2,6 +2,12 @@
 
 module Langchain::Tool
   class RubyCodeInterpreter < Base
+    #
+    # A tool that execute Ruby code in a sandboxed environment.
+    #
+    # Gem requirements: gem "safe_ruby", "~> 1.0.4"
+    #
+
     description <<~DESC
       A Ruby code interpreter. Use this to execute ruby expressions. Input should be a valid ruby expression. If you want to see the output of the tool, make sure to return a value.
     DESC

data/lib/langchain/tool/serp_api.rb

@@ -2,8 +2,12 @@
 
 module Langchain::Tool
   class SerpApi < Base
+    #
     # Wrapper around SerpAPI
-    # Set ENV["SERPAPI_API_KEY"] to use it
+    #
+    # Gem requirements: gem "google_search_results", "~> 2.0.0"
+    # ENV requirements: ENV["SERPAPI_API_KEY"] # https://serpapi.com/manage-api-key)
+    #
 
     description <<~DESC
       A wrapper around Google Search.

data/lib/langchain/tool/wikipedia.rb

@@ -2,7 +2,11 @@
 
 module Langchain::Tool
   class Wikipedia < Base
+    #
     # Tool that adds the capability to search using the Wikipedia API
+    #
+    # Gem requirements: gem "wikipedia-client", "~> 1.17.0"
+    #
 
     description <<~DESC
       A wrapper around Wikipedia.

data/lib/langchain/vectorsearch/base.rb

@@ -4,21 +4,16 @@ require "forwardable"
 
 module Langchain::Vectorsearch
   class Base
+    include Langchain::DependencyHelper
     extend Forwardable
 
-    attr_reader :client, :index_name, :llm, :llm_api_key, :llm_client
+    attr_reader :client, :index_name, :llm
 
     DEFAULT_METRIC = "cosine"
 
-    # @param llm [Symbol] The LLM to use
-    # @param llm_api_key [String] The API key for the LLM
-    def initialize(llm:, llm_api_key:)
-      Langchain::LLM::Base.validate_llm!(llm: llm)
-
+    # @param llm [Object] The LLM client to use
+    def initialize(llm:)
       @llm = llm
-      @llm_api_key = llm_api_key
-
-      @llm_client = Langchain::LLM.const_get(Langchain::LLM::Base::LLMS.fetch(llm)).new(api_key: llm_api_key)
     end
 
     # Method supported by Vectorsearch DB to create a default schema
@@ -47,7 +42,7 @@ module Langchain::Vectorsearch
       raise NotImplementedError, "#{self.class.name} does not support asking questions"
     end
 
-    def_delegators :llm_client,
+    def_delegators :llm,
       :default_dimension
 
     def generate_prompt(question:, context:)
@@ -68,11 +63,10 @@ module Langchain::Vectorsearch
       prompt_template.format(question: question)
     end
 
-    def add_data(path: nil, paths: nil)
-      raise ArgumentError, "Either path or paths must be provided" if path.nil? && paths.nil?
-      raise ArgumentError, "Either path or paths must be provided, not both" if !path.nil? && !paths.nil?
+    def add_data(paths:)
+      raise ArgumentError, "Paths must be provided" if paths.to_a.empty?
 
-      texts = Array(path || paths)
+      texts = Array(paths)
         .flatten
         .map { |path| Langchain::Loader.new(path)&.load&.value }
         .compact

data/lib/langchain/vectorsearch/chroma.rb

@@ -2,13 +2,21 @@
 
 module Langchain::Vectorsearch
   class Chroma < Base
+    #
+    # Wrapper around Chroma DB
+    #
+    # Gem requirements: gem "chroma-db", "~> 0.3.0"
+    #
+    # Usage:
+    # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, llm_api_key:, api_key: nil)
+    #
+
     # Initialize the Chroma client
     # @param url [String] The URL of the Qdrant server
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
-    # @param llm [Symbol] The LLM to use
-    # @param llm_api_key [String] The API key for the LLM
-    def initialize(url:, index_name:, llm:, llm_api_key:, api_key: nil)
+    # @param llm [Object] The LLM client to use
+    def initialize(url:, index_name:, llm:, api_key: nil)
       depends_on "chroma-db"
       require "chroma-db"
 
@@ -18,7 +26,7 @@ module Langchain::Vectorsearch
 
       @index_name = index_name
 
-      super(llm: llm, llm_api_key: llm_api_key)
+      super(llm: llm)
     end
 
     # Add a list of texts to the index
@@ -29,7 +37,7 @@ module Langchain::Vectorsearch
         ::Chroma::Resources::Embedding.new(
           # TODO: Add support for passing your own IDs
           id: SecureRandom.uuid,
-          embedding: llm_client.embed(text: text),
+          embedding: llm.embed(text: text),
           # TODO: Add support for passing metadata
           metadata: [], # metadatas[index],
           document: text # Do we actually need to store the whole original document?
@@ -54,7 +62,7 @@ module Langchain::Vectorsearch
       query:,
       k: 4
     )
-      embedding = llm_client.embed(text: query)
+      embedding = llm.embed(text: query)
 
       similarity_search_by_vector(
         embedding: embedding,
@@ -92,7 +100,7 @@ module Langchain::Vectorsearch
 
       prompt = generate_prompt(question: question, context: context)
 
-      llm_client.chat(prompt: prompt)
+      llm.chat(prompt: prompt)
     end
 
     private

data/lib/langchain/vectorsearch/milvus.rb

@@ -2,14 +2,23 @@
 
 module Langchain::Vectorsearch
   class Milvus < Base
-    def initialize(url:, index_name:, llm:, llm_api_key:, api_key: nil)
+    #
+    # Wrapper around Milvus REST APIs.
+    #
+    # Gem requirements: gem "milvus", "~> 0.9.0"
+    #
+    # Usage:
+    # milvus = Langchain::Vectorsearch::Milvus.new(url:, index_name:, llm:, llm_api_key:)
+    #
+
+    def initialize(url:, index_name:, llm:, api_key: nil)
       depends_on "milvus"
       require "milvus"
 
       @client = ::Milvus::Client.new(url: url)
       @index_name = index_name
 
-      super(llm: llm, llm_api_key: llm_api_key)
+      super(llm: llm)
     end
 
     def add_texts(texts:)
@@ -24,7 +33,7 @@ module Langchain::Vectorsearch
           }, {
             field_name: "vectors",
             type: ::Milvus::DATA_TYPES["binary_vector"],
-            field: Array(texts).map { |text| llm_client.embed(text: text) }
+            field: Array(texts).map { |text| llm.embed(text: text) }
           }
         ]
       )
@@ -69,7 +78,7 @@ module Langchain::Vectorsearch
     end
 
     def similarity_search(query:, k: 4)
-      embedding = llm_client.embed(text: query)
+      embedding = llm.embed(text: query)
 
       similarity_search_by_vector(
         embedding: embedding,

data/lib/langchain/vectorsearch/pgvector.rb

@@ -1,8 +1,16 @@
 # frozen_string_literal: true
 
 module Langchain::Vectorsearch
-  # The PostgreSQL vector search adapter
   class Pgvector < Base
+    #
+    # The PostgreSQL vector search adapter
+    #
+    # Gem requirements: gem "pgvector", "~> 0.2"
+    #
+    # Usage:
+    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:, llm_api_key:)
+    #
+
     # The operators supported by the PostgreSQL vector search adapter
     OPERATORS = {
       "cosine_distance" => "<=>",
@@ -14,10 +22,9 @@ module Langchain::Vectorsearch
 
     # @param url [String] The URL of the PostgreSQL database
     # @param index_name [String] The name of the table to use for the index
-    # @param llm [String] The URL of the Language Layer API
-    # @param llm_api_key [String] The API key for the Language Layer API
+    # @param llm [Object] The LLM client to use
     # @param api_key [String] The API key for the Vectorsearch DB (not used for PostgreSQL)
-    def initialize(url:, index_name:, llm:, llm_api_key:, api_key: nil)
+    def initialize(url:, index_name:, llm:, api_key: nil)
       require "pg"
       require "pgvector"
 
@@ -30,7 +37,7 @@ module Langchain::Vectorsearch
       @quoted_table_name = @client.quote_ident(index_name)
       @operator = OPERATORS[DEFAULT_OPERATOR]
 
-      super(llm: llm, llm_api_key: llm_api_key)
+      super(llm: llm)
     end
 
     # Add a list of texts to the index
@@ -38,7 +45,7 @@ module Langchain::Vectorsearch
     # @return [PG::Result] The response from the database
     def add_texts(texts:)
       data = texts.flat_map do |text|
-        [text, llm_client.embed(text: text)]
+        [text, llm.embed(text: text)]
       end
       values = texts.length.times.map { |i| "($#{2 * i + 1}, $#{2 * i + 2})" }.join(",")
       client.exec_params(
@@ -67,7 +74,7 @@ module Langchain::Vectorsearch
     # @param k [Integer] The number of top results to return
     # @return [Array<Hash>] The results of the search
     def similarity_search(query:, k: 4)
-      embedding = llm_client.embed(text: query)
+      embedding = llm.embed(text: query)
 
       similarity_search_by_vector(
         embedding: embedding,
@@ -105,7 +112,7 @@ module Langchain::Vectorsearch
 
       prompt = generate_prompt(question: question, context: context)
 
-      llm_client.chat(prompt: prompt)
+      llm.chat(prompt: prompt)
     end
   end
 end

data/lib/langchain/vectorsearch/pinecone.rb

@@ -2,13 +2,21 @@
 
 module Langchain::Vectorsearch
   class Pinecone < Base
+    #
+    # Wrapper around Pinecone API.
+    #
+    # Gem requirements: gem "pinecone", "~> 0.1.6"
+    #
+    # Usage:
+    # pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:, llm_api_key:)
+    #
+
     # Initialize the Pinecone client
     # @param environment [String] The environment to use
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
-    # @param llm [Symbol] The LLM to use
-    # @param llm_api_key [String] The API key for the LLM
-    def initialize(environment:, api_key:, index_name:, llm:, llm_api_key:)
+    # @param llm [Object] The LLM client to use
+    def initialize(environment:, api_key:, index_name:, llm:)
       depends_on "pinecone"
       require "pinecone"
 
@@ -20,7 +28,7 @@ module Langchain::Vectorsearch
       @client = ::Pinecone::Client.new
       @index_name = index_name
 
-      super(llm: llm, llm_api_key: llm_api_key)
+      super(llm: llm)
     end
 
     # Add a list of texts to the index
@@ -34,7 +42,7 @@ module Langchain::Vectorsearch
           # TODO: Allows passing in your own IDs
           id: SecureRandom.uuid,
           metadata: metadata || {content: text},
-          values: llm_client.embed(text: text)
+          values: llm.embed(text: text)
         }
       end
 
@@ -65,7 +73,7 @@ module Langchain::Vectorsearch
       namespace: "",
       filter: nil
     )
-      embedding = llm_client.embed(text: query)
+      embedding = llm.embed(text: query)
 
       similarity_search_by_vector(
         embedding: embedding,
@@ -112,7 +120,7 @@ module Langchain::Vectorsearch
 
       prompt = generate_prompt(question: question, context: context)
 
-      llm_client.chat(prompt: prompt)
+      llm.chat(prompt: prompt)
     end
   end
 end