langchainrb 0.7.5 → 0.12.0

data/lib/langchain/vectorsearch/elasticsearch.rb

@@ -46,6 +46,9 @@ module Langchain::Vectorsearch
       super(llm: llm)
     end
 
+    # Add a list of texts to the index
+    # @param texts [Array<String>] The list of texts to add
+    # @return [Elasticsearch::Response] from the Elasticsearch server
     def add_texts(texts: [])
       body = texts.map do |text|
         [
@@ -57,6 +60,10 @@ module Langchain::Vectorsearch
       es_client.bulk(body: body)
     end
 
+    # Add a list of texts to the index
+    # @param texts [Array<String>] The list of texts to update
+    # @param texts [Array<Integer>] The list of texts to update
+    # @return [Elasticsearch::Response] from the Elasticsearch server
     def update_texts(texts: [], ids: [])
       body = texts.map.with_index do |text, i|
         [
@@ -68,6 +75,19 @@ module Langchain::Vectorsearch
       es_client.bulk(body: body)
     end
 
+    # Remove a list of texts from the index
+    # @param ids [Array<Integer>] The list of ids to delete
+    # @return [Elasticsearch::Response] from the Elasticsearch server
+    def remove_texts(ids: [])
+      body = ids.map do |id|
+        {delete: {_index: index_name, _id: id}}
+      end
+
+      es_client.bulk(body: body)
+    end
+
+    # Create the index with the default schema
+    # @return [Elasticsearch::Response] Index creation
     def create_default_schema
       es_client.indices.create(
         index: index_name,
@@ -75,6 +95,8 @@ module Langchain::Vectorsearch
       )
     end
 
+    # Deletes the default schema
+    # @return [Elasticsearch::Response] Index deletion
     def delete_default_schema
       es_client.indices.delete(
         index: index_name
@@ -82,7 +104,7 @@ module Langchain::Vectorsearch
     end
 
     def default_vector_settings
-      {type: "dense_vector", dims: llm.default_dimension}
+      {type: "dense_vector", dims: llm.default_dimensions}
     end
 
     def vector_settings
@@ -116,10 +138,32 @@ module Langchain::Vectorsearch
       }
     end
 
-    # TODO: Implement this
-    # def ask()
-    # end
+    # Ask a question and return the answer
+    # @param question [String] The question to ask
+    # @param k [Integer] The number of results to have in context
+    # @yield [String] Stream responses back one String at a time
+    # @return [String] The answer to the question
+    def ask(question:, k: 4, &block)
+      search_results = similarity_search(query: question, k: k)
+
+      context = search_results.map do |result|
+        result[:input]
+      end.join("\n---\n")
+
+      prompt = generate_rag_prompt(question: question, context: context)
+
+      messages = [{role: "user", content: prompt}]
+      response = llm.chat(messages: messages, &block)
+
+      response.context = context
+      response
+    end
 
+    # Search for similar texts
+    # @param text [String] The text to search for
+    # @param k [Integer] The number of results to return
+    # @param query [Hash] Elasticsearch query that needs to be used while searching (Optional)
+    # @return [Elasticsearch::Response] The response from the server
     def similarity_search(text: "", k: 10, query: {})
       if text.empty? && query.empty?
         raise "Either text or query should pass as an argument"
@@ -134,6 +178,11 @@ module Langchain::Vectorsearch
       es_client.search(body: {query: query, size: k}).body
     end
 
+    # Search for similar texts by embedding
+    # @param embedding [Array<Float>] The embedding to search for
+    # @param k [Integer] The number of results to return
+    # @param query [Hash] Elasticsearch query that needs to be used while searching (Optional)
+    # @return [Elasticsearch::Response] The response from the server
     def similarity_search_by_vector(embedding: [], k: 10, query: {})
       if embedding.empty? && query.empty?
         raise "Either embedding or query should pass as an argument"

data/lib/langchain/vectorsearch/epsilla.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "json"
+require "timeout"
+require "uri"
+
+module Langchain::Vectorsearch
+  class Epsilla < Base
+    #
+    # Wrapper around Epsilla client library
+    #
+    # Gem requirements:
+    #     gem "epsilla-ruby", "~> 0.0.3"
+    #
+    # Usage:
+    #     epsilla = Langchain::Vectorsearch::Epsilla.new(url:, db_name:, db_path:, index_name:, llm:)
+    #
+    # Initialize Epsilla client
+    # @param url [String] URL to connect to the Epsilla db instance, protocol://host:port
+    # @param db_name [String] The name of the database to use
+    # @param db_path [String] The path to the database to use
+    # @param index_name [String] The name of the Epsilla table to use
+    # @param llm [Object] The LLM client to use
+    def initialize(url:, db_name:, db_path:, index_name:, llm:)
+      depends_on "epsilla-ruby", req: "epsilla"
+
+      uri = URI.parse(url)
+      protocol = uri.scheme
+      host = uri.host
+      port = uri.port
+
+      @client = ::Epsilla::Client.new(protocol, host, port)
+
+      Timeout.timeout(5) do
+        status_code, response = @client.database.load_db(db_name, db_path)
+
+        if status_code != 200
+          if status_code == 409 || (status_code == 500 && response["message"].include?("already loaded"))
+            # When db is already loaded, Epsilla may return HTTP 409 Conflict.
+            # This behavior is changed in https://github.com/epsilla-cloud/vectordb/pull/95
+            # Old behavior (HTTP 500) is preserved for backwards compatibility.
+            # It does not prevent us from using the db.
+            Langchain.logger.info("Database already loaded")
+          else
+            raise "Failed to load database: #{response}"
+          end
+        end
+      end
+
+      @client.database.use_db(db_name)
+
+      @db_name = db_name
+      @db_path = db_path
+      @table_name = index_name
+
+      @vector_dimensions = llm.default_dimensions
+
+      super(llm: llm)
+    end
+
+    # Create a table using the index_name passed in the constructor
+    def create_default_schema
+      status_code, response = @client.database.create_table(@table_name, [
+        {"name" => "ID", "dataType" => "STRING", "primaryKey" => true},
+        {"name" => "Doc", "dataType" => "STRING"},
+        {"name" => "Embedding", "dataType" => "VECTOR_FLOAT", "dimensions" => @vector_dimensions}
+      ])
+      raise "Failed to create table: #{response}" if status_code != 200
+
+      response
+    end
+
+    # Drop the table using the index_name passed in the constructor
+    def destroy_default_schema
+      status_code, response = @client.database.drop_table(@table_name)
+      raise "Failed to drop table: #{response}" if status_code != 200
+
+      response
+    end
+
+    # Add a list of texts to the database
+    # @param texts [Array<String>] The list of texts to add
+    # @param ids [Array<String>] The unique ids to add to the index, in the same order as the texts; if nil, it will be random uuids
+    def add_texts(texts:, ids: nil)
+      validated_ids = ids
+      if ids.nil?
+        validated_ids = texts.map { SecureRandom.uuid }
+      elsif ids.length != texts.length
+        raise "The number of ids must match the number of texts"
+      end
+
+      data = texts.map.with_index do |text, idx|
+        {Doc: text, Embedding: llm.embed(text: text).embedding, ID: validated_ids[idx]}
+      end
+
+      status_code, response = @client.database.insert(@table_name, data)
+      raise "Failed to insert texts: #{response}" if status_code != 200
+      response
+    end
+
+    # Search for similar texts
+    # @param query [String] The text to search for
+    # @param k [Integer] The number of results to return
+    # @return [String] The response from the server
+    def similarity_search(query:, k: 4)
+      embedding = llm.embed(text: query).embedding
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    # Search for entries by embedding
+    # @param embedding [Array<Float>] The embedding to search for
+    # @param k [Integer] The number of results to return
+    # @return [String] The response from the server
+    def similarity_search_by_vector(embedding:, k: 4)
+      status_code, response = @client.database.query(@table_name, "Embedding", embedding, ["Doc"], k, false)
+      raise "Failed to do similarity search: #{response}" if status_code != 200
+
+      data = JSON.parse(response)["result"]
+      data.map { |result| result["Doc"] }
+    end
+
+    # Ask a question and return the answer
+    # @param question [String] The question to ask
+    # @param k [Integer] The number of results to have in context
+    # @yield [String] Stream responses back one String at a time
+    # @return [String] The answer to the question
+    def ask(question:, k: 4, &block)
+      search_results = similarity_search(query: question, k: k)
+
+      context = search_results.map do |result|
+        result.to_s
+      end
+      context = context.join("\n---\n")
+
+      prompt = generate_rag_prompt(question: question, context: context)
+
+      messages = [{role: "user", content: prompt}]
+      response = llm.chat(messages: messages, &block)
+
+      response.context = context
+      response
+    end
+  end
+end

data/lib/langchain/vectorsearch/hnswlib.rb

@@ -26,7 +26,7 @@ module Langchain::Vectorsearch
 
       super(llm: llm)
 
-      @client = ::Hnswlib::HierarchicalNSW.new(space: DEFAULT_METRIC, dim: llm.default_dimension)
+      @client = ::Hnswlib::HierarchicalNSW.new(space: DEFAULT_METRIC, dim: llm.default_dimensions)
       @path_to_index = path_to_index
 
       initialize_index
@@ -86,6 +86,10 @@ module Langchain::Vectorsearch
       client.search_knn(embedding, k)
     end
 
+    # TODO: Add the ask() method
+    # def ask
+    # end
+
     private
 
     #

data/lib/langchain/vectorsearch/milvus.rb

@@ -71,7 +71,7 @@ module Langchain::Vectorsearch
             type_params: [
               {
                 key: "dim",
-                value: llm.default_dimension.to_s
+                value: llm.default_dimensions.to_s
               }
             ]
           }
@@ -151,7 +151,9 @@ module Langchain::Vectorsearch
 
       prompt = generate_rag_prompt(question: question, context: context)
 
-      response = llm.chat(prompt: prompt, &block)
+      messages = [{role: "user", content: prompt}]
+      response = llm.chat(messages: messages, &block)
+
       response.context = context
       response
     end

data/lib/langchain/vectorsearch/pgvector.rb

@@ -16,7 +16,8 @@ module Langchain::Vectorsearch
     # The operators supported by the PostgreSQL vector search adapter
     OPERATORS = {
       "cosine_distance" => "cosine",
-      "euclidean_distance" => "euclidean"
+      "euclidean_distance" => "euclidean",
+      "inner_product_distance" => "inner_product"
     }
     DEFAULT_OPERATOR = "cosine_distance"
 
@@ -89,15 +90,22 @@ module Langchain::Vectorsearch
       upsert_texts(texts: texts, ids: ids)
     end
 
+    # Remove a list of texts from the index
+    # @param ids [Array<Integer>] The ids of the texts to remove from the index
+    # @return [Integer] The number of texts removed from the index
+    def remove_texts(ids:)
+      @db[table_name.to_sym].where(id: ids).delete
+    end
+
     # Create default schema
     def create_default_schema
       db.run "CREATE EXTENSION IF NOT EXISTS vector"
       namespace_column = @namespace_column
-      vector_dimension = llm.default_dimension
+      vector_dimensions = llm.default_dimensions
       db.create_table? table_name.to_sym do
         primary_key :id
         text :content
-        column :vectors, "vector(#{vector_dimension})"
+        column :vectors, "vector(#{vector_dimensions})"
         text namespace_column.to_sym, default: nil
       end
     end
@@ -148,7 +156,9 @@ module Langchain::Vectorsearch
 
       prompt = generate_rag_prompt(question: question, context: context)
 
-      response = llm.chat(prompt: prompt, &block)
+      messages = [{role: "user", content: prompt}]
+      response = llm.chat(messages: messages, &block)
+
       response.context = context
       response
     end

data/lib/langchain/vectorsearch/pinecone.rb

@@ -17,12 +17,13 @@ module Langchain::Vectorsearch
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
-    def initialize(environment:, api_key:, index_name:, llm:)
+    def initialize(environment:, api_key:, index_name:, llm:, base_uri: nil)
       depends_on "pinecone"
 
       ::Pinecone.configure do |config|
         config.api_key = api_key
         config.environment = environment
+        config.base_uri = base_uri if base_uri
       end
 
       @client = ::Pinecone::Client.new
@@ -64,13 +65,13 @@ module Langchain::Vectorsearch
       index.upsert(vectors: vectors, namespace: namespace)
     end
 
-    def add_data(paths:, namespace: "")
+    def add_data(paths:, namespace: "", options: {}, chunker: Langchain::Chunker::Text)
       raise ArgumentError, "Paths must be provided" if Array(paths).empty?
 
       texts = Array(paths)
         .flatten
         .map do |path|
-          data = Langchain::Loader.new(path)&.load&.chunks
+          data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
           data.map { |chunk| chunk.text }
         end
 
@@ -103,7 +104,7 @@ module Langchain::Vectorsearch
       client.create_index(
         metric: DEFAULT_METRIC,
         name: index_name,
-        dimension: llm.default_dimension
+        dimension: llm.default_dimensions
       )
     end
 
@@ -180,7 +181,9 @@ module Langchain::Vectorsearch
 
       prompt = generate_rag_prompt(question: question, context: context)
 
-      response = llm.chat(prompt: prompt, &block)
+      messages = [{role: "user", content: prompt}]
+      response = llm.chat(messages: messages, &block)
+
       response.context = context
       response
     end

data/lib/langchain/vectorsearch/qdrant.rb

@@ -44,14 +44,14 @@ module Langchain::Vectorsearch
     # Add a list of texts to the index
     # @param texts [Array<String>] The list of texts to add
     # @return [Hash] The response from the server
-    def add_texts(texts:, ids: [])
+    def add_texts(texts:, ids: [], payload: {})
       batch = {ids: [], vectors: [], payloads: []}
 
       Array(texts).each_with_index do |text, i|
         id = ids[i] || SecureRandom.uuid
         batch[:ids].push(id)
         batch[:vectors].push(llm.embed(text: text).embedding)
-        batch[:payloads].push({content: text})
+        batch[:payloads].push({content: text}.merge(payload))
       end
 
       client.points.upsert(
@@ -64,6 +64,16 @@ module Langchain::Vectorsearch
       add_texts(texts: texts, ids: ids)
     end
 
+    # Remove a list of texts from the index
+    # @param ids [Array<Integer>] The ids to remove
+    # @return [Hash] The response from the server
+    def remove_texts(ids:)
+      client.points.delete(
+        collection_name: index_name,
+        points: ids
+      )
+    end
+
     # Get the default schema
     # @return [Hash] The response from the server
     def get_default_schema
@@ -83,7 +93,7 @@ module Langchain::Vectorsearch
         collection_name: index_name,
         vectors: {
           distance: DEFAULT_METRIC.capitalize,
-          size: llm.default_dimension
+          size: llm.default_dimensions
         }
       )
     end
@@ -137,7 +147,9 @@ module Langchain::Vectorsearch
 
       prompt = generate_rag_prompt(question: question, context: context)
 
-      response = llm.chat(prompt: prompt, &block)
+      messages = [{role: "user", content: prompt}]
+      response = llm.chat(messages: messages, &block)
+
       response.context = context
       response
     end

data/lib/langchain/vectorsearch/weaviate.rb

@@ -9,7 +9,7 @@ module Langchain::Vectorsearch
     #     gem "weaviate-ruby", "~> 0.8.9"
     #
     # Usage:
-    #     weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:)
+    #     weaviate = Langchain::Vectorsearch::Weaviate.new(url: ENV["WEAVIATE_URL"], api_key: ENV["WEAVIATE_API_KEY"], index_name: "Docs", llm: llm)
     #
 
     # Initialize the Weaviate adapter
@@ -71,6 +71,22 @@ module Langchain::Vectorsearch
       end
     end
 
+    # Deletes a list of texts in the index
+    # @param ids [Array] The ids of texts to delete
+    # @return [Hash] The response from the server
+    def remove_texts(ids:)
+      raise ArgumentError, "ids must be an array" unless ids.is_a?(Array)
+
+      client.objects.batch_delete(
+        class_name: index_name,
+        where: {
+          path: ["__id"],
+          operator: "ContainsAny",
+          valueTextArray: ids
+        }
+      )
+    end
+
     # Create default schema
     # @return [Hash] The response from the server
     def create_default_schema
@@ -137,7 +153,9 @@ module Langchain::Vectorsearch
 
       prompt = generate_rag_prompt(question: question, context: context)
 
-      response = llm.chat(prompt: prompt, &block)
+      messages = [{role: "user", content: prompt}]
+      response = llm.chat(messages: messages, &block)
+
       response.context = context
       response
     end

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.7.5"
+  VERSION = "0.12.0"
 end

data/lib/langchain.rb

@@ -16,14 +16,29 @@ loader.inflector.inflect(
   "json" => "JSON",
   "jsonl" => "JSONL",
   "llm" => "LLM",
+  "mistral_ai" => "MistralAI",
+  "mistral_ai_response" => "MistralAIResponse",
   "openai" => "OpenAI",
   "openai_validator" => "OpenAIValidator",
   "openai_response" => "OpenAIResponse",
-  "pdf" => "PDF",
-  "react_agent" => "ReActAgent",
-  "sql_query_agent" => "SQLQueryAgent"
+  "pdf" => "PDF"
 )
 loader.collapse("#{__dir__}/langchain/llm/response")
+loader.collapse("#{__dir__}/langchain/assistants")
+
+loader.collapse("#{__dir__}/langchain/tool/calculator")
+loader.collapse("#{__dir__}/langchain/tool/database")
+loader.collapse("#{__dir__}/langchain/tool/file_system")
+loader.collapse("#{__dir__}/langchain/tool/google_search")
+loader.collapse("#{__dir__}/langchain/tool/ruby_code_interpreter")
+loader.collapse("#{__dir__}/langchain/tool/vectorsearch")
+loader.collapse("#{__dir__}/langchain/tool/weather")
+loader.collapse("#{__dir__}/langchain/tool/wikipedia")
+
+# RubyCodeInterpreter does not work with Ruby 3.3;
+# https://github.com/ukutaht/safe_ruby/issues/4
+loader.ignore("#{__dir__}/langchain/tool/ruby_code_interpreter") if RUBY_VERSION >= "3.3.0"
+
 loader.setup
 
 # Langchain.rb a is library for building LLM-backed Ruby applications. It is an abstraction layer that sits on top of the emerging AI-related tools that makes it easy for developers to consume and string those services together.
@@ -64,7 +79,7 @@ loader.setup
 #
 # = Logging
 #
-# LangChain.rb uses standard logging mechanisms and defaults to :debug level. Most messages are at info level, but we will add debug or warn statements as needed. To show all log messages:
+# Langchain.rb uses standard logging mechanisms and defaults to :debug level. Most messages are at info level, but we will add debug or warn statements as needed. To show all log messages:
 #
 # Langchain.logger.level = :info
 module Langchain
@@ -82,7 +97,7 @@ module Langchain
     attr_reader :root
   end
 
-  self.logger ||= ::Logger.new($stdout, level: :warn)
+  self.logger ||= ::Logger.new($stdout, level: :debug)
 
   @root = Pathname.new(__dir__)