langchainrb 0.8.0 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ea2adff257b4151b8acf24a02de851df2d99fe8890d6afd06bcdc3a5f53e9e1
-  data.tar.gz: 646a5f9246bffc20654672393f9175c1f0f30533ba1546cef05ce951d449c9ec
+  metadata.gz: 13eec34cc529732ddfb8994956659bd4307a79ebfd76ff883fe3b6644d647c24
+  data.tar.gz: ce04acfe42a6a8da5a5951734651dd0083f7d2efc43cf4b3367710c8221ee96a
 SHA512:
-  metadata.gz: 3b2aaace63c46b7eec9d8cc04a2cd9cc84c79c90a5a1f1ce1bcb11e4416021f89293d40309ca35b0e4dbb2036a2962bde0faa28ad46d081846dcb00a9a1bf783
-  data.tar.gz: fd5e8e03053ab99a737b3ce17c12ae76da2bc1d0b4bda89eb16e16afe43f260325af78a7c62faf0041c8869cbd94c0a5bbbda920bb7e1d7f175ac35545b53f00
+  metadata.gz: 2094d99610311a1583d890f8c6898605bcd3e76d2fb72deb1ccd4b250f2b98f7a883401faf2e161b97b82fb29f6e64ead8843d8af22f0bd3e8a4c872c150c134
+  data.tar.gz: d7ce155cbb992e651aa8dc468ed1ee39bd96d1457f50faa11a32d7caac87086f5d8a381fc2b50aaba10ac934486ed415d5e609f47ee0426b4187540e2436b2e9

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## [Unreleased]
 
+## [0.8.2]
+- Introducing new `Langchain::Chunker::Markdown` chunker (thanks @spikex)
+- Fixes
+
+## [0.8.1]
+- Support for Epsilla vector DB
+- Fully functioning Google Vertex AI LLM
+- Bug fixes
+
 ## [0.8.0]
 - [BREAKING] Updated llama_cpp.rb to 0.9.4. The model file format used by the underlying llama.cpp library has changed to GGUF. llama.cpp ships with scripts to convert existing files and GGUF format models can be downloaded from HuggingFace.
 - Introducing Langchain::LLM::GoogleVertexAi LLM provider

data/README.md

@@ -90,22 +90,22 @@ llm.embed(text: "foo bar")
 
 Generate a text completion:
 ```ruby
-llm.complete(prompt: "What is the meaning of life?")
+llm.complete(prompt: "What is the meaning of life?").completion
 ```
 
 Generate a chat completion:
 ```ruby
-llm.chat(prompt: "Hey! How are you?")
+llm.chat(prompt: "Hey! How are you?").completion
 ```
 
 Summarize the text:
 ```ruby
-llm.complete(text: "...")
+llm.summarize(text: "...").completion
 ```
 
 You can use any other LLM by invoking the same interface:
 ```ruby
-llm = Langchain::LLM::GooglePalm.new(...)
+llm = Langchain::LLM::GooglePalm.new(api_key: ENV["GOOGLE_PALM_API_KEY"], default_options: { ... })
 ```
 
 ### Prompt Management
@@ -251,7 +251,7 @@ Then parse the llm response:
 
 ```ruby
 llm = Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
-llm_response = llm.chat(prompt: prompt_text)
+llm_response = llm.chat(prompt: prompt_text).completion
 parser.parse(llm_response)
 # {
 #   "name" => "Kim Ji-hyun",
@@ -310,6 +310,7 @@ Langchain.rb provides a convenient unified interface on top of supported vectors
 | Database                                                                                   | Open-source        | Cloud offering     |
 | --------                                                                                   |:------------------:| :------------:     |
 | [Chroma](https://trychroma.com/?utm_source=langchainrb&utm_medium=github)                  | ✅                 | ✅                 |
+| [Epsilla](https://epsilla.com/?utm_source=langchainrb&utm_medium=github)                   | ✅                 | ✅                 |
 | [Hnswlib](https://github.com/nmslib/hnswlib/?utm_source=langchainrb&utm_medium=github)     | ✅                 | ❌                 |
 | [Milvus](https://milvus.io/?utm_source=langchainrb&utm_medium=github)                      | ✅                 | ✅ Zilliz Cloud    |
 | [Pinecone](https://www.pinecone.io/?utm_source=langchainrb&utm_medium=github)              | ❌                 | ✅                 |
@@ -342,6 +343,7 @@ client = Langchain::Vectorsearch::Weaviate.new(
 You can instantiate any other supported vector search database:
 ```ruby
 client = Langchain::Vectorsearch::Chroma.new(...)   # `gem "chroma-db", "~> 0.6.0"`
+client = Langchain::Vectorsearch::Epsilla.new(...)  # `gem "epsilla-ruby", "~> 0.0.3"`
 client = Langchain::Vectorsearch::Hnswlib.new(...)  # `gem "hnswlib", "~> 0.8.1"`
 client = Langchain::Vectorsearch::Milvus.new(...)   # `gem "milvus", "~> 0.9.2"`
 client = Langchain::Vectorsearch::Pinecone.new(...) # `gem "pinecone", "~> 0.1.6"`

data/lib/langchain/chunker/markdown.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "baran"
+
+module Langchain
+  module Chunker
+    #
+    # Simple text chunker
+    #
+    # Usage:
+    #     Langchain::Chunker::Markdown.new(text).chunks
+    #
+    class Markdown < Base
+      attr_reader :text, :chunk_size, :chunk_overlap
+
+      # @param [String] text
+      # @param [Integer] chunk_size
+      # @param [Integer] chunk_overlap
+      # @param [String] separator
+      def initialize(text, chunk_size: 1000, chunk_overlap: 200)
+        @text = text
+        @chunk_size = chunk_size
+        @chunk_overlap = chunk_overlap
+      end
+
+      # @return [Array<Langchain::Chunk>]
+      def chunks
+        splitter = Baran::MarkdownSplitter.new(
+          chunk_size: chunk_size,
+          chunk_overlap: chunk_overlap
+        )
+
+        splitter.chunks(text).map do |chunk|
+          Langchain::Chunk.new(text: chunk[:text])
+        end
+      end
+    end
+  end
+end

data/lib/langchain/data.rb

@@ -9,9 +9,10 @@ module Langchain
 
     # @param data [String] data that was loaded
     # @option options [String] :source URL or Path of the data source
-    def initialize(data, options = {})
-      @source = options[:source]
+    def initialize(data, source: nil, chunker: Langchain::Chunker::Text)
+      @source = source
       @data = data
+      @chunker_klass = chunker
     end
 
     # @return [String]
@@ -22,7 +23,7 @@ module Langchain
     # @param opts [Hash] options passed to the chunker
     # @return [Array<String>]
     def chunks(opts = {})
-      Langchain::Chunker::Text.new(@data, **opts).chunks
+      @chunker_klass.new(@data, **opts).chunks
     end
   end
 end

data/lib/langchain/llm/google_palm.rb

@@ -131,7 +131,7 @@ module Langchain::LLM
         prompt: prompt,
         temperature: @defaults[:temperature],
         # Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
-        max_tokens: 2048
+        max_tokens: 256
       )
     end
 

data/lib/langchain/llm/google_vertex_ai.rb

@@ -12,22 +12,30 @@ module Langchain::LLM
   #
   class GoogleVertexAi < Base
     DEFAULTS = {
-      temperature: 0.2,
+      temperature: 0.1, # 0.1 is the default in the API, quite low ("grounded")
+      max_output_tokens: 1000,
+      top_p: 0.8,
+      top_k: 40,
       dimension: 768,
+      completion_model_name: "text-bison", # Optional: tect-bison@001
       embeddings_model_name: "textembedding-gecko"
     }.freeze
 
-    attr_reader :project_id, :client
+    # Google Cloud has a project id and a specific region of deployment.
+    # For GenAI-related things, a safe choice is us-central1.
+    attr_reader :project_id, :client, :region
 
     def initialize(project_id:, default_options: {})
       depends_on "google-apis-aiplatform_v1"
 
       @project_id = project_id
+      @region = default_options.fetch :region, "us-central1"
 
       @client = Google::Apis::AiplatformV1::AiplatformService.new
 
       # TODO: Adapt for other regions; Pass it in via the constructor
-      @client.root_url = "https://us-central1-aiplatform.googleapis.com/"
+      # For the moment only us-central1 available so no big deal.
+      @client.root_url = "https://#{@region}-aiplatform.googleapis.com/"
       @client.authorization = Google::Auth.get_application_default
 
       @defaults = DEFAULTS.merge(default_options)
@@ -37,7 +45,7 @@ module Langchain::LLM
     # Generate an embedding for a given text
     #
     # @param text [String] The text to generate an embedding for
-    # @return [Langchain::LLM::GooglePalmResponse] Response object
+    # @return [Langchain::LLM::GoogleVertexAiResponse] Response object
     #
     def embed(text:)
       content = [{content: text}]
@@ -45,11 +53,97 @@ module Langchain::LLM
 
       api_path = "projects/#{@project_id}/locations/us-central1/publishers/google/models/#{@defaults[:embeddings_model_name]}"
 
-      puts("api_path: #{api_path}")
+      # puts("api_path: #{api_path}")
 
       response = client.predict_project_location_publisher_model(api_path, request)
 
       Langchain::LLM::GoogleVertexAiResponse.new(response.to_h, model: @defaults[:embeddings_model_name])
     end
+
+    #
+    # Generate a completion for a given prompt
+    #
+    # @param prompt [String] The prompt to generate a completion for
+    # @param params extra parameters passed to GooglePalmAPI::Client#generate_text
+    # @return [Langchain::LLM::GooglePalmResponse] Response object
+    #
+    def complete(prompt:, **params)
+      default_params = {
+        prompt: prompt,
+        temperature: @defaults[:temperature],
+        top_k: @defaults[:top_k],
+        top_p: @defaults[:top_p],
+        max_output_tokens: @defaults[:max_output_tokens],
+        model: @defaults[:completion_model_name]
+      }
+
+      if params[:stop_sequences]
+        default_params[:stop_sequences] = params.delete(:stop_sequences)
+      end
+
+      if params[:max_output_tokens]
+        default_params[:max_output_tokens] = params.delete(:max_output_tokens)
+      end
+
+      # to be tested
+      temperature = params.delete(:temperature) || @defaults[:temperature]
+      max_output_tokens = default_params.fetch(:max_output_tokens, @defaults[:max_output_tokens])
+
+      default_params.merge!(params)
+
+      # response = client.generate_text(**default_params)
+      request = Google::Apis::AiplatformV1::GoogleCloudAiplatformV1PredictRequest.new \
+        instances: [{
+          prompt: prompt # key used to be :content, changed to :prompt
+        }],
+        parameters: {
+          temperature: temperature,
+          maxOutputTokens: max_output_tokens,
+          topP: 0.8,
+          topK: 40
+        }
+
+      response = client.predict_project_location_publisher_model \
+        "projects/#{project_id}/locations/us-central1/publishers/google/models/#{@defaults[:completion_model_name]}",
+        request
+
+      Langchain::LLM::GoogleVertexAiResponse.new(response, model: default_params[:model])
+    end
+
+    #
+    # Generate a summarization for a given text
+    #
+    # @param text [String] The text to generate a summarization for
+    # @return [String] The summarization
+    #
+    # TODO(ricc): add params for Temp, topP, topK, MaxTokens and have it default to these 4 values.
+    def summarize(text:)
+      prompt_template = Langchain::Prompt.load_from_path(
+        file_path: Langchain.root.join("langchain/llm/prompts/summarize_template.yaml")
+      )
+      prompt = prompt_template.format(text: text)
+
+      complete(
+        prompt: prompt,
+        # For best temperature, topP, topK, MaxTokens for summarization: see
+        # https://cloud.google.com/vertex-ai/docs/samples/aiplatform-sdk-summarization
+        temperature: 0.2,
+        top_p: 0.95,
+        top_k: 40,
+        # Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
+        max_output_tokens: 256
+      )
+    end
+
+    def chat(...)
+      # https://cloud.google.com/vertex-ai/docs/samples/aiplatform-sdk-chathat
+      # Chat params: https://cloud.google.com/vertex-ai/docs/samples/aiplatform-sdk-chat
+      # \"temperature\": 0.3,\n"
+      #       + "  \"maxDecodeSteps\": 200,\n"
+      #       + "  \"topP\": 0.8,\n"
+      #       + "  \"topK\": 40\n"
+      #       + "}";
+      raise NotImplementedError, "coming soon for Vertex AI.."
+    end
   end
 end

data/lib/langchain/llm/response/google_vertex_ai_response.rb

@@ -9,10 +9,19 @@ module Langchain::LLM
       super(raw_response, model: model)
     end
 
+    def completion
+      # completions&.dig(0, "output")
+      raw_response.predictions[0]["content"]
+    end
+
     def embedding
       embeddings.first
     end
 
+    def completions
+      raw_response.predictions.map { |p| p["content"] }
+    end
+
     def total_tokens
       raw_response.dig(:predictions, 0, :embeddings, :statistics, :token_count)
     end

data/lib/langchain/llm/response/ollama_response.rb

@@ -8,7 +8,7 @@ module Langchain::LLM
     end
 
     def completion
-      raw_response.first
+      completions.first
     end
 
     def completions

data/lib/langchain/loader.rb

@@ -37,9 +37,10 @@ module Langchain
     # @param path [String | Pathname] path to file or URL
     # @param options [Hash] options passed to the processor class used to process the data
     # @return [Langchain::Loader] loader instance
-    def initialize(path, options = {})
+    def initialize(path, options = {}, chunker: Langchain::Chunker::Text)
       @options = options
       @path = path
+      @chunker = chunker
     end
 
     # Is the path a URL?
@@ -112,7 +113,7 @@ module Langchain
         processor_klass.new(@options).parse(@raw_data)
       end
 
-      Langchain::Data.new(result)
+      Langchain::Data.new(result, source: @options[:source], chunker: @chunker)
     end
 
     def processor_klass

data/lib/langchain/output_parsers/output_fixing_parser.rb

@@ -58,7 +58,7 @@ module Langchain::OutputParsers
           completion: completion,
           error: e
         )
-      )
+      ).completion
       parser.parse(new_completion)
     end
 

data/lib/langchain/processors/markdown.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Processors
+    class Markdown < Base
+      EXTENSIONS = [".markdown", ".md"]
+      CONTENT_TYPES = ["text/markdown"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [String]
+      def parse(data)
+        data.read
+      end
+    end
+  end
+end

data/lib/langchain/prompt/loading.rb

@@ -33,7 +33,7 @@ module Langchain::Prompt
         when ".json"
           config = JSON.parse(File.read(file_path))
         when ".yaml", ".yml"
-          config = YAML.safe_load(File.read(file_path))
+          config = YAML.safe_load_file(file_path)
         else
           raise ArgumentError, "Got unsupported file type #{file_path.extname}"
         end

data/lib/langchain/utils/token_length/ai21_validator.rb

@@ -31,6 +31,10 @@ module Langchain
           TOKEN_LIMITS[model_name]
         end
         singleton_class.alias_method :completion_token_limit, :token_limit
+
+        def self.token_length_from_messages(messages, model_name, options)
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/base_validator.rb

@@ -14,7 +14,7 @@ module Langchain
       class BaseValidator
         def self.validate_max_tokens!(content, model_name, options = {})
           text_token_length = if content.is_a?(Array)
-            content.sum { |item| token_length(item.to_json, model_name, options) }
+            token_length_from_messages(content, model_name, options)
           else
             token_length(content, model_name, options)
           end

data/lib/langchain/utils/token_length/cohere_validator.rb

@@ -39,6 +39,10 @@ module Langchain
           TOKEN_LIMITS[model_name]
         end
         singleton_class.alias_method :completion_token_limit, :token_limit
+
+        def self.token_length_from_messages(messages, model_name, options)
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/google_palm_validator.rb

@@ -43,6 +43,10 @@ module Langchain
           response.dig("tokenCount")
         end
 
+        def self.token_length_from_messages(messages, model_name, options)
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
+
         def self.token_limit(model_name)
           TOKEN_LIMITS.dig(model_name, "input_token_limit")
         end

data/lib/langchain/utils/token_length/openai_validator.rb

@@ -75,6 +75,47 @@ module Langchain
           max_tokens = super(content, model_name, options)
           [options[:max_tokens], max_tokens].reject(&:nil?).min
         end
+
+        # Copied from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+        # Return the number of tokens used by a list of messages
+        #
+        # @param messages [Array<Hash>] The messages to calculate the token length for
+        # @param model [String] The model name to validate against
+        # @return [Integer] The token length of the messages
+        #
+        def self.token_length_from_messages(messages, model_name, options = {})
+          encoding = Tiktoken.encoding_for_model(model_name)
+
+          if ["gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613", "gpt-4-0314", "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613"].include?(model_name)
+            tokens_per_message = 3
+            tokens_per_name = 1
+          elsif model_name == "gpt-3.5-turbo-0301"
+            tokens_per_message = 4  # every message follows {role/name}\n{content}\n
+            tokens_per_name = -1  # if there's a name, the role is omitted
+          elsif model_name.include?("gpt-3.5-turbo")
+            puts "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613."
+            return token_length_from_messages(messages, "gpt-3.5-turbo-0613", options)
+          elsif model_name.include?("gpt-4")
+            puts "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613."
+            return token_length_from_messages(messages, "gpt-4-0613", options)
+          else
+            raise NotImplementedError.new(
+              "token_length_from_messages() is not implemented for model #{model_name}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."
+            )
+          end
+
+          num_tokens = 0
+          messages.each do |message|
+            num_tokens += tokens_per_message
+            message.each do |key, value|
+              num_tokens += encoding.encode(value).length
+              num_tokens += tokens_per_name if ["name", :name].include?(key)
+            end
+          end
+
+          num_tokens += 3  # every reply is primed with assistant
+          num_tokens
+        end
       end
     end
   end

data/lib/langchain/vectorsearch/base.rb

@@ -7,6 +7,7 @@ module Langchain::Vectorsearch
   # == Available vector databases
   #
   # - {Langchain::Vectorsearch::Chroma}
+  # - {Langchain::Vectorsearch::Epsilla}
   # - {Langchain::Vectorsearch::Elasticsearch}
   # - {Langchain::Vectorsearch::Hnswlib}
   # - {Langchain::Vectorsearch::Milvus}
@@ -29,10 +30,11 @@ module Langchain::Vectorsearch
   #     )
   #
   #     # You can instantiate other supported vector databases the same way:
+  #     epsilla  = Langchain::Vectorsearch::Epsilla.new(...)
   #     milvus   = Langchain::Vectorsearch::Milvus.new(...)
   #     qdrant   = Langchain::Vectorsearch::Qdrant.new(...)
   #     pinecone = Langchain::Vectorsearch::Pinecone.new(...)
-  #     chrome   = Langchain::Vectorsearch::Chroma.new(...)
+  #     chroma   = Langchain::Vectorsearch::Chroma.new(...)
   #     pgvector = Langchain::Vectorsearch::Pgvector.new(...)
   #
   # == Schema Creation
@@ -173,13 +175,13 @@ module Langchain::Vectorsearch
       prompt_template.format(question: question, context: context)
     end
 
-    def add_data(paths:)
+    def add_data(paths:, options: {}, chunker: Langchain::Chunker::Text)
       raise ArgumentError, "Paths must be provided" if Array(paths).empty?
 
       texts = Array(paths)
         .flatten
         .map do |path|
-          data = Langchain::Loader.new(path)&.load&.chunks
+          data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
           data.map { |chunk| chunk.text }
         end
 

data/lib/langchain/vectorsearch/epsilla.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "json"
+require "timeout"
+require "uri"
+
+module Langchain::Vectorsearch
+  class Epsilla < Base
+    #
+    # Wrapper around Epsilla client library
+    #
+    # Gem requirements:
+    #     gem "epsilla-ruby", "~> 0.0.3"
+    #
+    # Usage:
+    #     epsilla = Langchain::Vectorsearch::Epsilla.new(url:, db_name:, db_path:, index_name:, llm:)
+    #
+    # Initialize Epsilla client
+    # @param url [String] URL to connect to the Epsilla db instance, protocol://host:port
+    # @param db_name [String] The name of the database to use
+    # @param db_path [String] The path to the database to use
+    # @param index_name [String] The name of the Epsilla table to use
+    # @param llm [Object] The LLM client to use
+    def initialize(url:, db_name:, db_path:, index_name:, llm:)
+      depends_on "epsilla-ruby", req: "epsilla"
+
+      uri = URI.parse(url)
+      protocol = uri.scheme
+      host = uri.host
+      port = uri.port
+
+      @client = ::Epsilla::Client.new(protocol, host, port)
+
+      Timeout.timeout(5) do
+        status_code, response = @client.database.load_db(db_name, db_path)
+
+        if status_code != 200
+          if status_code == 409 || (status_code == 500 && response["message"].include?("already loaded"))
+            # When db is already loaded, Epsilla may return HTTP 409 Conflict.
+            # This behavior is changed in https://github.com/epsilla-cloud/vectordb/pull/95
+            # Old behavior (HTTP 500) is preserved for backwards compatibility.
+            # It does not prevent us from using the db.
+            Langchain.logger.info("Database already loaded")
+          else
+            raise "Failed to load database: #{response}"
+          end
+        end
+      end
+
+      @client.database.use_db(db_name)
+
+      @db_name = db_name
+      @db_path = db_path
+      @table_name = index_name
+
+      @vector_dimension = llm.default_dimension
+
+      super(llm: llm)
+    end
+
+    # Create a table using the index_name passed in the constructor
+    def create_default_schema
+      status_code, response = @client.database.create_table(@table_name, [
+        {"name" => "ID", "dataType" => "STRING", "primaryKey" => true},
+        {"name" => "Doc", "dataType" => "STRING"},
+        {"name" => "Embedding", "dataType" => "VECTOR_FLOAT", "dimensions" => @vector_dimension}
+      ])
+      raise "Failed to create table: #{response}" if status_code != 200
+
+      response
+    end
+
+    # Drop the table using the index_name passed in the constructor
+    def destroy_default_schema
+      status_code, response = @client.database.drop_table(@table_name)
+      raise "Failed to drop table: #{response}" if status_code != 200
+
+      response
+    end
+
+    # Add a list of texts to the database
+    # @param texts [Array<String>] The list of texts to add
+    # @param ids [Array<String>] The unique ids to add to the index, in the same order as the texts; if nil, it will be random uuids
+    def add_texts(texts:, ids: nil)
+      validated_ids = ids
+      if ids.nil?
+        validated_ids = texts.map { SecureRandom.uuid }
+      elsif ids.length != texts.length
+        raise "The number of ids must match the number of texts"
+      end
+
+      data = texts.map.with_index do |text, idx|
+        {Doc: text, Embedding: llm.embed(text: text).embedding, ID: validated_ids[idx]}
+      end
+
+      status_code, response = @client.database.insert(@table_name, data)
+      raise "Failed to insert texts: #{response}" if status_code != 200
+      response
+    end
+
+    # Search for similar texts
+    # @param query [String] The text to search for
+    # @param k [Integer] The number of results to return
+    # @return [String] The response from the server
+    def similarity_search(query:, k: 4)
+      embedding = llm.embed(text: query).embedding
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    # Search for entries by embedding
+    # @param embedding [Array<Float>] The embedding to search for
+    # @param k [Integer] The number of results to return
+    # @return [String] The response from the server
+    def similarity_search_by_vector(embedding:, k: 4)
+      status_code, response = @client.database.query(@table_name, "Embedding", embedding, ["Doc"], k, false)
+      raise "Failed to do similarity search: #{response}" if status_code != 200
+
+      data = JSON.parse(response)["result"]
+      data.map { |result| result["Doc"] }
+    end
+
+    # Ask a question and return the answer
+    # @param question [String] The question to ask
+    # @param k [Integer] The number of results to have in context
+    # @yield [String] Stream responses back one String at a time
+    # @return [String] The answer to the question
+    def ask(question:, k: 4, &block)
+      search_results = similarity_search(query: question, k: k)
+
+      context = search_results.map do |result|
+        result.to_s
+      end
+      context = context.join("\n---\n")
+
+      prompt = generate_rag_prompt(question: question, context: context)
+
+      response = llm.chat(prompt: prompt, &block)
+      response.context = context
+      response
+    end
+  end
+end

data/lib/langchain/vectorsearch/pinecone.rb

@@ -64,13 +64,13 @@ module Langchain::Vectorsearch
       index.upsert(vectors: vectors, namespace: namespace)
     end
 
-    def add_data(paths:, namespace: "")
+    def add_data(paths:, namespace: "", options: {}, chunker: Langchain::Chunker::Text)
       raise ArgumentError, "Paths must be provided" if Array(paths).empty?
 
       texts = Array(paths)
         .flatten
         .map do |path|
-          data = Langchain::Loader.new(path)&.load&.chunks
+          data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
           data.map { |chunk| chunk.text }
         end
 

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.8.0"
+  VERSION = "0.8.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.2
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-29 00:00:00.000000000 Z
+date: 2023-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: baran
@@ -276,6 +276,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 8.2.0
+- !ruby/object:Gem::Dependency
+  name: epsilla-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
 - !ruby/object:Gem::Dependency
   name: eqn
   requirement: !ruby/object:Gem::Requirement
@@ -604,6 +618,7 @@ files:
 - lib/langchain/agent/sql_query_agent/sql_query_agent_sql_prompt.yaml
 - lib/langchain/chunk.rb
 - lib/langchain/chunker/base.rb
+- lib/langchain/chunker/markdown.rb
 - lib/langchain/chunker/prompts/semantic_prompt_template.yml
 - lib/langchain/chunker/recursive_text.rb
 - lib/langchain/chunker/semantic.rb
@@ -663,6 +678,7 @@ files:
 - lib/langchain/processors/html.rb
 - lib/langchain/processors/json.rb
 - lib/langchain/processors/jsonl.rb
+- lib/langchain/processors/markdown.rb
 - lib/langchain/processors/pdf.rb
 - lib/langchain/processors/text.rb
 - lib/langchain/processors/xlsx.rb
@@ -688,6 +704,7 @@ files:
 - lib/langchain/vectorsearch/base.rb
 - lib/langchain/vectorsearch/chroma.rb
 - lib/langchain/vectorsearch/elasticsearch.rb
+- lib/langchain/vectorsearch/epsilla.rb
 - lib/langchain/vectorsearch/hnswlib.rb
 - lib/langchain/vectorsearch/milvus.rb
 - lib/langchain/vectorsearch/pgvector.rb