langchainrb 0.3.12 → 0.3.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 974f0a2b8ce3fe42144016bd740ee9d4f7e597834319cc92fbf1d50bd1f4468e
-  data.tar.gz: 3686a42c37eb117e6d7485ef4f7777c0f12968bb9cdcc3a30c7721c86c0a4325
+  metadata.gz: 33c9436ac8d6a73dc06d30f63c11e4f246b3705aa8934765a53ee59325c3a9cd
+  data.tar.gz: 9cc85603694f9367dd162e25379029a345aa0b5c88cccf303c2af114d43a4010
 SHA512:
-  metadata.gz: a61f9b36d9d19eb6cf87af18c7fb40f55d39771257d08a6af2ec3384988419dfb158ffa8fc81c3769c0149f1ffa8b03200366bbea55b03b0d1553912af8d9ae6
-  data.tar.gz: 7dc53be923fe5b8587f61617198b24c42e8793fbd8e18c42a17035bf68279c59c37c6c691cabe13c83adc5dc2cff66ea293f198297ab9a9de30aa68ca72bd9c4
+  metadata.gz: ca5e81638625939d11999a64d44c92fc57c762a934aa8fd5b110c3f5aacc9a736ab5f02da4366e7a1b9b9ec0335dd1eb1683f5b9d90bd97c81914ea0a698dc7c
+  data.tar.gz: aff49ef9451bcbc9a97d181757a5b913737cbfbb4fc3ca49d423cbd2e59a4a71091816e98c2996ff7f8292cb6e0c0d69931a4f3e4e36e2a69fdd7f745640e266

data/.env.example

@@ -11,4 +11,5 @@ QDRANT_API_KEY=
 QDRANT_URL=
 SERPAPI_API_KEY=
 WEAVIATE_API_KEY=
-WEAVIATE_URL=
+WEAVIATE_URL=
+POSTGRES_URL=

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 ## [Unreleased]
 
+## [0.3.14] - 2023-05-28
+- 🔍 Vectorsearch
+  - Not relying on Weaviate modules anymore
+  - Adding missing specs for Qdrant and Milvus classes
+- 🚚 Loaders
+  - Add Langchain::Data result object for data loaders
+- 🗣️ LLMs
+  - Add `summarize()` method to the LLMs
+
+## [0.3.13] - 2023-05-26
+- 🔍 Vectorsearch
+  - Pgvector support
+- 🚚 Loaders
+  - CSV loader
+  - JSON loader
+  - JSONL loader
+
 ## [0.3.12] - 2023-05-25
 - 🔍 Vectorsearch
   - Introduce namespace support for Pinecone

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.3.12)
+    langchainrb (0.3.14)
 
 GEM
   remote: https://rubygems.org/
@@ -148,9 +148,13 @@ GEM
     milvus (0.9.1)
       faraday (~> 1)
     mini_mime (1.1.2)
+    mini_portile2 (2.8.2)
     minitest (5.18.0)
     multi_xml (0.6.0)
     multipart-post (2.3.0)
+    nokogiri (1.14.3)
+      mini_portile2 (~> 2.8.0)
+      racc (~> 1.4)
     nokogiri (1.14.3-arm64-darwin)
       racc (~> 1.4)
     nokogiri (1.14.3-x86_64-darwin)
@@ -166,6 +170,8 @@ GEM
       hashery (~> 2.0)
       ruby-rc4
       ttfunk
+    pg (1.5.3)
+    pgvector (0.1.1)
     pinecone (0.1.71)
       dry-struct (~> 1.6.0)
       dry-validation (~> 1.10.0)
@@ -273,6 +279,7 @@ GEM
 PLATFORMS
   arm64-darwin-21
   arm64-darwin-22
+  ruby
   x86_64-darwin-19
   x86_64-darwin-22
   x86_64-linux
@@ -290,6 +297,8 @@ DEPENDENCIES
   milvus (~> 0.9.0)
   nokogiri (~> 1.13)
   pdf-reader (~> 1.4)
+  pg (~> 1.5)
+  pgvector (< 0.2)
   pinecone (~> 0.1.6)
   pry-byebug (~> 3.10.0)
   qdrant-ruby (~> 0.9.0)

data/README.md

@@ -284,12 +284,16 @@ Langchain::Loader.load('https://www.example.com/file.pdf')
 
 ##### Supported Formats
 
-| Format | Pocessor         |       Gem Requirements       |
-| ------ | ---------------- | :--------------------------: |
-| docx   | Processors::Docx |   `gem "docx", "~> 0.8.0"`   |
-| html   | Processors::HTML | `gem "nokogiri", "~> 1.13"`  |
-| pdf    | Processors::PDF  | `gem "pdf-reader", "~> 1.4"` |
-| text   | Processors::Text |                              |
+
+| Format | Pocessor                     |       Gem Requirements       |
+| ------ | ---------------------------- | :--------------------------: |
+| docx   | Langchain::Processors::Docx  |   `gem "docx", "~> 0.8.0"`   |
+| html   | Langchain::Processors::HTML  | `gem "nokogiri", "~> 1.13"`  |
+| pdf    | Langchain::Processors::PDF   | `gem "pdf-reader", "~> 1.4"` |
+| text   | Langchain::Processors::Text  |                              |
+| JSON   | Langchain::Processors::JSON  |                              |
+| JSONL  | Langchain::Processors::JSONL |                              |
+| csv    | Langchain::Processors::CSV   |                              |
 
 ## Examples
 Additional examples available: [/examples](https://github.com/andreibondarev/langchainrb/tree/main/examples)
@@ -317,6 +321,7 @@ Langchain.logger.level = :info
 [<img style="border-radius:50%" alt="Andrei Bondarev" src="https://avatars.githubusercontent.com/u/541665?v=4" width="80" height="80" class="avatar">](https://github.com/andreibondarev)
 [<img style="border-radius:50%" alt="Rafael Figueiredo" src="https://avatars.githubusercontent.com/u/35845775?v=4" width="80" height="80" class="avatar">](https://github.com/rafaelqfigueiredo)
 [<img style="border-radius:50%" alt="Ricky Chilcott" src="https://avatars.githubusercontent.com/u/445759?v=4" width="80" height="80" class="avatar">](https://github.com/rickychilcott)
+[<img style="border-radius:50%" alt="Alex Chaplinsky" src="https://avatars.githubusercontent.com/u/695947?v=4" width="80" height="80" class="avatar">](https://github.com/alchaplinsky)
 
 (Criteria for becoming an Honorary Contributor or Core Contributor is pending...)
 

data/lib/langchain/data.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Langchain
+  class Data
+    attr_reader :source
+
+    def initialize(data, options = {})
+      @source = options[:source]
+      @data = data
+    end
+
+    def value
+      @data
+    end
+  end
+end

data/lib/langchain/loader.rb

@@ -58,11 +58,12 @@ module Langchain
     end
 
     def process(&block)
-      data, processor = yield
+      raw_data, kind = yield
 
-      raise UnknownFormatError unless processor
+      raise UnknownFormatError unless kind
 
-      Langchain::Processors.const_get(processor).new.parse(data)
+      processor = Langchain::Processors.const_get(kind).new
+      Langchain::Data.new(processor.parse(raw_data), source: @path)
     end
 
     def find_processor(constant, value)

data/lib/langchain/processors/csv.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Langchain
+  module Processors
+    class CSV < Base
+      EXTENSIONS = [".csv"]
+      CONTENT_TYPES = ["text/csv"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [Array of Hash]
+      def parse(data)
+        ::CSV.new(data.read).map do |row|
+          row.map(&:strip)
+        end
+      end
+    end
+  end
+end

data/lib/langchain/processors/json.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Processors
+    class JSON < Base
+      EXTENSIONS = [".json"]
+      CONTENT_TYPES = ["application/json"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [Hash]
+      def parse(data)
+        ::JSON.parse(data.read)
+      end
+    end
+  end
+end

data/lib/langchain/processors/jsonl.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Processors
+    class JSONL < Base
+      EXTENSIONS = [".jsonl"]
+      CONTENT_TYPES = ["application/jsonl", "application/json-lines", "application/jsonlines"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [Array of Hash]
+      def parse(data)
+        data.read.lines.map do |line|
+          ::JSON.parse(line)
+        end
+      end
+    end
+  end
+end

data/lib/langchain.rb

@@ -17,13 +17,17 @@ module Langchain
   @root = Pathname.new(__dir__)
 
   autoload :Loader, "langchain/loader"
+  autoload :Data, "langchain/data"
 
   module Processors
     autoload :Base, "langchain/processors/base"
-    autoload :PDF, "langchain/processors/pdf"
+    autoload :CSV, "langchain/processors/csv"
+    autoload :Docx, "langchain/processors/docx"
     autoload :HTML, "langchain/processors/html"
+    autoload :JSON, "langchain/processors/json"
+    autoload :JSONL, "langchain/processors/jsonl"
+    autoload :PDF, "langchain/processors/pdf"
     autoload :Text, "langchain/processors/text"
-    autoload :Docx, "langchain/processors/docx"
   end
 end
 
@@ -37,6 +41,7 @@ module Vectorsearch
   autoload :Chroma, "vectorsearch/chroma"
   autoload :Milvus, "vectorsearch/milvus"
   autoload :Pinecone, "vectorsearch/pinecone"
+  autoload :Pgvector, "vectorsearch/pgvector"
   autoload :Qdrant, "vectorsearch/qdrant"
   autoload :Weaviate, "vectorsearch/weaviate"
 end

data/lib/llm/base.rb

@@ -33,6 +33,11 @@ module LLM
       raise NotImplementedError, "#{self.class.name} does not support generating embeddings"
     end
 
+    # Method supported by an LLM that summarizes a given text
+    def summarize(...)
+      raise NotImplementedError, "#{self.class.name} does not support summarization"
+    end
+
     # Ensure that the LLM value passed in is supported
     # @param llm [Symbol] The LLM to use
     def self.validate_llm!(llm:)

data/lib/llm/cohere.rb

@@ -16,9 +16,12 @@ module LLM
       @client = ::Cohere::Client.new(api_key: api_key)
     end
 
+    #
     # Generate an embedding for a given text
+    #
     # @param text [String] The text to generate an embedding for
     # @return [Hash] The embedding
+    #
     def embed(text:)
       response = client.embed(
         texts: [text],
@@ -27,9 +30,12 @@ module LLM
       response.dig("embeddings").first
     end
 
+    #
     # Generate a completion for a given prompt
+    #
     # @param prompt [String] The prompt to generate a completion for
     # @return [Hash] The completion
+    #
     def complete(prompt:, **params)
       default_params = {
         prompt: prompt,
@@ -51,5 +57,16 @@ module LLM
     def chat(...)
       complete(...)
     end
+
+    # Generate a summary in English for a given text
+    #
+    # More parameters available to extend this method with: https://github.com/andreibondarev/cohere-ruby/blob/0.9.4/lib/cohere/client.rb#L107-L115
+    #
+    # @param text [String] The text to generate a summary for
+    # @return [String] The summary
+    def summarize(text:)
+      response = client.summarize(text: text)
+      response.dig("summary")
+    end
   end
 end

data/lib/llm/google_palm.rb

@@ -81,5 +81,25 @@ module LLM
       response = client.generate_chat_message(**default_params)
       response.dig("candidates", 0, "content")
     end
+
+    #
+    # Generate a summarization for a given text
+    #
+    # @param text [String] The text to generate a summarization for
+    # @return [String] The summarization
+    #
+    def summarize(text:)
+      prompt_template = Prompt.load_from_path(
+        file_path: Langchain.root.join("llm/prompts/summarize_template.json")
+      )
+      prompt = prompt_template.format(text: text)
+
+      complete(
+        prompt: prompt,
+        temperature: DEFAULTS[:temperature],
+        # Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
+        max_tokens: 2048
+      )
+    end
   end
 end

data/lib/llm/hugging_face.rb

@@ -12,6 +12,7 @@ module LLM
 
     #
     # Intialize the HuggingFace LLM
+    #
     # @param api_key [String] The API key to use
     #
     def initialize(api_key:)
@@ -21,9 +22,12 @@ module LLM
       @client = ::HuggingFace::InferenceApi.new(api_token: api_key)
     end
 
+    #
     # Generate an embedding for a given text
+    #
     # @param text [String] The text to embed
     # @return [Array] The embedding
+    #
     def embed(text:)
       client.embedding(
         input: text,

data/lib/llm/openai.rb

@@ -18,9 +18,12 @@ module LLM
       @client = ::OpenAI::Client.new(access_token: api_key)
     end
 
+    #
     # Generate an embedding for a given text
+    #
     # @param text [String] The text to generate an embedding for
     # @return [Array] The embedding
+    #
     def embed(text:)
       response = client.embeddings(
         parameters: {
@@ -31,9 +34,12 @@ module LLM
       response.dig("data").first.dig("embedding")
     end
 
+    #
     # Generate a completion for a given prompt
+    #
     # @param prompt [String] The prompt to generate a completion for
     # @return [String] The completion
+    #
     def complete(prompt:, **params)
       default_params = {
         model: DEFAULTS[:completion_model_name],
@@ -51,9 +57,12 @@ module LLM
       response.dig("choices", 0, "text")
     end
 
+    #
     # Generate a chat completion for a given prompt
+    #
     # @param prompt [String] The prompt to generate a chat completion for
     # @return [String] The chat completion
+    #
     def chat(prompt:, **params)
       default_params = {
         model: DEFAULTS[:chat_completion_model_name],
@@ -71,5 +80,25 @@ module LLM
       response = client.chat(parameters: default_params)
       response.dig("choices", 0, "message", "content")
     end
+
+    #
+    # Generate a summary for a given text
+    #
+    # @param text [String] The text to generate a summary for
+    # @return [String] The summary
+    #
+    def summarize(text:)
+      prompt_template = Prompt.load_from_path(
+        file_path: Langchain.root.join("llm/prompts/summarize_template.json")
+      )
+      prompt = prompt_template.format(text: text)
+
+      complete(
+        prompt: prompt,
+        temperature: DEFAULTS[:temperature],
+        # Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
+        max_tokens: 2048
+      )
+    end
   end
 end

data/lib/llm/prompts/summarize_template.json

@@ -0,0 +1,5 @@
+{
+  "_type": "prompt",
+  "input_variables": ["text"],
+  "template": "Write a concise summary of the following:\n\n{text}\n\nCONCISE SUMMARY:"
+}

data/lib/llm/replicate.rb

@@ -23,8 +23,11 @@ module LLM
       dimension: 384
     }.freeze
 
+    #
     # Intialize the Replicate LLM
+    #
     # @param api_key [String] The API key to use
+    #
     def initialize(api_key:)
       depends_on "replicate-ruby"
       require "replicate"
@@ -36,9 +39,12 @@ module LLM
       @client = ::Replicate.client
     end
 
+    #
     # Generate an embedding for a given text
+    #
     # @param text [String] The text to generate an embedding for
     # @return [Hash] The embedding
+    #
     def embed(text:)
       response = embeddings_model.predict(input: text)
 
@@ -50,9 +56,12 @@ module LLM
       response.output
     end
 
+    #
     # Generate a completion for a given prompt
+    #
     # @param prompt [String] The prompt to generate a completion for
     # @return [Hash] The completion
+    #
     def complete(prompt:, **params)
       response = completion_model.predict(prompt: prompt)
 
@@ -73,6 +82,26 @@ module LLM
       complete(...)
     end
 
+    #
+    # Generate a summary for a given text
+    #
+    # @param text [String] The text to generate a summary for
+    # @return [String] The summary
+    #
+    def summarize(text:)
+      prompt_template = Prompt.load_from_path(
+        file_path: Langchain.root.join("llm/prompts/summarize_template.json")
+      )
+      prompt = prompt_template.format(text: text)
+
+      complete(
+        prompt: prompt,
+        temperature: DEFAULTS[:temperature],
+        # Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
+        max_tokens: 2048
+      )
+    end
+
     alias_method :generate_embedding, :embed
 
     private

data/lib/vectorsearch/base.rb

@@ -74,7 +74,7 @@ module Vectorsearch
 
       texts = Array(path || paths)
         .flatten
-        .map { |path| Langchain::Loader.new(path)&.load }
+        .map { |path| Langchain::Loader.new(path)&.load&.value }
         .compact
 
       add_texts(texts: texts)

data/lib/vectorsearch/pgvector.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Vectorsearch
+  # The PostgreSQL vector search adapter
+  class Pgvector < Base
+    # @param url [String] The URL of the PostgreSQL database
+    # @param index_name [String] The name of the table to use for the index
+    # @param llm [String] The URL of the Language Layer API
+    # @param llm_api_key [String] The API key for the Language Layer API
+    # @param api_key [String] The API key for the Vectorsearch DB (not used for PostgreSQL)
+    def initialize(url:, index_name:, llm:, llm_api_key:, api_key: nil)
+      require "pg"
+      require "pgvector"
+
+      @client = ::PG.connect(url)
+      registry = ::PG::BasicTypeRegistry.new.define_default_types
+      ::Pgvector::PG.register_vector(registry)
+      @client.type_map_for_results = PG::BasicTypeMapForResults.new(@client, registry: registry)
+
+      @index_name = index_name
+
+      super(llm: llm, llm_api_key: llm_api_key)
+    end
+
+    # Add a list of texts to the index
+    # @param texts [Array<String>] The texts to add to the index
+    # @return [PG::Result] The response from the database
+    def add_texts(texts:)
+      data = texts.flat_map do |text|
+        [text, llm_client.embed(text: text)]
+      end
+      values = texts.length.times.map { |i| "($#{2 * i + 1}, $#{2 * i + 2})" }.join(",")
+      client.exec_params(
+        "INSERT INTO #{@index_name} (content, vectors) VALUES #{values};",
+        data
+      )
+    end
+
+    # Create default schema
+    # @return [PG::Result] The response from the database
+    def create_default_schema
+      client.exec("CREATE EXTENSION IF NOT EXISTS vector;")
+      client.exec(
+        <<~SQL
+          CREATE TABLE IF NOT EXISTS #{@index_name} (
+            id serial PRIMARY KEY,
+            content TEXT,
+            vectors VECTOR(#{default_dimension})
+          );
+        SQL
+      )
+    end
+
+    # Search for similar texts in the index
+    # @param query [String] The text to search for
+    # @param k [Integer] The number of top results to return
+    # @return [Array<Hash>] The results of the search
+    def similarity_search(query:, k: 4)
+      embedding = llm_client.embed(text: query)
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    # Search for similar texts in the index by the passed in vector.
+    # You must generate your own vector using the same LLM that generated the embeddings stored in the Vectorsearch DB.
+    # @param embedding [Array<Float>] The vector to search for
+    # @param k [Integer] The number of top results to return
+    # @return [Array<Hash>] The results of the search
+    def similarity_search_by_vector(embedding:, k: 4)
+      result = client.transaction do |conn|
+        conn.exec("SET LOCAL ivfflat.probes = 10;")
+        query = <<~SQL
+          SELECT id, content FROM #{@index_name} ORDER BY vectors <-> $1 ASC LIMIT $2;
+        SQL
+        conn.exec_params(query, [embedding, k])
+      end
+
+      result.to_a
+    end
+
+    # Ask a question and return the answer
+    # @param question [String] The question to ask
+    # @return [String] The answer to the question
+    def ask(question:)
+      search_results = similarity_search(query: question)
+
+      context = search_results.map do |result|
+        result["content"].to_s
+      end
+      context = context.join("\n---\n")
+
+      prompt = generate_prompt(question: question, context: context)
+
+      llm_client.chat(prompt: prompt)
+    end
+  end
+end

data/lib/vectorsearch/weaviate.rb

@@ -14,9 +14,7 @@ module Vectorsearch
 
       @client = ::Weaviate::Client.new(
         url: url,
-        api_key: api_key,
-        model_service: llm,
-        model_service_api_key: llm_api_key
+        api_key: api_key
       )
       @index_name = index_name
 
@@ -30,7 +28,8 @@ module Vectorsearch
       objects = Array(texts).map do |text|
         {
           class: index_name,
-          properties: {content: text}
+          properties: {content: text},
+          vector: llm_client.embed(text: text)
         }
       end
 
@@ -43,11 +42,7 @@ module Vectorsearch
     def create_default_schema
       client.schema.create(
         class_name: index_name,
-        vectorizer: "text2vec-#{llm}",
-        # TODO: Figure out a way to optionally enable it
-        # "module_config": {
-        #   "qna-openai": {}
-        # },
+        vectorizer: "none",
         properties: [
           # TODO: Allow passing in your own IDs
           {
@@ -63,14 +58,9 @@ module Vectorsearch
     # @param k [Integer|String] The number of results to return
     # @return [Hash] The search results
     def similarity_search(query:, k: 4)
-      near_text = "{ concepts: [\"#{query}\"] }"
+      embedding = llm_client.embed(text: query)
 
-      client.query.get(
-        class_name: index_name,
-        near_text: near_text,
-        limit: k.to_s,
-        fields: "content _additional { id }"
-      )
+      similarity_search_by_vector(embedding: embedding, k: k)
     end
 
     # Return documents similar to the vector
@@ -92,29 +82,16 @@ module Vectorsearch
     # @param question [String] The question to ask
     # @return [Hash] The answer
     def ask(question:)
-      # Weaviate currently supports the `ask:` parameter only for the OpenAI LLM (with `qna-openai` module enabled).
-      # The Cohere support is on the way: https://github.com/weaviate/weaviate/pull/2600
-      if llm == :openai
-        ask_object = "{ question: \"#{question}\" }"
-
-        client.query.get(
-          class_name: index_name,
-          ask: ask_object,
-          limit: "1",
-          fields: "_additional { answer { result } }"
-        )
-      elsif llm == :cohere
-        search_results = similarity_search(query: question)
+      search_results = similarity_search(query: question)
 
-        context = search_results.map do |result|
-          result.dig("content").to_s
-        end
-        context = context.join("\n---\n")
+      context = search_results.map do |result|
+        result.dig("content").to_s
+      end
+      context = context.join("\n---\n")
 
-        prompt = generate_prompt(question: question, context: context)
+      prompt = generate_prompt(question: question, context: context)
 
-        llm_client.chat(prompt: prompt)
-      end
+      llm_client.chat(prompt: prompt)
     end
   end
 end

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.3.12"
+  VERSION = "0.3.14"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.14
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-25 00:00:00.000000000 Z
+date: 2023-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv-rails
@@ -164,6 +164,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: pgvector
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: pdf-reader
   requirement: !ruby/object:Gem::Requirement
@@ -288,10 +316,14 @@ files:
 - lib/agent/chain_of_thought_agent/chain_of_thought_agent_prompt.json
 - lib/dependency_helper.rb
 - lib/langchain.rb
+- lib/langchain/data.rb
 - lib/langchain/loader.rb
 - lib/langchain/processors/base.rb
+- lib/langchain/processors/csv.rb
 - lib/langchain/processors/docx.rb
 - lib/langchain/processors/html.rb
+- lib/langchain/processors/json.rb
+- lib/langchain/processors/jsonl.rb
 - lib/langchain/processors/pdf.rb
 - lib/langchain/processors/text.rb
 - lib/langchainrb.rb
@@ -300,6 +332,7 @@ files:
 - lib/llm/google_palm.rb
 - lib/llm/hugging_face.rb
 - lib/llm/openai.rb
+- lib/llm/prompts/summarize_template.json
 - lib/llm/replicate.rb
 - lib/prompt/base.rb
 - lib/prompt/few_shot_prompt_template.rb
@@ -312,6 +345,7 @@ files:
 - lib/vectorsearch/base.rb
 - lib/vectorsearch/chroma.rb
 - lib/vectorsearch/milvus.rb
+- lib/vectorsearch/pgvector.rb
 - lib/vectorsearch/pinecone.rb
 - lib/vectorsearch/qdrant.rb
 - lib/vectorsearch/weaviate.rb