langchainrb 0.3.12 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 974f0a2b8ce3fe42144016bd740ee9d4f7e597834319cc92fbf1d50bd1f4468e
-  data.tar.gz: 3686a42c37eb117e6d7485ef4f7777c0f12968bb9cdcc3a30c7721c86c0a4325
+  metadata.gz: 2ee811b2bac8fadea4d90c4212363a901829a4aac219da0f2a2dcbe7c6f59c5b
+  data.tar.gz: 8fa32e6df4aaf69cb6d29977913c1b8a30d6f65b777b1f90c8a7f504d869ca8f
 SHA512:
-  metadata.gz: a61f9b36d9d19eb6cf87af18c7fb40f55d39771257d08a6af2ec3384988419dfb158ffa8fc81c3769c0149f1ffa8b03200366bbea55b03b0d1553912af8d9ae6
-  data.tar.gz: 7dc53be923fe5b8587f61617198b24c42e8793fbd8e18c42a17035bf68279c59c37c6c691cabe13c83adc5dc2cff66ea293f198297ab9a9de30aa68ca72bd9c4
+  metadata.gz: cbb7e0c975333248c01082a47f7096fb9d6807c3b7619424eb9348238008d7b4257518287d9358114bf4e3a589349520ebf71ace00bf1fe8906afd27e8b1418a
+  data.tar.gz: 759444abe0b17518c6ef31fed6980f6bc0d3d096606860c4d6fddb8baeda4e0a23fc3909e42eba0f32912a786abec76cac54384533db2787e05d741f0907fa1d

data/.env.example

@@ -11,4 +11,5 @@ QDRANT_API_KEY=
 QDRANT_URL=
 SERPAPI_API_KEY=
 WEAVIATE_API_KEY=
-WEAVIATE_URL=
+WEAVIATE_URL=
+POSTGRES_URL=

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [0.3.13] - 2023-05-26
+- 🔍 Vectorsearch
+  - Pgvector support
+- 🚚 Loaders
+  - CSV loader
+  - JSON loader
+  - JSONL loader
+
 ## [0.3.12] - 2023-05-25
 - 🔍 Vectorsearch
   - Introduce namespace support for Pinecone

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.3.12)
+    langchainrb (0.3.13)
 
 GEM
   remote: https://rubygems.org/
@@ -148,9 +148,13 @@ GEM
     milvus (0.9.1)
       faraday (~> 1)
     mini_mime (1.1.2)
+    mini_portile2 (2.8.2)
     minitest (5.18.0)
     multi_xml (0.6.0)
     multipart-post (2.3.0)
+    nokogiri (1.14.3)
+      mini_portile2 (~> 2.8.0)
+      racc (~> 1.4)
     nokogiri (1.14.3-arm64-darwin)
       racc (~> 1.4)
     nokogiri (1.14.3-x86_64-darwin)
@@ -166,6 +170,8 @@ GEM
       hashery (~> 2.0)
       ruby-rc4
       ttfunk
+    pg (1.5.3)
+    pgvector (0.1.1)
     pinecone (0.1.71)
       dry-struct (~> 1.6.0)
       dry-validation (~> 1.10.0)
@@ -273,6 +279,7 @@ GEM
 PLATFORMS
   arm64-darwin-21
   arm64-darwin-22
+  ruby
   x86_64-darwin-19
   x86_64-darwin-22
   x86_64-linux
@@ -290,6 +297,8 @@ DEPENDENCIES
   milvus (~> 0.9.0)
   nokogiri (~> 1.13)
   pdf-reader (~> 1.4)
+  pg (~> 1.5)
+  pgvector (< 0.2)
   pinecone (~> 0.1.6)
   pry-byebug (~> 3.10.0)
   qdrant-ruby (~> 0.9.0)

data/README.md

@@ -284,12 +284,16 @@ Langchain::Loader.load('https://www.example.com/file.pdf')
 
 ##### Supported Formats
 
-| Format | Pocessor         |       Gem Requirements       |
-| ------ | ---------------- | :--------------------------: |
-| docx   | Processors::Docx |   `gem "docx", "~> 0.8.0"`   |
-| html   | Processors::HTML | `gem "nokogiri", "~> 1.13"`  |
-| pdf    | Processors::PDF  | `gem "pdf-reader", "~> 1.4"` |
-| text   | Processors::Text |                              |
+
+| Format | Pocessor                     |       Gem Requirements       |
+| ------ | ---------------------------- | :--------------------------: |
+| docx   | Langchain::Processors::Docx  |   `gem "docx", "~> 0.8.0"`   |
+| html   | Langchain::Processors::HTML  | `gem "nokogiri", "~> 1.13"`  |
+| pdf    | Langchain::Processors::PDF   | `gem "pdf-reader", "~> 1.4"` |
+| text   | Langchain::Processors::Text  |                              |
+| JSON   | Langchain::Processors::JSON  |                              |
+| JSONL  | Langchain::Processors::JSONL |                              |
+| csv    | Langchain::Processors::CSV   |                              |
 
 ## Examples
 Additional examples available: [/examples](https://github.com/andreibondarev/langchainrb/tree/main/examples)

data/lib/langchain/processors/csv.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Langchain
+  module Processors
+    class CSV < Base
+      EXTENSIONS = [".csv"]
+      CONTENT_TYPES = ["text/csv"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [Array of Hash]
+      def parse(data)
+        ::CSV.new(data.read).map do |row|
+          row.map(&:strip)
+        end
+      end
+    end
+  end
+end

data/lib/langchain/processors/json.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Processors
+    class JSON < Base
+      EXTENSIONS = [".json"]
+      CONTENT_TYPES = ["application/json"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [Hash]
+      def parse(data)
+        ::JSON.parse(data.read)
+      end
+    end
+  end
+end

data/lib/langchain/processors/jsonl.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Processors
+    class JSONL < Base
+      EXTENSIONS = [".jsonl"]
+      CONTENT_TYPES = ["application/jsonl", "application/json-lines", "application/jsonlines"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [Array of Hash]
+      def parse(data)
+        data.read.lines.map do |line|
+          ::JSON.parse(line)
+        end
+      end
+    end
+  end
+end

data/lib/langchain.rb

@@ -20,10 +20,13 @@ module Langchain
 
   module Processors
     autoload :Base, "langchain/processors/base"
-    autoload :PDF, "langchain/processors/pdf"
+    autoload :CSV, "langchain/processors/csv"
+    autoload :Docx, "langchain/processors/docx"
     autoload :HTML, "langchain/processors/html"
+    autoload :JSON, "langchain/processors/json"
+    autoload :JSONL, "langchain/processors/jsonl"
+    autoload :PDF, "langchain/processors/pdf"
     autoload :Text, "langchain/processors/text"
-    autoload :Docx, "langchain/processors/docx"
   end
 end
 
@@ -37,6 +40,7 @@ module Vectorsearch
   autoload :Chroma, "vectorsearch/chroma"
   autoload :Milvus, "vectorsearch/milvus"
   autoload :Pinecone, "vectorsearch/pinecone"
+  autoload :Pgvector, "vectorsearch/pgvector"
   autoload :Qdrant, "vectorsearch/qdrant"
   autoload :Weaviate, "vectorsearch/weaviate"
 end

data/lib/vectorsearch/pgvector.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Vectorsearch
+  # The PostgreSQL vector search adapter
+  class Pgvector < Base
+    # @param url [String] The URL of the PostgreSQL database
+    # @param index_name [String] The name of the table to use for the index
+    # @param llm [String] The URL of the Language Layer API
+    # @param llm_api_key [String] The API key for the Language Layer API
+    # @param api_key [String] The API key for the Vectorsearch DB (not used for PostgreSQL)
+    def initialize(url:, index_name:, llm:, llm_api_key:, api_key: nil)
+      require "pg"
+      require "pgvector"
+
+      @client = ::PG.connect(url)
+      registry = ::PG::BasicTypeRegistry.new.define_default_types
+      ::Pgvector::PG.register_vector(registry)
+      @client.type_map_for_results = PG::BasicTypeMapForResults.new(@client, registry: registry)
+
+      @index_name = index_name
+
+      super(llm: llm, llm_api_key: llm_api_key)
+    end
+
+    # Add a list of texts to the index
+    # @param texts [Array<String>] The texts to add to the index
+    # @return [PG::Result] The response from the database
+    def add_texts(texts:)
+      data = texts.flat_map do |text|
+        [text, llm_client.embed(text: text)]
+      end
+      values = texts.length.times.map { |i| "($#{2 * i + 1}, $#{2 * i + 2})" }.join(",")
+      client.exec_params(
+        "INSERT INTO #{@index_name} (content, vectors) VALUES #{values};",
+        data
+      )
+    end
+
+    # Create default schema
+    # @return [PG::Result] The response from the database
+    def create_default_schema
+      client.exec("CREATE EXTENSION IF NOT EXISTS vector;")
+      client.exec(
+        <<~SQL
+          CREATE TABLE IF NOT EXISTS #{@index_name} (
+            id serial PRIMARY KEY,
+            content TEXT,
+            vectors VECTOR(#{default_dimension})
+          );
+        SQL
+      )
+    end
+
+    # Search for similar texts in the index
+    # @param query [String] The text to search for
+    # @param k [Integer] The number of top results to return
+    # @return [Array<Hash>] The results of the search
+    def similarity_search(query:, k: 4)
+      embedding = llm_client.embed(text: query)
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    # Search for similar texts in the index by the passed in vector.
+    # You must generate your own vector using the same LLM that generated the embeddings stored in the Vectorsearch DB.
+    # @param embedding [Array<Float>] The vector to search for
+    # @param k [Integer] The number of top results to return
+    # @return [Array<Hash>] The results of the search
+    def similarity_search_by_vector(embedding:, k: 4)
+      result = client.transaction do |conn|
+        conn.exec("SET LOCAL ivfflat.probes = 10;")
+        query = <<~SQL
+          SELECT id, content FROM #{@index_name} ORDER BY vectors <-> $1 ASC LIMIT $2;
+        SQL
+        conn.exec_params(query, [embedding, k])
+      end
+
+      result.to_a
+    end
+
+    # Ask a question and return the answer
+    # @param question [String] The question to ask
+    # @return [String] The answer to the question
+    def ask(question:)
+      search_results = similarity_search(query: question)
+
+      context = search_results.map do |result|
+        result["content"].to_s
+      end
+      context = context.join("\n---\n")
+
+      prompt = generate_prompt(question: question, context: context)
+
+      llm_client.chat(prompt: prompt)
+    end
+  end
+end

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.3.12"
+  VERSION = "0.3.13"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.13
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-25 00:00:00.000000000 Z
+date: 2023-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv-rails
@@ -164,6 +164,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: pgvector
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: pdf-reader
   requirement: !ruby/object:Gem::Requirement
@@ -290,8 +318,11 @@ files:
 - lib/langchain.rb
 - lib/langchain/loader.rb
 - lib/langchain/processors/base.rb
+- lib/langchain/processors/csv.rb
 - lib/langchain/processors/docx.rb
 - lib/langchain/processors/html.rb
+- lib/langchain/processors/json.rb
+- lib/langchain/processors/jsonl.rb
 - lib/langchain/processors/pdf.rb
 - lib/langchain/processors/text.rb
 - lib/langchainrb.rb
@@ -312,6 +343,7 @@ files:
 - lib/vectorsearch/base.rb
 - lib/vectorsearch/chroma.rb
 - lib/vectorsearch/milvus.rb
+- lib/vectorsearch/pgvector.rb
 - lib/vectorsearch/pinecone.rb
 - lib/vectorsearch/qdrant.rb
 - lib/vectorsearch/weaviate.rb