langchainrb 0.3.7 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 93a3fcc195fbdf55ec52402c1db2f11c929069c03afa90477259e6bf2f542957
-  data.tar.gz: 737e456d831e40e8c388a1986f2483f9dff3934c8b4e05a9456529e017075637
+  metadata.gz: fa4976937d6ccf9b2c0e1f0e0d98f53a98f21f0d9d1abbe85d6cc83b646cb8d7
+  data.tar.gz: e563cf1aaece650c2cbe2727b06b15c2e284276c78f1e820070d566a6d3b7386
 SHA512:
-  metadata.gz: 23619f8e256a9856eb113afce8eef94f759beb84644a87d9b67cac4fac9d5aedfc06978b3baedad1d988308b3cdb72d32e19a0f5e21d8b8431e8c9ff04eda548
-  data.tar.gz: 6cd0fcc55553a5472e2ac6c69a6e49dbd4b52fe8bcc899c04710ae40397c39947a3418649737c0d21a99086f85736d52f5968b0cc8af761f186eedf790ba85db
+  metadata.gz: 70e086ce4951b95f1f232b436354f2566caedb0d200b55e3f7ed3bd9589043b06053e78198f97ed47335825edae528e6559b99378899b2c435d5f4baa66f137b
+  data.tar.gz: 75a358762a36f9d37881fe9b265764cb4b8f9578c6598dd38bd3f7323b919e90c1531b7dda769b687e29eb595ec7852cc5698254a80dd5a5a334929894e783df

data/.env.example

@@ -1,3 +1,4 @@
+CHROMA_URL=
 COHERE_API_KEY=
 HUGGING_FACE_API_KEY=
 MILVUS_URL=

data/CHANGELOG.md

@@ -1,19 +1,30 @@
 ## [Unreleased]
 
+## [0.3.9] - 2023-05-19
+- 🚚 Loaders
+  - Introduce `Loaders::Docx` to parse .docx files
+
+## [0.3.8] - 2023-05-19
+- 🔍 Vectorsearch
+  - Introduce support for Chroma DB
+
+- 🚚 Loaders
+  - Bug fix `Loaders::Text` to only parse .txt files
+
 ## [0.3.7] - 2023-05-19
-- Loaders
+- 🚚 Loaders
   - Introduce `Loaders::Text` to parse .txt files
-  - Introduec `Loaders::PDF` to parse .pdf files
+  - Introduce `Loaders::PDF` to parse .pdf files
 
 ## [0.3.6] - 2023-05-17
-- LLMs
+- 🗣️ LLMs
   - Bump `hugging-face` gem version
 
 ## [0.3.5] - 2023-05-16
 - Bug fixes
 
 ## [0.3.4] - 2023-05-16
-- LLMs
+- 🗣️ LLMs
   - Introducing support for HuggingFace
 
 ## [0.3.3] - 2023-05-16
@@ -22,32 +33,28 @@
 - Use the Ruby logger
 
 ## [0.3.2] - 2023-05-15
-- Agents
+- 🤖 Agents
   - Fix Chain of Thought prompt loader
 
 ## [0.3.1] - 2023-05-12
-- Tools
+- 🛠️ Tools
   - Introducing `Tool::Wikipedia`, a tool that looks up Wikipedia entries
 
 ## [0.3.0] - 2023-05-12
-
-- Agents
+- 🤖 Agents
   - Introducing `Agent::ChainOfThoughtAgent`, a semi-autonomous bot that uses Tools to retrieve additional information in order to make best-effort informed replies to user's questions.
-- Tools
+- 🛠️ Tools
   - Introducing `Tool::Calculator` tool that solves mathematical expressions.
   - Introducing `Tool::Search` tool that executes Google Searches.
 
 ## [0.2.0] - 2023-05-09
-
-- Prompt Templating
+- 📋 Prompt Templating
   - Ability to create prompt templates and save them to JSON files
   - Default `Prompt::FewShotPromptTemplate`
   - New examples added to `examples/`
 
 ## [0.1.4] - 2023-05-02
-
 - Backfilling missing specs
 
 ## [0.1.3] - 2023-05-01
-
 - Initial release

data/Gemfile

@@ -10,3 +10,7 @@ gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
 
 gem "standardrb"
+
+# TODO: Remove this once a new version of the gem is released
+# The new version of `docx` needs to include this fix: https://github.com/ruby-docx/docx/pull/130
+gem "docx", branch: "master", git: "https://github.com/ruby-docx/docx.git"

data/Gemfile.lock

@@ -1,7 +1,16 @@
+GIT
+  remote: https://github.com/ruby-docx/docx.git
+  revision: 08b734da349d980d2d7549b907459a9e2aa5d3bb
+  branch: master
+  specs:
+    docx (0.7.0)
+      nokogiri (~> 1.13, >= 1.13.0)
+      rubyzip (~> 2.0)
+
 PATH
   remote: .
   specs:
-    langchainrb (0.3.7)
+    langchainrb (0.3.9)
 
 GEM
   remote: https://rubygems.org/
@@ -31,6 +40,9 @@ GEM
     ast (2.4.2)
     builder (3.2.4)
     byebug (11.1.3)
+    chroma-db (0.3.0)
+      dry-monads (~> 1.6)
+      ruby-next-core (>= 0.15.0)
     coderay (1.1.3)
     cohere-ruby (0.9.3)
       faraday (~> 1)
@@ -54,6 +66,10 @@ GEM
       concurrent-ruby (~> 1.0)
       dry-core (~> 1.0, < 2)
       zeitwerk (~> 2.6)
+    dry-monads (1.6.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0, < 2)
+      zeitwerk (~> 2.6)
     dry-schema (1.13.1)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 1.0, >= 1.0.1)
@@ -216,12 +232,14 @@ GEM
     rubocop-performance (1.16.0)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
+    ruby-next-core (0.15.3)
     ruby-openai (4.0.0)
       faraday (>= 1)
       faraday-multipart (>= 1)
     ruby-progressbar (1.13.0)
     ruby-rc4 (0.1.5)
     ruby2_keywords (0.0.5)
+    rubyzip (2.3.2)
     standard (1.28.2)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
@@ -253,10 +271,13 @@ GEM
 PLATFORMS
   arm64-darwin-22
   x86_64-darwin-19
+  x86_64-darwin-22
   x86_64-linux
 
 DEPENDENCIES
+  chroma-db (~> 0.3.0)
   cohere-ruby (~> 0.9.3)
+  docx!
   dotenv-rails (~> 2.7.6)
   eqn (~> 1.6.5)
   google_search_results (~> 2.0.0)

data/README.md

@@ -30,10 +30,11 @@ require "langchain"
 
 | Database | Querying           | Storage | Schema Management | Backups | Rails Integration | ??? |
 | -------- |:------------------:| -------:| -----------------:| -------:| -----------------:| ---:|
-| Weaviate | :white_check_mark: | WIP     | WIP               | WIP     |                   |     |
-| Qdrant   | :white_check_mark: | WIP     | WIP               | WIP     |                   |     |
-| Milvus   | :white_check_mark: | WIP     | WIP               | WIP     |                   |     |
-| Pinecone | :white_check_mark: | WIP     | WIP               | WIP     |                   |     |
+| Chroma   | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
+| Milvus   | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
+| Pinecone | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
+| Qdrant   | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
+| Weaviate | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
 
 ### Using Vector Search Databases 🔍
 
@@ -54,6 +55,7 @@ client = Vectorsearch::Weaviate.new(
 client = Vectorsearch::Milvus.new(...) # `gem "milvus", "~> 0.9.0"`
 client = Vectorsearch::Qdrant.new(...) # `gem"qdrant-ruby", "~> 0.9.0"`
 client = Vectorsearch::Pinecone.new(...) # `gem "pinecone", "~> 0.1.6"`
+client = Vectorsearch::Chroma.new(...) # `gem "chroma-db", "~> 0.3.0"`
 ```
 
 ```ruby
@@ -255,6 +257,8 @@ Need to read data from various sources? Load it up.
 | pdf  | Loaders::PDF  | `gem "pdf-reader", "~> 1.4"` |
 | text | Loaders::Text |                              |
 
+## Examples
+Additional examples available: [/examples](https://github.com/andreibondarev/langchainrb/tree/main/examples)
 
 ## Logging
 

data/examples/pdf_store_and_query_with_chroma.rb

@@ -0,0 +1,36 @@
+require "langchain"
+
+# gem install chroma-db
+# or add `gem "chroma-db", "~> 0.3.0"` to your Gemfile
+
+# Instantiate the Chroma client
+chroma = Vectorsearch::Chroma.new(
+  url: ENV["CHROMA_URL"],
+  index_name: "documents",
+  llm: :openai,
+  llm_api_key: ENV["OPENAI_API_KEY"]
+)
+
+# Create the default schema.
+chroma.create_default_schema
+
+# Set up an array of PDF and TXT documents
+docs = [
+  Langchain.root.join("/docs/document.pdf"),
+  Langchain.root.join("/docs/document.txt")
+]
+
+# Add data to the index. Weaviate will use OpenAI to generate embeddings behind the scene.
+chroma.add_texts(
+  texts: docs
+)
+
+# Query your data
+chroma.similarity_search(
+  query: "..."
+)
+
+# Interact with your index through Q&A
+chroma.ask(
+  question: "..."
+)

data/lib/langchain.rb

@@ -24,6 +24,7 @@ end
 
 module Vectorsearch
   autoload :Base, "vectorsearch/base"
+  autoload :Chroma, "vectorsearch/chroma"
   autoload :Milvus, "vectorsearch/milvus"
   autoload :Pinecone, "vectorsearch/pinecone"
   autoload :Qdrant, "vectorsearch/qdrant"
@@ -54,6 +55,7 @@ end
 
 module Loaders
   autoload :Base, "loaders/base"
+  autoload :Docx, "loaders/docx"
   autoload :PDF, "loaders/pdf"
   autoload :Text, "loaders/text"
 end
@@ -61,4 +63,4 @@ end
 autoload :Loader, "loader"
 
 # Load the default Loaders
-Langchain.default_loaders ||= [::Loaders::Text, ::Loaders::PDF]
+Langchain.default_loaders ||= [::Loaders::Text, ::Loaders::PDF, ::Loaders::Docx]

data/lib/llm/cohere.rb

@@ -51,7 +51,5 @@ module LLM
     def chat(...)
       complete(...)
     end
-
-    alias_method :generate_embedding, :embed
   end
 end

data/lib/llm/openai.rb

@@ -71,7 +71,5 @@ module LLM
       response = client.chat(parameters: default_params)
       response.dig("choices", 0, "message", "content")
     end
-
-    alias_method :generate_embedding, :embed
   end
 end

data/lib/loaders/base.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# TODO: Add chunking options to the loaders
+
 module Loaders
   class Base
     def self.load(path)

data/lib/loaders/docx.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Loaders
+  class Docx < Base
+    #
+    # This Loader parses Docx files into text.
+    # If you'd like to use it directly you can do so like this:
+    # Loaders::Docx.new("path/to/my.docx").load
+    #
+    # This parser is also invoked when you're adding data to a Vectorsearch DB:
+    # qdrant = Vectorsearch::Qdrant.new(...)
+    # path = Langchain.root.join("path/to/my.docx")
+    # qdrant.add_data(path: path)
+    #
+
+    def initialize(path)
+      depends_on "docx"
+      require "docx"
+
+      @path = path
+    end
+
+    # Check that the file is a `.docx` file
+    def loadable?
+      @path.to_s.end_with?(".docx")
+    end
+
+    def load
+      ::Docx::Document
+        .open(@path.to_s)
+        .text
+    end
+  end
+end

data/lib/loaders/pdf.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Loaders
   class PDF < Base
     #

data/lib/loaders/text.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Loaders
   class Text < Base
     #
@@ -12,7 +14,7 @@ module Loaders
     #
 
     def loadable?
-      true
+      @path.to_s.end_with?(".txt")
     end
 
     def load

data/lib/vectorsearch/base.rb

@@ -50,7 +50,6 @@ module Vectorsearch
     end
 
     def_delegators :llm_client,
-      :generate_embedding,
       :default_dimension
 
     def generate_prompt(question:, context:)

data/lib/vectorsearch/chroma.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Vectorsearch
+  class Chroma < Base
+    # Initialize the Chroma client
+    # @param url [String] The URL of the Qdrant server
+    # @param api_key [String] The API key to use
+    # @param index_name [String] The name of the index to use
+    # @param llm [Symbol] The LLM to use
+    # @param llm_api_key [String] The API key for the LLM
+    def initialize(url:, index_name:, llm:, llm_api_key:, api_key: nil)
+      depends_on "chroma-db"
+      require "chroma-db"
+
+      ::Chroma.connect_host = url
+      ::Chroma.logger = Langchain.logger
+      ::Chroma.log_level = Langchain.logger.level
+
+      @index_name = index_name
+
+      super(llm: llm, llm_api_key: llm_api_key)
+    end
+
+    # Add a list of texts to the index
+    # @param texts [Array] The list of texts to add
+    # @return [Hash] The response from the server
+    def add_texts(texts:)
+      embeddings = Array(texts).map do |text|
+        ::Chroma::Resources::Embedding.new(
+          # TODO: Add support for passing your own IDs
+          id: SecureRandom.uuid,
+          embedding: llm_client.embed(text: text),
+          # TODO: Add support for passing metadata
+          metadata: [], # metadatas[index],
+          document: text # Do we actually need to store the whole original document?
+        )
+      end
+
+      collection = ::Chroma::Resources::Collection.get(index_name)
+      collection.add(embeddings)
+    end
+
+    # Create the collection with the default schema
+    # @return [Hash] The response from the server
+    def create_default_schema
+      ::Chroma::Resources::Collection.create(index_name)
+    end
+
+    # Search for similar texts
+    # @param query [String] The text to search for
+    # @param k [Integer] The number of results to return
+    # @return [Chroma::Resources::Embedding] The response from the server
+    def similarity_search(
+      query:,
+      k: 4
+    )
+      embedding = llm_client.embed(text: query)
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    # Search for similar texts by embedding
+    # @param embedding [Array] The embedding to search for
+    # @param k [Integer] The number of results to return
+    # @return [Chroma::Resources::Embedding] The response from the server
+    def similarity_search_by_vector(
+      embedding:,
+      k: 4
+    )
+      # Requesting more results than the number of documents in the collection currently throws an error in Chroma DB
+      # Temporary fix inspired by this comment: https://github.com/chroma-core/chroma/issues/301#issuecomment-1520494512
+      count = collection.count
+      n_results = [count, k].min
+
+      collection.query(query_embeddings: [embedding], results: n_results)
+    end
+
+    # Ask a question and return the answer
+    # @param question [String] The question to ask
+    # @return [String] The answer to the question
+    def ask(question:)
+      search_results = similarity_search(query: question)
+
+      context = search_results.map do |result|
+        result.document
+      end
+
+      context = context.join("\n---\n")
+
+      prompt = generate_prompt(question: question, context: context)
+
+      llm_client.chat(prompt: prompt)
+    end
+
+    private
+
+    # @return [Chroma::Resources::Collection] The collection
+    def collection
+      @collection ||= ::Chroma::Resources::Collection.get(index_name)
+    end
+  end
+end

data/lib/vectorsearch/milvus.rb

@@ -15,16 +15,16 @@ module Vectorsearch
     def add_texts(texts:)
       client.entities.insert(
         collection_name: index_name,
-        num_rows: texts.count,
+        num_rows: Array(texts).size,
         fields_data: [
           {
             field_name: "content",
             type: ::Milvus::DATA_TYPES["varchar"],
-            field: texts
+            field: Array(texts)
           }, {
             field_name: "vectors",
             type: ::Milvus::DATA_TYPES["binary_vector"],
-            field: texts.map { |text| generate_embedding(text: text) }
+            field: Array(texts).map { |text| llm_client.embed(text: text) }
           }
         ]
       )
@@ -69,7 +69,7 @@ module Vectorsearch
     end
 
     def similarity_search(query:, k: 4)
-      embedding = generate_embedding(text: query)
+      embedding = llm_client.embed(text: query)
 
       similarity_search_by_vector(
         embedding: embedding,

data/lib/vectorsearch/pinecone.rb

@@ -32,7 +32,7 @@ module Vectorsearch
           # TODO: Allows passing in your own IDs
           id: SecureRandom.uuid,
           metadata: {content: text},
-          values: generate_embedding(text: text)
+          values: llm_client.embed(text: text)
         }
       end
 
@@ -59,7 +59,7 @@ module Vectorsearch
       query:,
       k: 4
     )
-      embedding = generate_embedding(text: query)
+      embedding = llm_client.embed(text: query)
 
       similarity_search_by_vector(
         embedding: embedding,

data/lib/vectorsearch/qdrant.rb

@@ -27,9 +27,9 @@ module Vectorsearch
     def add_texts(texts:)
       batch = {ids: [], vectors: [], payloads: []}
 
-      texts.each do |text|
+      Array(texts).each do |text|
         batch[:ids].push(SecureRandom.uuid)
-        batch[:vectors].push(generate_embedding(text: text))
+        batch[:vectors].push(llm_client.embed(text: text))
         batch[:payloads].push({content: text})
       end
 
@@ -59,7 +59,7 @@ module Vectorsearch
       query:,
       k: 4
     )
-      embedding = generate_embedding(text: query)
+      embedding = llm_client.embed(text: query)
 
       similarity_search_by_vector(
         embedding: embedding,

data/lib/vectorsearch/weaviate.rb

@@ -27,7 +27,7 @@ module Vectorsearch
     # @param texts [Array] The list of texts to add
     # @return [Hash] The response from the server
     def add_texts(texts:)
-      objects = texts.map do |text|
+      objects = Array(texts).map do |text|
         {
           class: index_name,
           properties: {content: text}

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.3.7"
+  VERSION = "0.3.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.3.9
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-19 00:00:00.000000000 Z
+date: 2023-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv-rails
@@ -52,6 +52,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.3
+- !ruby/object:Gem::Dependency
+  name: chroma-db
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: docx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: eqn
   requirement: !ruby/object:Gem::Requirement
@@ -207,9 +235,9 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- examples/.keep
 - examples/create_and_manage_few_shot_prompt_templates.rb
 - examples/create_and_manage_prompt_templates.rb
+- examples/pdf_store_and_query_with_chroma.rb
 - examples/store_and_query_with_pinecone.rb
 - examples/store_and_query_with_qdrant.rb
 - examples/store_and_query_with_weaviate.rb
@@ -224,6 +252,7 @@ files:
 - lib/llm/openai.rb
 - lib/loader.rb
 - lib/loaders/base.rb
+- lib/loaders/docx.rb
 - lib/loaders/pdf.rb
 - lib/loaders/text.rb
 - lib/prompt/base.rb
@@ -235,6 +264,7 @@ files:
 - lib/tool/serp_api.rb
 - lib/tool/wikipedia.rb
 - lib/vectorsearch/base.rb
+- lib/vectorsearch/chroma.rb
 - lib/vectorsearch/milvus.rb
 - lib/vectorsearch/pinecone.rb
 - lib/vectorsearch/qdrant.rb