langchainrb 0.3.8 → 0.3.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b208f5fc51ce342bd7ffcfb776487452a40fb0505e4fa6a6b371e0db1d2a278
-  data.tar.gz: 8551edf0406827f92026c8fde54b3b27f32727dec6381f5a33cd58c9c39d40a5
+  metadata.gz: 7537d1ae285b9052051f58cfa43d1d79f9fbcf7590651b3e9a9742495aa9d58a
+  data.tar.gz: 9ec416a4c257c3218f065ec0d42f9a467eb0298961e6d144ec3f642949e4e087
 SHA512:
-  metadata.gz: 0d0d10e84dd47b768979e4f004e9026aac48c45ed5e15ffe499dc0fc9679e806408cc5688cdbd06931e7f63e8840dbb33b5ad7f58ca311eb05a4528757fc9581
-  data.tar.gz: 8723656cefc802cdd4464d24f452a858a1315e654d64d1c256cab9e1de5297c1de0950a4a625278fe33aa8f149db698878bfe608cd06051bc0f8eb8c5abb22f3
+  metadata.gz: 2919f1aea592d394555b9d1b87e533f112ec36a651c41fa5ff2875741bc581f9d545936e5a24b63e89dbe122d289c1fd15c0f3c33c075b3c7cfd2fdd60e9c75f
+  data.tar.gz: f399765255e33aa215e2ef15bb768f73eb8f04ba279b7c0e2bd2c66cacd61bdc1fd537e7f9e6f140e7107a3d5696ef8cc37e1d2d6d330b1b619f1aacada589f9

data/.env.example

@@ -5,6 +5,7 @@ MILVUS_URL=
 OPENAI_API_KEY=
 PINECONE_API_KEY=
 PINECONE_ENVIRONMENT=
+REPLICATE_API_KEY=
 QDRANT_API_KEY=
 QDRANT_URL=
 SERPAPI_API_KEY=

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [0.3.10] - 2023-05-19
+- 🗣️ LLMs
+  - Introducing support for Replicate.com
+
+## [0.3.9] - 2023-05-19
+- 🚚 Loaders
+  - Introduce `Loaders::Docx` to parse .docx files
+
 ## [0.3.8] - 2023-05-19
 - 🔍 Vectorsearch
   - Introduce support for Chroma DB

data/Gemfile

@@ -10,3 +10,7 @@ gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
 
 gem "standardrb"
+
+# TODO: Fix this `faraday` issue where some gems are using 1.x and others are using 2.x
+# Most likely everything will just need to be updated to `faraday 2.x`
+gem "replicate-ruby", git: "https://github.com/andreibondarev/replicate-ruby.git", branch: "faraday-1.x"

data/Gemfile.lock

@@ -1,7 +1,18 @@
+GIT
+  remote: https://github.com/andreibondarev/replicate-ruby.git
+  revision: 6276dec02ce33ae68a57cdd362eb8e87ed29c8e6
+  branch: faraday-1.x
+  specs:
+    replicate-ruby (0.2.1)
+      addressable
+      faraday (>= 1.0)
+      faraday-multipart
+      faraday-retry
+
 PATH
   remote: .
   specs:
-    langchainrb (0.3.8)
+    langchainrb (0.3.10)
 
 GEM
   remote: https://rubygems.org/
@@ -35,12 +46,15 @@ GEM
       dry-monads (~> 1.6)
       ruby-next-core (>= 0.15.0)
     coderay (1.1.3)
-    cohere-ruby (0.9.3)
-      faraday (~> 1)
-      faraday_middleware (~> 1)
+    cohere-ruby (0.9.4)
+      faraday (>= 1.0.0)
+      faraday_middleware (>= 1.0.0)
     concurrent-ruby (1.2.2)
     crass (1.0.6)
     diff-lcs (1.5.0)
+    docx (0.8.0)
+      nokogiri (~> 1.13, >= 1.13.0)
+      rubyzip (~> 2.0)
     dotenv (2.7.6)
     dotenv-rails (2.7.6)
       dotenv (= 2.7.6)
@@ -127,8 +141,8 @@ GEM
     httparty (0.21.0)
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
-    hugging-face (0.3.3)
-      faraday (~> 1.0)
+    hugging-face (0.3.4)
+      faraday (>= 1.0)
     i18n (1.13.0)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
@@ -230,6 +244,7 @@ GEM
     ruby-progressbar (1.13.0)
     ruby-rc4 (0.1.5)
     ruby2_keywords (0.0.5)
+    rubyzip (2.3.2)
     standard (1.28.2)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
@@ -266,11 +281,12 @@ PLATFORMS
 
 DEPENDENCIES
   chroma-db (~> 0.3.0)
-  cohere-ruby (~> 0.9.3)
+  cohere-ruby (~> 0.9.4)
+  docx (~> 0.8.0)
   dotenv-rails (~> 2.7.6)
   eqn (~> 1.6.5)
   google_search_results (~> 2.0.0)
-  hugging-face (~> 0.3.3)
+  hugging-face (~> 0.3.4)
   langchainrb!
   milvus (~> 0.9.0)
   pdf-reader (~> 1.4)
@@ -278,6 +294,7 @@ DEPENDENCIES
   pry-byebug (~> 3.10.0)
   qdrant-ruby (~> 0.9.0)
   rake (~> 13.0)
+  replicate-ruby!
   rspec (~> 3.0)
   ruby-openai (~> 4.0.0)
   standardrb

data/README.md

@@ -28,13 +28,13 @@ require "langchain"
 
 #### Supported vector search databases and features:
 
-| Database | Querying           | Storage | Schema Management | Backups | Rails Integration | ??? |
-| -------- |:------------------:| -------:| -----------------:| -------:| -----------------:| ---:|
-| Chroma   | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
-| Milvus   | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
-| Pinecone | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
-| Qdrant   | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
-| Weaviate | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
+| Database | Querying           | Storage | Schema Management | Backups | Rails Integration |
+| -------- |:------------------:| -------:| -----------------:| -------:| -----------------:|
+| [Chroma](https://trychroma.com/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Milvus](https://milvus.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Pinecone](https://www.pinecone.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Qdrant](https://qdrant.tech/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Weaviate](https://weaviate.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 
 ### Using Vector Search Databases 🔍
 
@@ -76,8 +76,9 @@ client.add_texts(
 # Store the contents of your files in your vector search database
 my_pdf = Langchain.root.join("path/to/my.pdf")
 my_text = Langchain.root.join("path/to/my.txt")
+my_docx = Langchain.root.join("path/to/my.docx")
 
-client.add_data(paths: [my_pdf, my_text])
+client.add_data(paths: [my_pdf, my_text, my_docx])
 ```
 ```ruby
 # Retrieve similar documents based on the query string passed in
@@ -130,6 +131,15 @@ cohere.complete(prompt: "What is the meaning of life?")
 
 #### HuggingFace
 Add `gem "hugging-face", "~> 0.3.2"` to your Gemfile.
+```ruby
+cohere = LLM::HuggingFace.new(api_key: ENV["HUGGING_FACE_API_KEY"])
+```
+
+#### Replicate
+Add `gem "replicate-ruby", git: "https://github.com/andreibondarev/replicate-ruby.git", branch: "faraday-1.x"` to your Gemfile.
+```ruby
+cohere = LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
+```
 
 ### Using Prompts 📋
 
@@ -254,6 +264,7 @@ Need to read data from various sources? Load it up.
 
 | Name | Class         | Gem Requirements             |
 | ---- | ------------- | :--------------------------: |
+| docx | Loaders::Docx | `gem "docx", branch: "master", git: "https://github.com/ruby-docx/docx.git"` |
 | pdf  | Loaders::PDF  | `gem "pdf-reader", "~> 1.4"` |
 | text | Loaders::Text |                              |
 

data/examples/pdf_store_and_query_with_chroma.rb

@@ -14,10 +14,15 @@ chroma = Vectorsearch::Chroma.new(
 # Create the default schema.
 chroma.create_default_schema
 
+# gem install these or add them to your Gemfile
+# Add `gem "pdf-reader", "~> 1.4"` to your Gemfile
+# Add `gem "docx", branch: "master", git: "https://github.com/ruby-docx/docx.git"` to your Gemfile
+
 # Set up an array of PDF and TXT documents
 docs = [
   Langchain.root.join("/docs/document.pdf"),
-  Langchain.root.join("/docs/document.txt")
+  Langchain.root.join("/docs/document.txt"),
+  Langchain.root.join("/docs/document.docx")
 ]
 
 # Add data to the index. Weaviate will use OpenAI to generate embeddings behind the scene.

data/lib/langchain.rb

@@ -36,6 +36,7 @@ module LLM
   autoload :Cohere, "llm/cohere"
   autoload :HuggingFace, "llm/hugging_face"
   autoload :OpenAI, "llm/openai"
+  autoload :Replicate, "llm/replicate"
 end
 
 module Prompt
@@ -55,6 +56,7 @@ end
 
 module Loaders
   autoload :Base, "loaders/base"
+  autoload :Docx, "loaders/docx"
   autoload :PDF, "loaders/pdf"
   autoload :Text, "loaders/text"
 end
@@ -62,4 +64,4 @@ end
 autoload :Loader, "loader"
 
 # Load the default Loaders
-Langchain.default_loaders ||= [::Loaders::Text, ::Loaders::PDF]
+Langchain.default_loaders ||= [::Loaders::Text, ::Loaders::PDF, ::Loaders::Docx]

data/lib/llm/base.rb

@@ -9,7 +9,8 @@ module LLM
     LLMS = {
       openai: "OpenAI",
       cohere: "Cohere",
-      huggingface: "HuggingFace"
+      huggingface: "HuggingFace",
+      replicate: "Replicate"
     }.freeze
 
     def default_dimension

data/lib/llm/replicate.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module LLM
+  class Replicate < Base
+    # Wrapper around Replicate.com LLM provider
+    # Use it directly:
+    # replicate = LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
+    #
+    # Or pass it to be instantiated by a vector search DB:
+    # chroma = Vectorsearch::Chroma.new(
+    #   url: ENV["CHROMA_URL"],
+    #   index_name: "...",
+    #   llm: :replicate,
+    #   llm_api_key: ENV["REPLICATE_API_KEY"],
+    # )
+
+    DEFAULTS = {
+      # TODO: Figure out how to send the temperature to the API
+      temperature: 0.01, # Minimum accepted value
+      # TODO: Design the interface to pass and use different models
+      completion_model_name: "replicate/vicuna-13b",
+      embeddings_model_name: "creatorrr/all-mpnet-base-v2",
+      dimension: 384
+    }.freeze
+
+    # Intialize the Replicate LLM
+    # @param api_key [String] The API key to use
+    def initialize(api_key:)
+      depends_on "replicate-ruby"
+      require "replicate"
+
+      ::Replicate.configure do |config|
+        config.api_token = api_key
+      end
+
+      @client = ::Replicate.client
+    end
+
+    # Generate an embedding for a given text
+    # @param text [String] The text to generate an embedding for
+    # @return [Hash] The embedding
+    def embed(text:)
+      response = embeddings_model.predict(input: text)
+
+      until response.finished?
+        response.refetch
+        sleep(1)
+      end
+
+      response.output
+    end
+
+    # Generate a completion for a given prompt
+    # @param prompt [String] The prompt to generate a completion for
+    # @return [Hash] The completion
+    def complete(prompt:, **params)
+      response = completion_model.predict(prompt: prompt)
+
+      until response.finished?
+        response.refetch
+        sleep(1)
+      end
+
+      # Response comes back as an array of strings, e.g.: ["Hi", "how ", "are ", "you?"]
+      # The first array element is missing a space at the end, so we add it manually
+      response.output[0] += " "
+
+      response.output.join
+    end
+
+    # Cohere does not have a dedicated chat endpoint, so instead we call `complete()`
+    def chat(...)
+      complete(...)
+    end
+
+    alias_method :generate_embedding, :embed
+
+    private
+
+    def completion_model
+      @completion_model ||= client.retrieve_model(DEFAULTS[:completion_model_name]).latest_version
+    end
+
+    def embeddings_model
+      @embeddings_model ||= client.retrieve_model(DEFAULTS[:embeddings_model_name]).latest_version
+    end
+  end
+end

data/lib/loaders/base.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# TODO: Add chunking options to the loaders
+
 module Loaders
   class Base
     def self.load(path)

data/lib/loaders/docx.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Loaders
+  class Docx < Base
+    #
+    # This Loader parses Docx files into text.
+    # If you'd like to use it directly you can do so like this:
+    # Loaders::Docx.new("path/to/my.docx").load
+    #
+    # This parser is also invoked when you're adding data to a Vectorsearch DB:
+    # qdrant = Vectorsearch::Qdrant.new(...)
+    # path = Langchain.root.join("path/to/my.docx")
+    # qdrant.add_data(path: path)
+    #
+
+    def initialize(path)
+      depends_on "docx"
+      require "docx"
+
+      @path = path
+    end
+
+    # Check that the file is a `.docx` file
+    def loadable?
+      @path.to_s.end_with?(".docx")
+    end
+
+    def load
+      ::Docx::Document
+        .open(@path.to_s)
+        .text
+    end
+  end
+end

data/lib/loaders/pdf.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Loaders
   class PDF < Base
     #

data/lib/loaders/text.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Loaders
   class Text < Base
     #

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.3.8"
+  VERSION = "0.3.10"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.3.10
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-20 00:00:00.000000000 Z
+date: 2023-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv-rails
@@ -38,34 +38,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.10.0
+- !ruby/object:Gem::Dependency
+  name: chroma-db
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
   name: cohere-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.3
+        version: 0.9.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.3
+        version: 0.9.4
 - !ruby/object:Gem::Dependency
-  name: chroma-db
+  name: docx
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 0.8.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: eqn
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.3.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.3.4
 - !ruby/object:Gem::Dependency
   name: milvus
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +164,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.6
+- !ruby/object:Gem::Dependency
+  name: replicate-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: qdrant-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -236,8 +264,10 @@ files:
 - lib/llm/cohere.rb
 - lib/llm/hugging_face.rb
 - lib/llm/openai.rb
+- lib/llm/replicate.rb
 - lib/loader.rb
 - lib/loaders/base.rb
+- lib/loaders/docx.rb
 - lib/loaders/pdf.rb
 - lib/loaders/text.rb
 - lib/prompt/base.rb