langchainrb 0.3.9 → 0.3.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa4976937d6ccf9b2c0e1f0e0d98f53a98f21f0d9d1abbe85d6cc83b646cb8d7
-  data.tar.gz: e563cf1aaece650c2cbe2727b06b15c2e284276c78f1e820070d566a6d3b7386
+  metadata.gz: ff7f99d961b09e827df297ddb3144821c9103fd40eabb32688ca92588a73415c
+  data.tar.gz: bb83eaa99055cf45cceaccb18a84e9fd4ee3ea4a93a6a0c66e04ede43e5d4bc0
 SHA512:
-  metadata.gz: 70e086ce4951b95f1f232b436354f2566caedb0d200b55e3f7ed3bd9589043b06053e78198f97ed47335825edae528e6559b99378899b2c435d5f4baa66f137b
-  data.tar.gz: 75a358762a36f9d37881fe9b265764cb4b8f9578c6598dd38bd3f7323b919e90c1531b7dda769b687e29eb595ec7852cc5698254a80dd5a5a334929894e783df
+  metadata.gz: 40e5362520220d3ffc1b4c29c3e430b051de334c2f281d9cb7d7549a93be40b26b379dbd35d0c91ccb5010c1a495a653e31768f1b7a95bc087059d59339fd1a7
+  data.tar.gz: 04f24944b590ee8b577419a95718ad6796bc4cdc34d52cf05e287806912c05ba8ace22e07181a8537892124881b65a8a221e1c75d8c6245231dd5660c6b4308c

data/.env.example

@@ -3,8 +3,10 @@ COHERE_API_KEY=
 HUGGING_FACE_API_KEY=
 MILVUS_URL=
 OPENAI_API_KEY=
+GOOGLE_PALM_API_KEY=
 PINECONE_API_KEY=
 PINECONE_ENVIRONMENT=
+REPLICATE_API_KEY=
 QDRANT_API_KEY=
 QDRANT_URL=
 SERPAPI_API_KEY=

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## [Unreleased]
 
+## [0.3.11] - 2023-05-23
+- 🗣️ LLMs
+  - Introducing support for Google PaLM (Pathways Language Model)
+- Bug fixes and improvements
+
+## [0.3.10] - 2023-05-19
+- 🗣️ LLMs
+  - Introducing support for Replicate.com
+
 ## [0.3.9] - 2023-05-19
 - 🚚 Loaders
   - Introduce `Loaders::Docx` to parse .docx files

data/Gemfile

@@ -10,7 +10,3 @@ gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
 
 gem "standardrb"
-
-# TODO: Remove this once a new version of the gem is released
-# The new version of `docx` needs to include this fix: https://github.com/ruby-docx/docx/pull/130
-gem "docx", branch: "master", git: "https://github.com/ruby-docx/docx.git"

data/Gemfile.lock

@@ -1,16 +1,7 @@
-GIT
-  remote: https://github.com/ruby-docx/docx.git
-  revision: 08b734da349d980d2d7549b907459a9e2aa5d3bb
-  branch: master
-  specs:
-    docx (0.7.0)
-      nokogiri (~> 1.13, >= 1.13.0)
-      rubyzip (~> 2.0)
-
 PATH
   remote: .
   specs:
-    langchainrb (0.3.9)
+    langchainrb (0.3.11)
 
 GEM
   remote: https://rubygems.org/
@@ -44,12 +35,15 @@ GEM
       dry-monads (~> 1.6)
       ruby-next-core (>= 0.15.0)
     coderay (1.1.3)
-    cohere-ruby (0.9.3)
-      faraday (~> 1)
-      faraday_middleware (~> 1)
+    cohere-ruby (0.9.4)
+      faraday (>= 1.0.0)
+      faraday_middleware (>= 1.0.0)
     concurrent-ruby (1.2.2)
     crass (1.0.6)
     diff-lcs (1.5.0)
+    docx (0.8.0)
+      nokogiri (~> 1.13, >= 1.13.0)
+      rubyzip (~> 2.0)
     dotenv (2.7.6)
     dotenv-rails (2.7.6)
       dotenv (= 2.7.6)
@@ -123,6 +117,9 @@ GEM
     faraday-retry (1.0.3)
     faraday_middleware (1.2.0)
       faraday (~> 1.0)
+    google_palm_api (0.1.0)
+      faraday (>= 1.0.0)
+      faraday_middleware (>= 1.0.0)
     google_search_results (2.0.1)
     graphlient (0.6.0)
       faraday (>= 1.0)
@@ -136,8 +133,8 @@ GEM
     httparty (0.21.0)
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
-    hugging-face (0.3.3)
-      faraday (~> 1.0)
+    hugging-face (0.3.4)
+      faraday (>= 1.0)
     i18n (1.13.0)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
@@ -203,6 +200,11 @@ GEM
     rainbow (3.1.1)
     rake (13.0.6)
     regexp_parser (2.8.0)
+    replicate-ruby (0.2.2)
+      addressable
+      faraday (>= 1.0)
+      faraday-multipart
+      faraday-retry
     rexml (3.2.5)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -276,19 +278,22 @@ PLATFORMS
 
 DEPENDENCIES
   chroma-db (~> 0.3.0)
-  cohere-ruby (~> 0.9.3)
-  docx!
+  cohere-ruby (~> 0.9.4)
+  docx (~> 0.8.0)
   dotenv-rails (~> 2.7.6)
   eqn (~> 1.6.5)
+  google_palm_api (~> 0.1.0)
   google_search_results (~> 2.0.0)
-  hugging-face (~> 0.3.3)
+  hugging-face (~> 0.3.4)
   langchainrb!
   milvus (~> 0.9.0)
+  nokogiri (~> 1.13)
   pdf-reader (~> 1.4)
   pinecone (~> 0.1.6)
   pry-byebug (~> 3.10.0)
   qdrant-ruby (~> 0.9.0)
   rake (~> 13.0)
+  replicate-ruby (~> 0.2.2)
   rspec (~> 3.0)
   ruby-openai (~> 4.0.0)
   standardrb

data/README.md

@@ -28,13 +28,13 @@ require "langchain"
 
 #### Supported vector search databases and features:
 
-| Database | Querying           | Storage | Schema Management | Backups | Rails Integration | ??? |
-| -------- |:------------------:| -------:| -----------------:| -------:| -----------------:| ---:|
-| Chroma   | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
-| Milvus   | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
-| Pinecone | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
-| Qdrant   | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
-| Weaviate | :white_check_mark: | WIP     | WIP               | WIP     | WIP               |     |
+| Database | Querying           | Storage | Schema Management | Backups | Rails Integration |
+| -------- |:------------------:| -------:| -----------------:| -------:| -----------------:|
+| [Chroma](https://trychroma.com/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Milvus](https://milvus.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Pinecone](https://www.pinecone.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Qdrant](https://qdrant.tech/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Weaviate](https://weaviate.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 
 ### Using Vector Search Databases 🔍
 
@@ -76,8 +76,9 @@ client.add_texts(
 # Store the contents of your files in your vector search database
 my_pdf = Langchain.root.join("path/to/my.pdf")
 my_text = Langchain.root.join("path/to/my.txt")
+my_docx = Langchain.root.join("path/to/my.docx")
 
-client.add_data(paths: [my_pdf, my_text])
+client.add_data(paths: [my_pdf, my_text, my_docx])
 ```
 ```ruby
 # Retrieve similar documents based on the query string passed in
@@ -130,6 +131,21 @@ cohere.complete(prompt: "What is the meaning of life?")
 
 #### HuggingFace
 Add `gem "hugging-face", "~> 0.3.2"` to your Gemfile.
+```ruby
+cohere = LLM::HuggingFace.new(api_key: ENV["HUGGING_FACE_API_KEY"])
+```
+
+#### Replicate
+Add `gem "replicate-ruby", "~> 0.2.2"` to your Gemfile.
+```ruby
+cohere = LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
+```
+
+#### Google PaLM (Pathways Language Model)
+Add `"google_palm_api", "~> 0.1.0"` to your Gemfile.
+```ruby
+google_palm = LLM::GooglePalm.new(api_key: ENV["GOOGLE_PALM_API_KEY"])
+```
 
 ### Using Prompts 📋
 
@@ -254,6 +270,8 @@ Need to read data from various sources? Load it up.
 
 | Name | Class         | Gem Requirements             |
 | ---- | ------------- | :--------------------------: |
+| docx | Loaders::Docx | `gem "docx", "~> 0.8.0"`     |
+| html | Loaders::HTML | `gem "nokogiri", "~> 1.13"`  |
 | pdf  | Loaders::PDF  | `gem "pdf-reader", "~> 1.4"` |
 | text | Loaders::Text |                              |
 
@@ -273,7 +291,7 @@ Langchain.logger.level = :info
 
 1. `git clone https://github.com/andreibondarev/langchainrb.git`
 2. `cp .env.example .env`, then fill out the environment variables in `.env`
-3. `rspec spec/` to ensure that the tests pass
+3. `bundle exec rake` to ensure that the tests pass and to run standardrb
 4. `bin/console` to load the gem in a REPL session. Feel free to add your own instances of LLMs, Tools, Agents, etc. and experiment with them.
 
 ## Core Contributors

data/examples/pdf_store_and_query_with_chroma.rb

@@ -14,10 +14,15 @@ chroma = Vectorsearch::Chroma.new(
 # Create the default schema.
 chroma.create_default_schema
 
+# gem install these or add them to your Gemfile
+# Add `gem "pdf-reader", "~> 1.4"` to your Gemfile
+# Add `gem "docx", branch: "master", git: "https://github.com/ruby-docx/docx.git"` to your Gemfile
+
 # Set up an array of PDF and TXT documents
 docs = [
   Langchain.root.join("/docs/document.pdf"),
-  Langchain.root.join("/docs/document.txt")
+  Langchain.root.join("/docs/document.txt"),
+  Langchain.root.join("/docs/document.docx")
 ]
 
 # Add data to the index. Weaviate will use OpenAI to generate embeddings behind the scene.

data/lib/dependency_helper.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
+VersionError = Class.new(ScriptError)
+
 # This method requires and loads the given gem, and then checks to see if the version of the gem meets the requirements listed in `langchain.gemspec`
 # This solution was built to avoid auto-loading every single gem in the Gemfile when the developer will mostly likely be only using a few of them.
 #
 # @param gem_name [String] The name of the gem to load
 # @return [Boolean] Whether or not the gem was loaded successfully
 # @raise [LoadError] If the gem is not installed
-# @raise [LoadError] If the gem is installed, but the version does not meet the requirements
+# @raise [VersionError] If the gem is installed, but the version does not meet the requirements
 #
 def depends_on(gem_name)
   gem(gem_name) # require the gem
@@ -14,10 +16,12 @@ def depends_on(gem_name)
   return(true) unless defined?(Bundler) # If we're in a non-bundler environment, we're no longer able to determine if we'll meet requirements
 
   gem_version = Gem.loaded_specs[gem_name].version
-  gem_requirement = Bundler.load.dependencies.find { |g| g.name == gem_name }.requirement
+  gem_requirement = Bundler.load.dependencies.find { |g| g.name == gem_name }&.requirement
+
+  raise LoadError unless gem_requirement
 
-  if !gem_requirement.satisfied_by?(gem_version)
-    raise "The #{gem_name} gem is installed, but version #{gem_requirement} is required. You have #{gem_version}."
+  unless gem_requirement.satisfied_by?(gem_version)
+    raise VersionError, "The #{gem_name} gem is installed, but version #{gem_requirement} is required. You have #{gem_version}."
   end
 
   true

data/lib/langchain.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "logger"
+require "pathname"
 
 require_relative "./version"
 require_relative "./dependency_helper"
@@ -34,8 +35,10 @@ end
 module LLM
   autoload :Base, "llm/base"
   autoload :Cohere, "llm/cohere"
+  autoload :GooglePalm, "llm/google_palm"
   autoload :HuggingFace, "llm/hugging_face"
   autoload :OpenAI, "llm/openai"
+  autoload :Replicate, "llm/replicate"
 end
 
 module Prompt
@@ -58,6 +61,7 @@ module Loaders
   autoload :Docx, "loaders/docx"
   autoload :PDF, "loaders/pdf"
   autoload :Text, "loaders/text"
+  autoload :HTML, "loaders/html"
 end
 
 autoload :Loader, "loader"

data/lib/llm/base.rb

@@ -7,9 +7,11 @@ module LLM
     # Currently supported LLMs
     # TODO: Add support for HuggingFace and other LLMs
     LLMS = {
-      openai: "OpenAI",
       cohere: "Cohere",
-      huggingface: "HuggingFace"
+      google_palm: "GooglePalm",
+      huggingface: "HuggingFace",
+      openai: "OpenAI",
+      replicate: "Replicate"
     }.freeze
 
     def default_dimension

data/lib/llm/google_palm.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module LLM
+  class GooglePalm < Base
+    # Wrapper around the Google PaLM (Pathways Language Model) APIs.
+
+    DEFAULTS = {
+      temperature: 0.0,
+      dimension: 768 # This is what the `embedding-gecko-001` model generates
+    }.freeze
+
+    def initialize(api_key:)
+      depends_on "google_palm_api"
+      require "google_palm_api"
+
+      @client = ::GooglePalmApi::Client.new(api_key: api_key)
+    end
+
+    #
+    # Generate an embedding for a given text
+    #
+    # @param text [String] The text to generate an embedding for
+    # @return [Array] The embedding
+    #
+    def embed(text:)
+      response = client.embed(
+        text: text
+      )
+      response.dig("embedding", "value")
+    end
+
+    #
+    # Generate a completion for a given prompt
+    #
+    # @param prompt [String] The prompt to generate a completion for
+    # @return [String] The completion
+    #
+    def complete(prompt:, **params)
+      default_params = {
+        prompt: prompt,
+        temperature: DEFAULTS[:temperature]
+      }
+
+      if params[:stop_sequences]
+        default_params[:stop_sequences] = params.delete(:stop_sequences)
+      end
+
+      if params[:max_tokens]
+        default_params[:max_output_tokens] = params.delete(:max_tokens)
+      end
+
+      default_params.merge!(params)
+
+      response = client.generate_text(**default_params)
+      response.dig("candidates", 0, "output")
+    end
+
+    #
+    # Generate a chat completion for a given prompt
+    #
+    # @param prompt [String] The prompt to generate a chat completion for
+    # @return [String] The chat completion
+    #
+    def chat(prompt:, **params)
+      # TODO: Figure out how to introduce persisted conversations
+      default_params = {
+        prompt: prompt,
+        temperature: DEFAULTS[:temperature]
+      }
+
+      if params[:stop_sequences]
+        default_params[:stop] = params.delete(:stop_sequences)
+      end
+
+      if params[:max_tokens]
+        default_params[:max_output_tokens] = params.delete(:max_tokens)
+      end
+
+      default_params.merge!(params)
+
+      response = client.generate_chat_message(**default_params)
+      response.dig("candidates", 0, "content")
+    end
+  end
+end

data/lib/llm/replicate.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module LLM
+  class Replicate < Base
+    # Wrapper around Replicate.com LLM provider
+    # Use it directly:
+    # replicate = LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
+    #
+    # Or pass it to be instantiated by a vector search DB:
+    # chroma = Vectorsearch::Chroma.new(
+    #   url: ENV["CHROMA_URL"],
+    #   index_name: "...",
+    #   llm: :replicate,
+    #   llm_api_key: ENV["REPLICATE_API_KEY"],
+    # )
+
+    DEFAULTS = {
+      # TODO: Figure out how to send the temperature to the API
+      temperature: 0.01, # Minimum accepted value
+      # TODO: Design the interface to pass and use different models
+      completion_model_name: "replicate/vicuna-13b",
+      embeddings_model_name: "creatorrr/all-mpnet-base-v2",
+      dimension: 384
+    }.freeze
+
+    # Intialize the Replicate LLM
+    # @param api_key [String] The API key to use
+    def initialize(api_key:)
+      depends_on "replicate-ruby"
+      require "replicate"
+
+      ::Replicate.configure do |config|
+        config.api_token = api_key
+      end
+
+      @client = ::Replicate.client
+    end
+
+    # Generate an embedding for a given text
+    # @param text [String] The text to generate an embedding for
+    # @return [Hash] The embedding
+    def embed(text:)
+      response = embeddings_model.predict(input: text)
+
+      until response.finished?
+        response.refetch
+        sleep(1)
+      end
+
+      response.output
+    end
+
+    # Generate a completion for a given prompt
+    # @param prompt [String] The prompt to generate a completion for
+    # @return [Hash] The completion
+    def complete(prompt:, **params)
+      response = completion_model.predict(prompt: prompt)
+
+      until response.finished?
+        response.refetch
+        sleep(1)
+      end
+
+      # Response comes back as an array of strings, e.g.: ["Hi", "how ", "are ", "you?"]
+      # The first array element is missing a space at the end, so we add it manually
+      response.output[0] += " "
+
+      response.output.join
+    end
+
+    # Cohere does not have a dedicated chat endpoint, so instead we call `complete()`
+    def chat(...)
+      complete(...)
+    end
+
+    alias_method :generate_embedding, :embed
+
+    private
+
+    def completion_model
+      @completion_model ||= client.retrieve_model(DEFAULTS[:completion_model_name]).latest_version
+    end
+
+    def embeddings_model
+      @embeddings_model ||= client.retrieve_model(DEFAULTS[:embeddings_model_name]).latest_version
+    end
+  end
+end

data/lib/loaders/html.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "open-uri"
+
+module Loaders
+  class HTML < Base
+    # We only look for headings and paragraphs
+    TEXT_CONTENT_TAGS = %w[h1 h2 h3 h4 h5 h6 p]
+
+    #
+    # This Loader parses URL into a text.
+    # If you'd like to use it directly you can do so like this:
+    # Loaders::URL.new("https://nokogiri.org/").load
+    #
+    def initialize(url)
+      depends_on "nokogiri"
+      require "nokogiri"
+
+      @url = url
+    end
+
+    # Check that url is a valid URL
+    def loadable?
+      !!(@url =~ URI::DEFAULT_PARSER.make_regexp)
+    end
+
+    def load
+      return unless response.status.first == "200"
+
+      doc = Nokogiri::HTML(response.read)
+      doc.css(TEXT_CONTENT_TAGS.join(",")).map(&:inner_text).join("\n\n")
+    end
+
+    def response
+      @response ||= URI.parse(@url).open
+    end
+  end
+end

data/lib/prompt/base.rb

@@ -66,6 +66,8 @@ module Prompt
     # contained within the template. Input variables are defined as text enclosed in
     # curly braces (e.g. "{variable_name}").
     #
+    # Content within two consecutive curly braces (e.g. "{{ignore_me}}) are ignored.
+    #
     # @param template [String] The template string to extract variables from.
     #
     # @return [Array<String>] An array of input variable names.
@@ -74,9 +76,9 @@ module Prompt
       input_variables = []
       scanner = StringScanner.new(template)
 
-      while scanner.scan_until(/\{([^{}]*)\}/)
+      while scanner.scan_until(/\{([^}]*)\}/)
         variable = scanner[1].strip
-        input_variables << variable unless variable.empty?
+        input_variables << variable unless variable.empty? || variable[0] == "{"
       end
 
       input_variables

data/lib/prompt/prompt_template.rb

@@ -20,7 +20,7 @@ module Prompt
     end
 
     #
-    # Format the prompt with the inputs.
+    # Format the prompt with the inputs. Double {{}} replaced with single {} to adhere to f-string spec.
     #
     # @param kwargs [Hash] Any arguments to be passed to the prompt template.
     # @return [String] A formatted string.
@@ -28,7 +28,7 @@ module Prompt
     def format(**kwargs)
       result = @template
       kwargs.each { |key, value| result = result.gsub(/\{#{key}\}/, value.to_s) }
-      result
+      result.gsub(/{{/, "{").gsub(/}}/, "}")
     end
 
     #

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.3.9"
+  VERSION = "0.3.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.3.11
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-20 00:00:00.000000000 Z
+date: 2023-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv-rails
@@ -39,47 +39,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.10.0
 - !ruby/object:Gem::Dependency
-  name: cohere-ruby
+  name: chroma-db
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.3
+        version: 0.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.3
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
-  name: chroma-db
+  name: cohere-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 0.9.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 0.9.4
 - !ruby/object:Gem::Dependency
   name: docx
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: eqn
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.5
+- !ruby/object:Gem::Dependency
+  name: google_palm_api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: google_search_results
   requirement: !ruby/object:Gem::Requirement
@@ -114,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.3.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.3.4
 - !ruby/object:Gem::Dependency
   name: milvus
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +150,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
 - !ruby/object:Gem::Dependency
   name: pdf-reader
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +192,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.6
+- !ruby/object:Gem::Dependency
+  name: replicate-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
   name: qdrant-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -248,11 +290,14 @@ files:
 - lib/langchain.rb
 - lib/llm/base.rb
 - lib/llm/cohere.rb
+- lib/llm/google_palm.rb
 - lib/llm/hugging_face.rb
 - lib/llm/openai.rb
+- lib/llm/replicate.rb
 - lib/loader.rb
 - lib/loaders/base.rb
 - lib/loaders/docx.rb
+- lib/loaders/html.rb
 - lib/loaders/pdf.rb
 - lib/loaders/text.rb
 - lib/prompt/base.rb