langchainrb 0.3.10 → 0.3.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7537d1ae285b9052051f58cfa43d1d79f9fbcf7590651b3e9a9742495aa9d58a
-  data.tar.gz: 9ec416a4c257c3218f065ec0d42f9a467eb0298961e6d144ec3f642949e4e087
+  metadata.gz: ff7f99d961b09e827df297ddb3144821c9103fd40eabb32688ca92588a73415c
+  data.tar.gz: bb83eaa99055cf45cceaccb18a84e9fd4ee3ea4a93a6a0c66e04ede43e5d4bc0
 SHA512:
-  metadata.gz: 2919f1aea592d394555b9d1b87e533f112ec36a651c41fa5ff2875741bc581f9d545936e5a24b63e89dbe122d289c1fd15c0f3c33c075b3c7cfd2fdd60e9c75f
-  data.tar.gz: f399765255e33aa215e2ef15bb768f73eb8f04ba279b7c0e2bd2c66cacd61bdc1fd537e7f9e6f140e7107a3d5696ef8cc37e1d2d6d330b1b619f1aacada589f9
+  metadata.gz: 40e5362520220d3ffc1b4c29c3e430b051de334c2f281d9cb7d7549a93be40b26b379dbd35d0c91ccb5010c1a495a653e31768f1b7a95bc087059d59339fd1a7
+  data.tar.gz: 04f24944b590ee8b577419a95718ad6796bc4cdc34d52cf05e287806912c05ba8ace22e07181a8537892124881b65a8a221e1c75d8c6245231dd5660c6b4308c

data/.env.example

@@ -3,6 +3,7 @@ COHERE_API_KEY=
 HUGGING_FACE_API_KEY=
 MILVUS_URL=
 OPENAI_API_KEY=
+GOOGLE_PALM_API_KEY=
 PINECONE_API_KEY=
 PINECONE_ENVIRONMENT=
 REPLICATE_API_KEY=

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ## [Unreleased]
 
+## [0.3.11] - 2023-05-23
+- 🗣️ LLMs
+  - Introducing support for Google PaLM (Pathways Language Model)
+- Bug fixes and improvements
+
 ## [0.3.10] - 2023-05-19
 - 🗣️ LLMs
   - Introducing support for Replicate.com

data/Gemfile

@@ -10,7 +10,3 @@ gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
 
 gem "standardrb"
-
-# TODO: Fix this `faraday` issue where some gems are using 1.x and others are using 2.x
-# Most likely everything will just need to be updated to `faraday 2.x`
-gem "replicate-ruby", git: "https://github.com/andreibondarev/replicate-ruby.git", branch: "faraday-1.x"

data/Gemfile.lock

@@ -1,18 +1,7 @@
-GIT
-  remote: https://github.com/andreibondarev/replicate-ruby.git
-  revision: 6276dec02ce33ae68a57cdd362eb8e87ed29c8e6
-  branch: faraday-1.x
-  specs:
-    replicate-ruby (0.2.1)
-      addressable
-      faraday (>= 1.0)
-      faraday-multipart
-      faraday-retry
-
 PATH
   remote: .
   specs:
-    langchainrb (0.3.10)
+    langchainrb (0.3.11)
 
 GEM
   remote: https://rubygems.org/
@@ -128,6 +117,9 @@ GEM
     faraday-retry (1.0.3)
     faraday_middleware (1.2.0)
       faraday (~> 1.0)
+    google_palm_api (0.1.0)
+      faraday (>= 1.0.0)
+      faraday_middleware (>= 1.0.0)
     google_search_results (2.0.1)
     graphlient (0.6.0)
       faraday (>= 1.0)
@@ -208,6 +200,11 @@ GEM
     rainbow (3.1.1)
     rake (13.0.6)
     regexp_parser (2.8.0)
+    replicate-ruby (0.2.2)
+      addressable
+      faraday (>= 1.0)
+      faraday-multipart
+      faraday-retry
     rexml (3.2.5)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -285,16 +282,18 @@ DEPENDENCIES
   docx (~> 0.8.0)
   dotenv-rails (~> 2.7.6)
   eqn (~> 1.6.5)
+  google_palm_api (~> 0.1.0)
   google_search_results (~> 2.0.0)
   hugging-face (~> 0.3.4)
   langchainrb!
   milvus (~> 0.9.0)
+  nokogiri (~> 1.13)
   pdf-reader (~> 1.4)
   pinecone (~> 0.1.6)
   pry-byebug (~> 3.10.0)
   qdrant-ruby (~> 0.9.0)
   rake (~> 13.0)
-  replicate-ruby!
+  replicate-ruby (~> 0.2.2)
   rspec (~> 3.0)
   ruby-openai (~> 4.0.0)
   standardrb

data/README.md

@@ -136,11 +136,17 @@ cohere = LLM::HuggingFace.new(api_key: ENV["HUGGING_FACE_API_KEY"])
 ```
 
 #### Replicate
-Add `gem "replicate-ruby", git: "https://github.com/andreibondarev/replicate-ruby.git", branch: "faraday-1.x"` to your Gemfile.
+Add `gem "replicate-ruby", "~> 0.2.2"` to your Gemfile.
 ```ruby
 cohere = LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
 ```
 
+#### Google PaLM (Pathways Language Model)
+Add `"google_palm_api", "~> 0.1.0"` to your Gemfile.
+```ruby
+google_palm = LLM::GooglePalm.new(api_key: ENV["GOOGLE_PALM_API_KEY"])
+```
+
 ### Using Prompts 📋
 
 #### Prompt Templates
@@ -264,7 +270,8 @@ Need to read data from various sources? Load it up.
 
 | Name | Class         | Gem Requirements             |
 | ---- | ------------- | :--------------------------: |
-| docx | Loaders::Docx | `gem "docx", branch: "master", git: "https://github.com/ruby-docx/docx.git"` |
+| docx | Loaders::Docx | `gem "docx", "~> 0.8.0"`     |
+| html | Loaders::HTML | `gem "nokogiri", "~> 1.13"`  |
 | pdf  | Loaders::PDF  | `gem "pdf-reader", "~> 1.4"` |
 | text | Loaders::Text |                              |
 
@@ -284,7 +291,7 @@ Langchain.logger.level = :info
 
 1. `git clone https://github.com/andreibondarev/langchainrb.git`
 2. `cp .env.example .env`, then fill out the environment variables in `.env`
-3. `rspec spec/` to ensure that the tests pass
+3. `bundle exec rake` to ensure that the tests pass and to run standardrb
 4. `bin/console` to load the gem in a REPL session. Feel free to add your own instances of LLMs, Tools, Agents, etc. and experiment with them.
 
 ## Core Contributors

data/lib/dependency_helper.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
+VersionError = Class.new(ScriptError)
+
 # This method requires and loads the given gem, and then checks to see if the version of the gem meets the requirements listed in `langchain.gemspec`
 # This solution was built to avoid auto-loading every single gem in the Gemfile when the developer will mostly likely be only using a few of them.
 #
 # @param gem_name [String] The name of the gem to load
 # @return [Boolean] Whether or not the gem was loaded successfully
 # @raise [LoadError] If the gem is not installed
-# @raise [LoadError] If the gem is installed, but the version does not meet the requirements
+# @raise [VersionError] If the gem is installed, but the version does not meet the requirements
 #
 def depends_on(gem_name)
   gem(gem_name) # require the gem
@@ -14,10 +16,12 @@ def depends_on(gem_name)
   return(true) unless defined?(Bundler) # If we're in a non-bundler environment, we're no longer able to determine if we'll meet requirements
 
   gem_version = Gem.loaded_specs[gem_name].version
-  gem_requirement = Bundler.load.dependencies.find { |g| g.name == gem_name }.requirement
+  gem_requirement = Bundler.load.dependencies.find { |g| g.name == gem_name }&.requirement
+
+  raise LoadError unless gem_requirement
 
-  if !gem_requirement.satisfied_by?(gem_version)
-    raise "The #{gem_name} gem is installed, but version #{gem_requirement} is required. You have #{gem_version}."
+  unless gem_requirement.satisfied_by?(gem_version)
+    raise VersionError, "The #{gem_name} gem is installed, but version #{gem_requirement} is required. You have #{gem_version}."
   end
 
   true

data/lib/langchain.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "logger"
+require "pathname"
 
 require_relative "./version"
 require_relative "./dependency_helper"
@@ -34,6 +35,7 @@ end
 module LLM
   autoload :Base, "llm/base"
   autoload :Cohere, "llm/cohere"
+  autoload :GooglePalm, "llm/google_palm"
   autoload :HuggingFace, "llm/hugging_face"
   autoload :OpenAI, "llm/openai"
   autoload :Replicate, "llm/replicate"
@@ -59,6 +61,7 @@ module Loaders
   autoload :Docx, "loaders/docx"
   autoload :PDF, "loaders/pdf"
   autoload :Text, "loaders/text"
+  autoload :HTML, "loaders/html"
 end
 
 autoload :Loader, "loader"

data/lib/llm/base.rb

@@ -7,9 +7,10 @@ module LLM
     # Currently supported LLMs
     # TODO: Add support for HuggingFace and other LLMs
     LLMS = {
-      openai: "OpenAI",
       cohere: "Cohere",
+      google_palm: "GooglePalm",
       huggingface: "HuggingFace",
+      openai: "OpenAI",
       replicate: "Replicate"
     }.freeze
 

data/lib/llm/google_palm.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module LLM
+  class GooglePalm < Base
+    # Wrapper around the Google PaLM (Pathways Language Model) APIs.
+
+    DEFAULTS = {
+      temperature: 0.0,
+      dimension: 768 # This is what the `embedding-gecko-001` model generates
+    }.freeze
+
+    def initialize(api_key:)
+      depends_on "google_palm_api"
+      require "google_palm_api"
+
+      @client = ::GooglePalmApi::Client.new(api_key: api_key)
+    end
+
+    #
+    # Generate an embedding for a given text
+    #
+    # @param text [String] The text to generate an embedding for
+    # @return [Array] The embedding
+    #
+    def embed(text:)
+      response = client.embed(
+        text: text
+      )
+      response.dig("embedding", "value")
+    end
+
+    #
+    # Generate a completion for a given prompt
+    #
+    # @param prompt [String] The prompt to generate a completion for
+    # @return [String] The completion
+    #
+    def complete(prompt:, **params)
+      default_params = {
+        prompt: prompt,
+        temperature: DEFAULTS[:temperature]
+      }
+
+      if params[:stop_sequences]
+        default_params[:stop_sequences] = params.delete(:stop_sequences)
+      end
+
+      if params[:max_tokens]
+        default_params[:max_output_tokens] = params.delete(:max_tokens)
+      end
+
+      default_params.merge!(params)
+
+      response = client.generate_text(**default_params)
+      response.dig("candidates", 0, "output")
+    end
+
+    #
+    # Generate a chat completion for a given prompt
+    #
+    # @param prompt [String] The prompt to generate a chat completion for
+    # @return [String] The chat completion
+    #
+    def chat(prompt:, **params)
+      # TODO: Figure out how to introduce persisted conversations
+      default_params = {
+        prompt: prompt,
+        temperature: DEFAULTS[:temperature]
+      }
+
+      if params[:stop_sequences]
+        default_params[:stop] = params.delete(:stop_sequences)
+      end
+
+      if params[:max_tokens]
+        default_params[:max_output_tokens] = params.delete(:max_tokens)
+      end
+
+      default_params.merge!(params)
+
+      response = client.generate_chat_message(**default_params)
+      response.dig("candidates", 0, "content")
+    end
+  end
+end

data/lib/loaders/html.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "open-uri"
+
+module Loaders
+  class HTML < Base
+    # We only look for headings and paragraphs
+    TEXT_CONTENT_TAGS = %w[h1 h2 h3 h4 h5 h6 p]
+
+    #
+    # This Loader parses URL into a text.
+    # If you'd like to use it directly you can do so like this:
+    # Loaders::URL.new("https://nokogiri.org/").load
+    #
+    def initialize(url)
+      depends_on "nokogiri"
+      require "nokogiri"
+
+      @url = url
+    end
+
+    # Check that url is a valid URL
+    def loadable?
+      !!(@url =~ URI::DEFAULT_PARSER.make_regexp)
+    end
+
+    def load
+      return unless response.status.first == "200"
+
+      doc = Nokogiri::HTML(response.read)
+      doc.css(TEXT_CONTENT_TAGS.join(",")).map(&:inner_text).join("\n\n")
+    end
+
+    def response
+      @response ||= URI.parse(@url).open
+    end
+  end
+end

data/lib/prompt/base.rb

@@ -66,6 +66,8 @@ module Prompt
     # contained within the template. Input variables are defined as text enclosed in
     # curly braces (e.g. "{variable_name}").
     #
+    # Content within two consecutive curly braces (e.g. "{{ignore_me}}) are ignored.
+    #
     # @param template [String] The template string to extract variables from.
     #
     # @return [Array<String>] An array of input variable names.
@@ -74,9 +76,9 @@ module Prompt
       input_variables = []
       scanner = StringScanner.new(template)
 
-      while scanner.scan_until(/\{([^{}]*)\}/)
+      while scanner.scan_until(/\{([^}]*)\}/)
         variable = scanner[1].strip
-        input_variables << variable unless variable.empty?
+        input_variables << variable unless variable.empty? || variable[0] == "{"
       end
 
       input_variables

data/lib/prompt/prompt_template.rb

@@ -20,7 +20,7 @@ module Prompt
     end
 
     #
-    # Format the prompt with the inputs.
+    # Format the prompt with the inputs. Double {{}} replaced with single {} to adhere to f-string spec.
     #
     # @param kwargs [Hash] Any arguments to be passed to the prompt template.
     # @return [String] A formatted string.
@@ -28,7 +28,7 @@ module Prompt
     def format(**kwargs)
       result = @template
       kwargs.each { |key, value| result = result.gsub(/\{#{key}\}/, value.to_s) }
-      result
+      result.gsub(/{{/, "{").gsub(/}}/, "}")
     end
 
     #

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.3.10"
+  VERSION = "0.3.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.3.11
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-21 00:00:00.000000000 Z
+date: 2023-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv-rails
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.5
+- !ruby/object:Gem::Dependency
+  name: google_palm_api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: google_search_results
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +150,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
 - !ruby/object:Gem::Dependency
   name: pdf-reader
   requirement: !ruby/object:Gem::Requirement
@@ -168,16 +196,16 @@ dependencies:
   name: replicate-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
   name: qdrant-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -262,12 +290,14 @@ files:
 - lib/langchain.rb
 - lib/llm/base.rb
 - lib/llm/cohere.rb
+- lib/llm/google_palm.rb
 - lib/llm/hugging_face.rb
 - lib/llm/openai.rb
 - lib/llm/replicate.rb
 - lib/loader.rb
 - lib/loaders/base.rb
 - lib/loaders/docx.rb
+- lib/loaders/html.rb
 - lib/loaders/pdf.rb
 - lib/loaders/text.rb
 - lib/prompt/base.rb