langchainrb 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 576c331bc4372bc1934a6a75fc496469242d0b645b56ffb9163cd9f812007414
-  data.tar.gz: 82a9ea6c734cab1490fbe8060732c41c4aba6e3d06d6895ea40e9f2f5db2f6ac
+  metadata.gz: 93a3fcc195fbdf55ec52402c1db2f11c929069c03afa90477259e6bf2f542957
+  data.tar.gz: 737e456d831e40e8c388a1986f2483f9dff3934c8b4e05a9456529e017075637
 SHA512:
-  metadata.gz: ce0325a59c2257e35c0be5e3e78ad44d046a058070585ba8127b97338c4280d4897f0dd070b16b9bea9cc0f5f7cd7f6b330611bb49f6180ecb0dafc55bb77d16
-  data.tar.gz: f38012ae7d0da8c70d76f37f2a24431e709fe0841e62812f1471a1b9a2b33763235c4eb246bb6f20d61e50e903c137156abb851a25b4d46e4b7fd85333f2dfe2
+  metadata.gz: 23619f8e256a9856eb113afce8eef94f759beb84644a87d9b67cac4fac9d5aedfc06978b3baedad1d988308b3cdb72d32e19a0f5e21d8b8431e8c9ff04eda548
+  data.tar.gz: 6cd0fcc55553a5472e2ac6c69a6e49dbd4b52fe8bcc899c04710ae40397c39947a3418649737c0d21a99086f85736d52f5968b0cc8af761f186eedf790ba85db

data/.env.example

@@ -0,0 +1,11 @@
+COHERE_API_KEY=
+HUGGING_FACE_API_KEY=
+MILVUS_URL=
+OPENAI_API_KEY=
+PINECONE_API_KEY=
+PINECONE_ENVIRONMENT=
+QDRANT_API_KEY=
+QDRANT_URL=
+SERPAPI_API_KEY=
+WEAVIATE_API_KEY=
+WEAVIATE_URL=

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ## [Unreleased]
 
+## [0.3.7] - 2023-05-19
+- Loaders
+  - Introduce `Loaders::Text` to parse .txt files
+  - Introduec `Loaders::PDF` to parse .pdf files
+
 ## [0.3.6] - 2023-05-17
 - LLMs
   - Bump `hugging-face` gem version

data/Gemfile.lock

@@ -1,11 +1,12 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.3.6)
+    langchainrb (0.3.7)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    Ascii85 (1.0.3)
     actionpack (7.0.4.3)
       actionview (= 7.0.4.3)
       activesupport (= 7.0.4.3)
@@ -26,6 +27,7 @@ GEM
       tzinfo (~> 2.0)
     addressable (2.8.4)
       public_suffix (>= 2.0.2, < 6.0)
+    afm (0.2.2)
     ast (2.4.2)
     builder (3.2.4)
     byebug (11.1.3)
@@ -114,6 +116,7 @@ GEM
     graphql-client (0.18.0)
       activesupport (>= 3.0)
       graphql
+    hashery (2.1.2)
     httparty (0.21.0)
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
@@ -144,6 +147,12 @@ GEM
     parallel (1.23.0)
     parser (3.2.2.1)
       ast (~> 2.4.1)
+    pdf-reader (1.4.1)
+      Ascii85 (~> 1.0.0)
+      afm (~> 0.2.1)
+      hashery (~> 2.0)
+      ruby-rc4
+      ttfunk
     pinecone (0.1.71)
       dry-struct (~> 1.6.0)
       dry-validation (~> 1.10.0)
@@ -211,6 +220,7 @@ GEM
       faraday (>= 1)
       faraday-multipart (>= 1)
     ruby-progressbar (1.13.0)
+    ruby-rc4 (0.1.5)
     ruby2_keywords (0.0.5)
     standard (1.28.2)
       language_server-protocol (~> 3.17.0.2)
@@ -228,6 +238,7 @@ GEM
     thor (1.2.1)
     treetop (1.6.12)
       polyglot (~> 0.3)
+    ttfunk (1.7.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.4.2)
@@ -252,6 +263,7 @@ DEPENDENCIES
   hugging-face (~> 0.3.3)
   langchainrb!
   milvus (~> 0.9.0)
+  pdf-reader (~> 1.4)
   pinecone (~> 0.1.6)
   pry-byebug (~> 3.10.0)
   qdrant-ruby (~> 0.9.0)

data/README.md

@@ -62,7 +62,7 @@ client.create_default_schema
 ```
 
 ```ruby
-# Store your documents in your vector search database
+# Store plain texts in your vector search database
 client.add_texts(
     texts: [
         "Begin by preheating your oven to 375°F (190°C). Prepare four boneless, skinless chicken breasts by cutting a pocket into the side of each breast, being careful not to cut all the way through. Season the chicken with salt and pepper to taste. In a large skillet, melt 2 tablespoons of unsalted butter over medium heat. Add 1 small diced onion and 2 minced garlic cloves, and cook until softened, about 3-4 minutes. Add 8 ounces of fresh spinach and cook until wilted, about 3 minutes. Remove the skillet from heat and let the mixture cool slightly.",
@@ -70,7 +70,13 @@ client.add_texts(
     ]
 )
 ```
+```ruby
+# Store the contents of your files in your vector search database
+my_pdf = Langchain.root.join("path/to/my.pdf")
+my_text = Langchain.root.join("path/to/my.txt")
 
+client.add_data(paths: [my_pdf, my_text])
+```
 ```ruby
 # Retrieve similar documents based on the query string passed in
 client.similarity_search(
@@ -233,11 +239,21 @@ agent.run(question: "How many full soccer fields would be needed to cover the di
 
 #### Available Tools 🛠️
 
-| Name     | Description          | Requirements         |
-| -------- | :------------------: | :------------------: |
-| "calculator" | Useful for getting the result of a math expression | |
-| "search" | A wrapper around Google Search | `ENV["SERPAPI_API_KEY"]` (https://serpapi.com/manage-api-key)
-| "wikipedia" | Calls Wikipedia API to retrieve the summary | |
+| Name         | Description                                        | ENV Requirements                                              | Gem Requirements                          |
+| ------------ | :------------------------------------------------: | :-----------------------------------------------------------: | :---------------------------------------: |
+| "calculator" | Useful for getting the result of a math expression |                                                               | `gem "eqn", "~> 1.6.5"`                   |
+| "search"     | A wrapper around Google Search                     | `ENV["SERPAPI_API_KEY"]` (https://serpapi.com/manage-api-key) | `gem "google_search_results", "~> 2.0.0"` |                      |
+| "wikipedia"  | Calls Wikipedia API to retrieve the summary        |                                                               | `gem "wikipedia-client", "~> 1.17.0"`     |
+
+
+#### Loaders 🚚
+
+Need to read data from various sources? Load it up.
+
+| Name | Class         | Gem Requirements             |
+| ---- | ------------- | :--------------------------: |
+| pdf  | Loaders::PDF  | `gem "pdf-reader", "~> 1.4"` |
+| text | Loaders::Text |                              |
 
 
 ## Logging
@@ -251,9 +267,10 @@ Langchain.logger.level = :info
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
-
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+1. `git clone https://github.com/andreibondarev/langchainrb.git`
+2. `cp .env.example .env`, then fill out the environment variables in `.env`
+3. `rspec spec/` to ensure that the tests pass
+4. `bin/console` to load the gem in a REPL session. Feel free to add your own instances of LLMs, Tools, Agents, etc. and experiment with them.
 
 ## Core Contributors
 [<img style="border-radius:50%" alt="Andrei Bondarev" src="https://avatars.githubusercontent.com/u/541665?v=4" width="80" height="80" class="avatar">](https://github.com/andreibondarev)
@@ -261,8 +278,9 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 ## Honorary Contributors
 [<img style="border-radius:50%" alt="Andrei Bondarev" src="https://avatars.githubusercontent.com/u/541665?v=4" width="80" height="80" class="avatar">](https://github.com/andreibondarev)
 [<img style="border-radius:50%" alt="Rafael Figueiredo" src="https://avatars.githubusercontent.com/u/35845775?v=4" width="80" height="80" class="avatar">](https://github.com/rafaelqfigueiredo)
+[<img style="border-radius:50%" alt="Ricky Chilcott" src="https://avatars.githubusercontent.com/u/445759?v=4" width="80" height="80" class="avatar">](https://github.com/rickychilcott)
 
-(Criteria of becoming an Honorary Contributor or Core Contributor is pending...)
+(Criteria for becoming an Honorary Contributor or Core Contributor is pending...)
 
 ## Contributing
 

data/lib/agent/chain_of_thought_agent/chain_of_thought_agent.rb

@@ -43,7 +43,7 @@ module Agent
 
       loop do
         Langchain.logger.info("Agent: Passing the prompt to the #{llm} LLM")
-        response = llm_client.generate_completion(
+        response = llm_client.complete(
           prompt: prompt,
           stop_sequences: ["Observation:"],
           max_tokens: 500
@@ -100,7 +100,7 @@ module Agent
     # @return [PromptTemplate] PromptTemplate instance
     def prompt_template
       @template ||= Prompt.load_from_path(
-        file_path: Pathname.new(__dir__).join("chain_of_thought_agent_prompt.json")
+        file_path: Langchain.root.join("agent/chain_of_thought_agent/chain_of_thought_agent_prompt.json")
       )
     end
   end

data/lib/dependency_helper.rb

@@ -1,5 +1,13 @@
 # frozen_string_literal: true
 
+# This method requires and loads the given gem, and then checks to see if the version of the gem meets the requirements listed in `langchain.gemspec`
+# This solution was built to avoid auto-loading every single gem in the Gemfile when the developer will mostly likely be only using a few of them.
+#
+# @param gem_name [String] The name of the gem to load
+# @return [Boolean] Whether or not the gem was loaded successfully
+# @raise [LoadError] If the gem is not installed
+# @raise [LoadError] If the gem is installed, but the version does not meet the requirements
+#
 def depends_on(gem_name)
   gem(gem_name) # require the gem
 

data/lib/langchain.rb

@@ -1,8 +1,21 @@
 # frozen_string_literal: true
 
+require "logger"
+
 require_relative "./version"
 require_relative "./dependency_helper"
-require_relative "./logging"
+module Langchain
+  class << self
+    attr_accessor :default_loaders
+    attr_accessor :logger
+
+    attr_reader :root
+  end
+
+  @logger ||= ::Logger.new($stdout, level: :warn, formatter: ->(severity, datetime, progname, msg) { "[LangChain.rb] #{msg}\n" })
+
+  @root = Pathname.new(__dir__)
+end
 
 module Agent
   autoload :Base, "agent/base"
@@ -38,3 +51,14 @@ module Tool
   autoload :SerpApi, "tool/serp_api"
   autoload :Wikipedia, "tool/wikipedia"
 end
+
+module Loaders
+  autoload :Base, "loaders/base"
+  autoload :PDF, "loaders/pdf"
+  autoload :Text, "loaders/text"
+end
+
+autoload :Loader, "loader"
+
+# Load the default Loaders
+Langchain.default_loaders ||= [::Loaders::Text, ::Loaders::PDF]

data/lib/llm/base.rb

@@ -16,6 +16,21 @@ module LLM
       self.class.const_get(:DEFAULTS).dig(:dimension)
     end
 
+    # Method supported by an LLM that generates a response for a given chat-style prompt
+    def chat(...)
+      raise NotImplementedError, "#{self.class.name} does not support chat"
+    end
+
+    # Method supported by an LLM that completes a given prompt
+    def complete(...)
+      raise NotImplementedError, "#{self.class.name} does not support completion"
+    end
+
+    # Method supported by an LLM that generates an embedding for a given text or array of texts
+    def embed(...)
+      raise NotImplementedError, "#{self.class.name} does not support generating embeddings"
+    end
+
     # Ensure that the LLM value passed in is supported
     # @param llm [Symbol] The LLM to use
     def self.validate_llm!(llm:)

data/lib/llm/cohere.rb

@@ -47,7 +47,11 @@ module LLM
       response.dig("generations").first.dig("text")
     end
 
-    alias_method :generate_completion, :complete
+    # Cohere does not have a dedicated chat endpoint, so instead we call `complete()`
+    def chat(...)
+      complete(...)
+    end
+
     alias_method :generate_embedding, :embed
   end
 end

data/lib/llm/hugging_face.rb

@@ -25,7 +25,7 @@ module LLM
     # @param text [String] The text to embed
     # @return [Array] The embedding
     def embed(text:)
-      response = client.embedding(
+      client.embedding(
         input: text,
         model: DEFAULTS[:embeddings_model_name]
       )

data/lib/llm/openai.rb

@@ -5,6 +5,7 @@ module LLM
     DEFAULTS = {
       temperature: 0.0,
       completion_model_name: "text-davinci-003",
+      chat_completion_model_name: "gpt-3.5-turbo",
       embeddings_model_name: "text-embedding-ada-002",
       dimension: 1536
     }.freeze
@@ -50,7 +51,27 @@ module LLM
       response.dig("choices", 0, "text")
     end
 
-    alias_method :generate_completion, :complete
+    # Generate a chat completion for a given prompt
+    # @param prompt [String] The prompt to generate a chat completion for
+    # @return [String] The chat completion
+    def chat(prompt:, **params)
+      default_params = {
+        model: DEFAULTS[:chat_completion_model_name],
+        temperature: DEFAULTS[:temperature],
+        # TODO: Figure out how to introduce persisted conversations
+        messages: [{role: "user", content: prompt}]
+      }
+
+      if params[:stop_sequences]
+        default_params[:stop] = params.delete(:stop_sequences)
+      end
+
+      default_params.merge!(params)
+
+      response = client.chat(parameters: default_params)
+      response.dig("choices", 0, "message", "content")
+    end
+
     alias_method :generate_embedding, :embed
   end
 end

data/lib/loader.rb

@@ -0,0 +1,26 @@
+module Loader
+  def self.with(*loaders)
+    LoaderSet.new(loaders)
+  end
+
+  class LoaderSet
+    def initialize(loaders)
+      @loaders = Array(loaders)
+    end
+
+    def load(*paths)
+      Array(paths)
+        .flatten
+        .map { |path| first_loadable_loader(path)&.load }
+        .compact
+    end
+
+    def first_loadable_loader(path)
+      @loaders
+        .each do |loader_klass|
+          loader_instance = loader_klass.new(path)
+          return(loader_instance) if loader_instance.loadable?
+        end
+    end
+  end
+end

data/lib/loaders/base.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Loaders
+  class Base
+    def self.load(path)
+      new.load(path)
+    end
+
+    def initialize(path)
+      @path = path
+    end
+
+    def loadable?
+      raise NotImplementedError
+    end
+  end
+end

data/lib/loaders/pdf.rb

@@ -0,0 +1,34 @@
+module Loaders
+  class PDF < Base
+    #
+    # This Loader parses PDF files into text.
+    # If you'd like to use it directly you can do so like this:
+    # Loaders::PDF.new("path/to/my.pdf").load
+    #
+    # This parser is also invoked when you're adding data to a Vectorsearch DB:
+    # qdrant = Vectorsearch::Qdrant.new(...)
+    # path = Langchain.root.join("path/to/my.pdf")
+    # qdrant.add_data(path: path)
+    #
+
+    def initialize(path)
+      depends_on "pdf-reader"
+      require "pdf-reader"
+
+      @path = path
+    end
+
+    # Check that the file is a PDF file
+    def loadable?
+      @path.to_s.end_with?(".pdf")
+    end
+
+    def load
+      ::PDF::Reader
+        .new(@path)
+        .pages
+        .map(&:text)
+        .join("\n\n")
+    end
+  end
+end

data/lib/loaders/text.rb

@@ -0,0 +1,22 @@
+module Loaders
+  class Text < Base
+    #
+    # This Loader parses .txt files.
+    # If you'd like to use it directly you can do so like this:
+    # Loaders::Text.new("path/to/my.txt").load
+    #
+    # This parser is also invoked when you're adding data to a Vectorsearch DB:
+    # qdrant = Vectorsearch::Qdrant.new(...)
+    # path = Langchain.root.join("path/to/my.txt")
+    # qdrant.add_data(path: path)
+    #
+
+    def loadable?
+      true
+    end
+
+    def load
+      @path.read
+    end
+  end
+end

data/lib/tool/base.rb

@@ -12,8 +12,7 @@ module Tool
     TOOLS = {
       "calculator" => "Tool::Calculator",
       "search" => "Tool::SerpApi",
-      "wikipedia" => "Tool::Wikipedia",
-      "news" => "Tool::News"
+      "wikipedia" => "Tool::Wikipedia"
     }
 
     def self.description(value)

data/lib/vectorsearch/base.rb

@@ -19,24 +19,38 @@ module Vectorsearch
       @llm_api_key = llm_api_key
 
       @llm_client = LLM.const_get(LLM::Base::LLMS.fetch(llm)).new(api_key: llm_api_key)
+
+      @loaders = Langchain.default_loaders
     end
 
+    # Method supported by Vectorsearch DB to create a default schema
     def create_default_schema
-      raise NotImplementedError
+      raise NotImplementedError, "#{self.class.name} does not support creating a default schema"
+    end
+
+    # Method supported by Vectorsearch DB to add a list of texts to the index
+    def add_texts(...)
+      raise NotImplementedError, "#{self.class.name} does not support adding texts"
     end
 
-    def add_texts(texts:)
-      raise NotImplementedError
+    # Method supported by Vectorsearch DB to search for similar texts in the index
+    def similarity_search(...)
+      raise NotImplementedError, "#{self.class.name} does not support similarity search"
     end
 
-    # NotImplementedError will be raised if the subclass does not implement this method
-    def ask(question:)
-      raise NotImplementedError
+    # Method supported by Vectorsearch DB to search for similar texts in the index by the passed in vector.
+    # You must generate your own vector using the same LLM that generated the embeddings stored in the Vectorsearch DB.
+    def similarity_search_by_vector(...)
+      raise NotImplementedError, "#{self.class.name} does not support similarity search by vector"
+    end
+
+    # Method supported by Vectorsearch DB to answer a question given a context (data) pulled from your Vectorsearch DB.
+    def ask(...)
+      raise NotImplementedError, "#{self.class.name} does not support asking questions"
     end
 
     def_delegators :llm_client,
       :generate_embedding,
-      :generate_completion,
       :default_dimension
 
     def generate_prompt(question:, context:)
@@ -56,5 +70,23 @@ module Vectorsearch
 
       prompt_template.format(question: question)
     end
+
+    def add_data(path: nil, paths: nil)
+      raise ArgumentError, "Either path or paths must be provided" if path.nil? && paths.nil?
+      raise ArgumentError, "Either path or paths must be provided, not both" if !path.nil? && !paths.nil?
+
+      texts =
+        Loader
+          .with(*loaders)
+          .load(path || paths)
+
+      add_texts(texts: texts)
+    end
+
+    attr_reader :loaders
+
+    def add_loader(*loaders)
+      loaders.each { |loader| @loaders << loader }
+    end
   end
 end

data/lib/vectorsearch/milvus.rb

@@ -88,9 +88,5 @@ module Vectorsearch
         metric_type: "L2"
       )
     end
-
-    def ask(question:)
-      raise NotImplementedError
-    end
   end
 end

data/lib/vectorsearch/pinecone.rb

@@ -96,7 +96,7 @@ module Vectorsearch
 
       prompt = generate_prompt(question: question, context: context)
 
-      generate_completion(prompt: prompt)
+      llm_client.chat(prompt: prompt)
     end
   end
 end

data/lib/vectorsearch/qdrant.rb

@@ -96,7 +96,7 @@ module Vectorsearch
 
       prompt = generate_prompt(question: question, context: context)
 
-      generate_completion(prompt: prompt)
+      llm_client.chat(prompt: prompt)
     end
   end
 end

data/lib/vectorsearch/weaviate.rb

@@ -113,7 +113,7 @@ module Vectorsearch
 
         prompt = generate_prompt(question: question, context: context)
 
-        generate_completion(prompt: prompt)
+        llm_client.chat(prompt: prompt)
       end
     end
   end

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.3.6"
+  VERSION = "0.3.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-17 00:00:00.000000000 Z
+date: 2023-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv-rails
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: pdf-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: pinecone
   requirement: !ruby/object:Gem::Requirement
@@ -185,6 +199,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".env.example"
 - ".rspec"
 - CHANGELOG.md
 - Gemfile
@@ -207,7 +222,10 @@ files:
 - lib/llm/cohere.rb
 - lib/llm/hugging_face.rb
 - lib/llm/openai.rb
-- lib/logging.rb
+- lib/loader.rb
+- lib/loaders/base.rb
+- lib/loaders/pdf.rb
+- lib/loaders/text.rb
 - lib/prompt/base.rb
 - lib/prompt/few_shot_prompt_template.rb
 - lib/prompt/loading.rb

data/lib/logging.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-require "logger"
-
-module Langchain
-  def self.logger
-    @@logger ||= Logger.new($stdout, level: :warn, formatter: ->(severity, datetime, progname, msg) { "[LangChain.rb] #{msg}\n" })
-  end
-
-  def self.logger=(instance)
-    @@logger = instance
-  end
-end