langchainrb 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d36de4206b792714ba9b6773c03272e9638b14caf7140e0bc00c3e767aa5fdef
-  data.tar.gz: 819fab9de55a34e4e6dc865febc19bb9979df55fa8fc6a753774cf1961c40103
+  metadata.gz: 9ff52d2013c31933c6c12e3b824db979122decf0f7a4be2a99ec22da22ecb4f3
+  data.tar.gz: f632e80bd634e1ff899216498a13e8dc24073d435754077612f8753f51c9fbcf
 SHA512:
-  metadata.gz: 6e180b41bbca96bd5523c276923f223bbebe470314086c6a909df440890793bcc70dbd66ecf59bf5d0fd52426650cc5d2684c56cc8fc643209cc1679527cbef4
-  data.tar.gz: af5db76c2b22b5c7bdc1170de437921e8464a16566f46a5cad465d69e6da47c97a82f7331a5ea5747840e58acc71463aa8456b03e9bc8851efda7b734e5d23cc
+  metadata.gz: e79f3c438f18b90d6f926088a613dfafdf7009e3c9d768adcb4ba097f688731073af1a983ddbeb754e479d82ca34492c085a05d9bb6d2ddcd4996a5be418d03c
+  data.tar.gz: 32682afc3ce3d3ce63d351ff10695d97f46f19b725f975ed6901b4970f6254a95c50fc3575bfc3a3003c95c0cd5a955e1ccdbb17e062cbcf65c1fa9f541a286a

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [0.5.3] - 2023-06-09
+- 🗣️ LLMs
+  - Chat message history support for Langchain::LLM::GooglePalm and Langchain::LLM::OpenAI
+
 ## [0.5.2] - 2023-06-07
 - 🗣️ LLMs
   - Auto-calculate the max_tokens: setting to be passed on to OpenAI

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.5.2)
+    langchainrb (0.5.3)
       colorize (~> 0.8.1)
       tiktoken_ruby (~> 0.0.5)
 
@@ -122,7 +122,7 @@ GEM
     faraday-retry (1.0.3)
     faraday_middleware (1.2.0)
       faraday (~> 1.0)
-    google_palm_api (0.1.0)
+    google_palm_api (0.1.1)
       faraday (>= 1.0.0)
       faraday_middleware (>= 1.0.0)
     google_search_results (2.0.1)
@@ -310,7 +310,7 @@ DEPENDENCIES
   docx (~> 0.8.0)
   dotenv-rails (~> 2.7.6)
   eqn (~> 1.6.5)
-  google_palm_api (~> 0.1.0)
+  google_palm_api (~> 0.1.1)
   google_search_results (~> 2.0.0)
   hugging-face (~> 0.3.4)
   langchainrb!

data/README.md

@@ -1,4 +1,4 @@
-🦜️🔗 LangChain.rb
+💎🔗 LangChain.rb
 ---
 ⚡ Building applications with LLMs through composability ⚡
 
@@ -6,7 +6,10 @@
 
 :warning: UNDER ACTIVE AND RAPID DEVELOPMENT (MAY BE BUGGY AND UNTESTED)
 
-![Tests status](https://github.com/andreibondarev/langchainrb/actions/workflows/ci.yml/badge.svg) [![Gem Version](https://badge.fury.io/rb/langchainrb.svg)](https://badge.fury.io/rb/langchainrb)
+![Tests status](https://github.com/andreibondarev/langchainrb/actions/workflows/ci.yml/badge.svg)
+[![Gem Version](https://badge.fury.io/rb/langchainrb.svg)](https://badge.fury.io/rb/langchainrb)
+[![Docs](http://img.shields.io/badge/yard-docs-blue.svg)](http://rubydoc.info/gems/langchainrb)
+[![License](https://img.shields.io/badge/license-MIT-green.svg)](https://github.com/andreibondarev/langchainrb/blob/main/LICENSE.txt)
 
 Langchain.rb is a library that's an abstraction layer on top many emergent AI, ML and other DS tools. The goal is to abstract complexity and difficult concepts to make building AI/ML-supercharged applications approachable for traditional software engineers.
 
@@ -33,6 +36,7 @@ require "langchain"
 | [Chroma](https://trychroma.com/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Milvus](https://milvus.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Pinecone](https://www.pinecone.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Pgvector](https://github.com/pgvector/pgvector) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Qdrant](https://qdrant.tech/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Weaviate](https://weaviate.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 
@@ -47,6 +51,7 @@ Pick the vector search database you'll be using and instantiate the client:
 client = Langchain::Vectorsearch::Weaviate.new(
     url: ENV["WEAVIATE_URL"],
     api_key: ENV["WEAVIATE_API_KEY"],
+    index: "",
     llm: Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
 )
 
@@ -55,6 +60,7 @@ client = Langchain::Vectorsearch::Milvus.new(...) # `gem "milvus", "~> 0.9.0"`
 client = Langchain::Vectorsearch::Qdrant.new(...) # `gem"qdrant-ruby", "~> 0.9.0"`
 client = Langchain::Vectorsearch::Pinecone.new(...) # `gem "pinecone", "~> 0.1.6"`
 client = Langchain::Vectorsearch::Chroma.new(...) # `gem "chroma-db", "~> 0.3.0"`
+client = Langchain::Vectorsearch::Pgvector.new(...) # `gem "pgvector", "~> 0.2"`
 ```
 
 ```ruby
@@ -135,17 +141,17 @@ cohere.complete(prompt: "What is the meaning of life?")
 #### HuggingFace
 Add `gem "hugging-face", "~> 0.3.2"` to your Gemfile.
 ```ruby
-cohere = Langchain::LLM::HuggingFace.new(api_key: ENV["HUGGING_FACE_API_KEY"])
+hugging_face = Langchain::LLM::HuggingFace.new(api_key: ENV["HUGGING_FACE_API_KEY"])
 ```
 
 #### Replicate
 Add `gem "replicate-ruby", "~> 0.2.2"` to your Gemfile.
 ```ruby
-cohere = Langchain::LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
+replicate = Langchain::LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
 ```
 
 #### Google PaLM (Pathways Language Model)
-Add `"google_palm_api", "~> 0.1.0"` to your Gemfile.
+Add `"google_palm_api", "~> 0.1.1"` to your Gemfile.
 ```ruby
 google_palm = Langchain::LLM::GooglePalm.new(api_key: ENV["GOOGLE_PALM_API_KEY"])
 ```

data/Rakefile

@@ -14,5 +14,4 @@ Rake::Task["spec"].enhance do
 end
 
 YARD::Rake::YardocTask.new do |t|
-  t.options = ["--fail-on-warning"]
 end

data/lib/langchain/agent/base.rb

@@ -1,6 +1,14 @@
 # frozen_string_literal: true
 
 module Langchain::Agent
+  # = Agents
+  #
+  # Agents are semi-autonomous bots that can respond to user questions and use available to them Tools to provide informed replies. They break down problems into series of steps and define Actions (and Action Inputs) along the way that are executed and fed back to them as additional information. Once an Agent decides that it has the Final Answer it responds with it.
+  #
+  # Available:
+  # - {Langchain::Agent::ChainOfThoughtAgent}
+  #
+  # @abstract
   class Base
   end
 end

data/lib/langchain/agent/chain_of_thought_agent/chain_of_thought_agent.rb

@@ -1,6 +1,20 @@
 # frozen_string_literal: true
 
 module Langchain::Agent
+  # = Chain of Thought Agent
+  #
+  #     llm = Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"]) # or your choice of Langchain::LLM::Base implementation
+  #
+  #     agent = Langchain::Agent::ChainOfThoughtAgent.new(
+  #       llm: llm,
+  #       tools: ["search", "calculator", "wikipedia"]
+  #     )
+  #
+  #     agent.tools
+  #     # => ["search", "calculator", "wikipedia"]
+  #
+  #     agent.run(question: "How many full soccer fields would be needed to cover the distance between NYC and DC in a straight line?")
+  #     #=> "Approximately 2,945 soccer fields would be needed to cover the distance between NYC and DC in a straight line."
   class ChainOfThoughtAgent < Base
     attr_reader :llm, :tools
 

data/lib/langchain/llm/ai21.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  #
+  # Wrapper around AI21 Studio APIs.
+  #
+  # Gem requirements:
+  #   gem "ai21", "~> 0.2.0"
+  #
+  # Usage:
+  #     ai21 = Langchain::LLM::AI21.new(api_key:)
+  #
   class AI21 < Base
-    #
-    # Wrapper around AI21 Studio APIs.
-    #
-    # Gem requirements: gem "ai21", "~> 0.2.0"
-    #
-    # Usage:
-    # ai21 = Langchain::LLM::AI21.new(api_key:)
-    #
-
     def initialize(api_key:)
       depends_on "ai21"
       require "ai21"

data/lib/langchain/llm/base.rb

@@ -1,31 +1,58 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  # A LLM is a language model consisting of a neural network with many parameters (typically billions of weights or more), trained on large quantities of unlabeled text using self-supervised learning or semi-supervised learning.
+  #
+  # Langchain.rb provides a common interface to interact with all supported LLMs:
+  #
+  # - {Langchain::LLM::AI21}
+  # - {Langchain::LLM::Cohere}
+  # - {Langchain::LLM::GooglePalm}
+  # - {Langchain::LLM::HuggingFace}
+  # - {Langchain::LLM::OpenAI}
+  # - {Langchain::LLM::Replicate}
+  #
+  # @abstract
   class Base
     include Langchain::DependencyHelper
 
+    # A client for communicating with the LLM
     attr_reader :client
 
     def default_dimension
       self.class.const_get(:DEFAULTS).dig(:dimension)
     end
 
-    # Method supported by an LLM that generates a response for a given chat-style prompt
+    #
+    # Generate a chat completion for a given prompt. Parameters will depend on the LLM
+    #
+    # @raise NotImplementedError if not supported by the LLM
     def chat(...)
       raise NotImplementedError, "#{self.class.name} does not support chat"
     end
 
-    # Method supported by an LLM that completes a given prompt
+    #
+    # Generate a completion for a given prompt. Parameters will depend on the LLM.
+    #
+    # @raise NotImplementedError if not supported by the LLM
     def complete(...)
       raise NotImplementedError, "#{self.class.name} does not support completion"
     end
 
-    # Method supported by an LLM that generates an embedding for a given text or array of texts
+    #
+    # Generate an embedding for a given text. Parameters depends on the LLM.
+    #
+    # @raise NotImplementedError if not supported by the LLM
+    #
     def embed(...)
       raise NotImplementedError, "#{self.class.name} does not support generating embeddings"
     end
 
-    # Method supported by an LLM that summarizes a given text
+    #
+    # Generate a summary for a given text. Parameters depends on the LLM.
+    #
+    # @raise NotImplementedError if not supported by the LLM
+    #
     def summarize(...)
       raise NotImplementedError, "#{self.class.name} does not support summarization"
     end

data/lib/langchain/llm/cohere.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  #
+  # Wrapper around the Cohere API.
+  #
+  # Gem requirements:
+  #     gem "cohere-ruby", "~> 0.9.4"
+  #
+  # Usage:
+  #     cohere = Langchain::LLM::Cohere.new(api_key: "YOUR_API_KEY")
+  #
   class Cohere < Base
-    #
-    # Wrapper around the Cohere API.
-    #
-    # Gem requirements: gem "cohere-ruby", "~> 0.9.4"
-    #
-    # Usage:
-    # cohere = Langchain::LLM::Cohere.new(api_key: "YOUR_API_KEY")
-    #
-
     DEFAULTS = {
       temperature: 0.0,
       completion_model_name: "base",
@@ -43,6 +43,7 @@ module Langchain::LLM
     # Generate a completion for a given prompt
     #
     # @param prompt [String] The prompt to generate a completion for
+    # @param params[:stop_sequences]
     # @return [Hash] The completion
     #
     def complete(prompt:, **params)

data/lib/langchain/llm/google_palm.rb

@@ -1,11 +1,20 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  #
+  # Wrapper around the Google PaLM (Pathways Language Model) APIs: https://ai.google/build/machine-learning/
+  #
+  # Gem requirements:
+  #     gem "google_palm_api", "~> 0.1.0"
+  #
+  # Usage:
+  #     google_palm = Langchain::LLM::GooglePalm.new(api_key: "YOUR_API_KEY")
+  #
   class GooglePalm < Base
     #
     # Wrapper around the Google PaLM (Pathways Language Model) APIs.
     #
-    # Gem requirements: gem "google_palm_api", "~> 0.1.0"
+    # Gem requirements: gem "google_palm_api", "~> 0.1.1"
     #
     # Usage:
     # google_palm = Langchain::LLM::GooglePalm.new(api_key: "YOUR_API_KEY")
@@ -40,6 +49,7 @@ module Langchain::LLM
     # Generate a completion for a given prompt
     #
     # @param prompt [String] The prompt to generate a completion for
+    # @param params extra parameters passed to GooglePalmAPI::Client#generate_text
     # @return [String] The completion
     #
     def complete(prompt:, **params)
@@ -66,13 +76,19 @@ module Langchain::LLM
     # Generate a chat completion for a given prompt
     #
     # @param prompt [String] The prompt to generate a chat completion for
+    # @param messages [Array] The messages that have been sent in the conversation
+    # @param params extra parameters passed to GooglePalmAPI::Client#generate_chat_message
     # @return [String] The chat completion
     #
-    def chat(prompt:, **params)
+    def chat(prompt: "", messages: [], **params)
+      raise ArgumentError.new(":prompt or :messages argument is expected") if prompt.empty? && messages.empty?
+
+      messages << {author: "0", content: prompt} if !prompt.empty?
+
       # TODO: Figure out how to introduce persisted conversations
       default_params = {
-        prompt: prompt,
-        temperature: DEFAULTS[:temperature]
+        temperature: DEFAULTS[:temperature],
+        messages: messages
       }
 
       if params[:stop_sequences]

data/lib/langchain/llm/hugging_face.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  #
+  # Wrapper around the HuggingFace Inference API: https://huggingface.co/inference-api
+  #
+  # Gem requirements:
+  #     gem "hugging-face", "~> 0.3.4"
+  #
+  # Usage:
+  #     hf = Langchain::LLM::HuggingFace.new(api_key: "YOUR_API_KEY")
+  #
   class HuggingFace < Base
-    #
-    # Wrapper around the HuggingFace Inference API.
-    #
-    # Gem requirements: gem "hugging-face", "~> 0.3.4"
-    #
-    # Usage:
-    # hf = Langchain::LLM::HuggingFace.new(api_key: "YOUR_API_KEY")
-    #
-
     # The gem does not currently accept other models:
     # https://github.com/alchaplinsky/hugging-face/blob/main/lib/hugging_face/inference_api.rb#L32-L34
     DEFAULTS = {

data/lib/langchain/llm/openai.rb

@@ -1,16 +1,15 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  # LLM interface for OpenAI APIs: https://platform.openai.com/overview
+  #
+  # Gem requirements:
+  #    gem "ruby-openai", "~> 4.0.0"
+  #
+  # Usage:
+  #    openai = Langchain::LLM::OpenAI.new(api_key:, llm_options: {})
+  #
   class OpenAI < Base
-    #
-    # Wrapper around OpenAI APIs.
-    #
-    # Gem requirements: gem "ruby-openai", "~> 4.0.0"
-    #
-    # Usage:
-    # openai = Langchain::LLM::OpenAI.new(api_key:, llm_options: {})
-    #
-
     DEFAULTS = {
       temperature: 0.0,
       completion_model_name: "text-davinci-003",
@@ -30,6 +29,7 @@ module Langchain::LLM
     # Generate an embedding for a given text
     #
     # @param text [String] The text to generate an embedding for
+    # @param params extra parameters passed to OpenAI::Client#embeddings
     # @return [Array] The embedding
     #
     def embed(text:, **params)
@@ -45,6 +45,7 @@ module Langchain::LLM
     # Generate a completion for a given prompt
     #
     # @param prompt [String] The prompt to generate a completion for
+    # @param params  extra parameters passed to OpenAI::Client#complete
     # @return [String] The completion
     #
     def complete(prompt:, **params)
@@ -61,13 +62,18 @@ module Langchain::LLM
     # Generate a chat completion for a given prompt
     #
     # @param prompt [String] The prompt to generate a chat completion for
+    # @param messages [Array] The messages that have been sent in the conversation
+    # @param params extra parameters passed to OpenAI::Client#chat
     # @return [String] The chat completion
     #
-    def chat(prompt:, **params)
-      parameters = compose_parameters DEFAULTS[:chat_completion_model_name], params
+    def chat(prompt: "", messages: [], **params)
+      raise ArgumentError.new(":prompt or :messages argument is expected") if prompt.empty? && messages.empty?
 
-      parameters[:messages] = [{role: "user", content: prompt}]
-      parameters[:max_tokens] = Langchain::Utils::TokenLengthValidator.validate_max_tokens!(prompt, parameters[:model])
+      messages << {role: "user", content: prompt} if !prompt.empty?
+
+      parameters = compose_parameters DEFAULTS[:chat_completion_model_name], params
+      parameters[:messages] = messages
+      parameters[:max_tokens] = validate_max_tokens(messages, parameters[:model])
 
       response = client.chat(parameters: parameters)
       response.dig("choices", 0, "message", "content")
@@ -97,5 +103,9 @@ module Langchain::LLM
 
       default_params.merge(params)
     end
+
+    def validate_max_tokens(messages, model)
+      Langchain::Utils::TokenLengthValidator.validate_max_tokens!(messages, model)
+    end
   end
 end

data/lib/langchain/llm/replicate.rb

@@ -1,22 +1,23 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  #
+  # Wrapper around Replicate.com LLM provider
+  #
+  # Gem requirements:
+  #     gem "replicate-ruby", "~> 0.2.2"
+  #
+  # Use it directly:
+  #     replicate = Langchain::LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
+  #
+  # Or pass it to be used by a vector search DB:
+  #     chroma = Langchain::Vectorsearch::Chroma.new(
+  #       url: ENV["CHROMA_URL"],
+  #       index_name: "...",
+  #       llm: replicate
+  #     )
+  #
   class Replicate < Base
-    #
-    # Wrapper around Replicate.com LLM provider
-    #
-    # Gem requirements: gem "replicate-ruby", "~> 0.2.2"
-    #
-    # Use it directly:
-    # replicate = LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
-    #
-    # Or pass it to be instantiated by a vector search DB:
-    # chroma = Vectorsearch::Chroma.new(
-    #   url: ENV["CHROMA_URL"],
-    #   index_name: "...",
-    #   llm: Langchain::LLM::Replicate(api_key: ENV["REPLICATE_API_KEY"])
-    # )
-
     DEFAULTS = {
       # TODO: Figure out how to send the temperature to the API
       temperature: 0.01, # Minimum accepted value

data/lib/langchain/processors/base.rb

@@ -2,6 +2,7 @@
 
 module Langchain
   module Processors
+    # Processors load and parse/process various data types such as CSVs, PDFs, Word documents, HTML pages, and others.
     class Base
       include Langchain::DependencyHelper
 

data/lib/langchain/prompt/base.rb

@@ -5,15 +5,20 @@ require "json"
 require "yaml"
 
 module Langchain::Prompt
+  # Prompts are structured inputs to the LLMs. Prompts provide instructions, context and other user input that LLMs use to generate responses.
+  #
+  # @abstract
   class Base
     def format(**kwargs)
       raise NotImplementedError
     end
 
+    # @return [String] the type of the prompt
     def prompt_type
       raise NotImplementedError
     end
 
+    # @return [Hash] a hash representation of the prompt
     def to_h
       raise NotImplementedError
     end

data/lib/langchain/prompt/few_shot_prompt_template.rb

@@ -1,6 +1,51 @@
 # frozen_string_literal: true
 
 module Langchain::Prompt
+  # = Few Shot Prompt Templates
+  #
+  # Create a prompt with a few shot examples:
+  #
+  #     prompt = Langchain::Prompt::FewShotPromptTemplate.new(
+  #       prefix: "Write antonyms for the following words.",
+  #       suffix: "Input: <code>{adjective}</code>\nOutput:",
+  #       example_prompt: Langchain::Prompt::PromptTemplate.new(
+  #         input_variables: ["input", "output"],
+  #         template: "Input: {input}\nOutput: {output}"
+  #       ),
+  #       examples: [
+  #         { "input": "happy", "output": "sad" },
+  #         { "input": "tall", "output": "short" }
+  #       ],
+  #        input_variables: ["adjective"]
+  #     )
+  #
+  #     prompt.format(adjective: "good")
+  #
+  #     # Write antonyms for the following words.
+  #     #
+  #     # Input: happy
+  #     # Output: sad
+  #     #
+  #     # Input: tall
+  #     # Output: short
+  #     #
+  #     # Input: good
+  #     # Output:
+  #
+  # Save prompt template to JSON file:
+  #
+  #     prompt.save(file_path: "spec/fixtures/prompt/few_shot_prompt_template.json")
+  #
+  # Loading a new prompt template using a JSON file:
+  #
+  #     prompt = Langchain::Prompt.load_from_path(file_path: "spec/fixtures/prompt/few_shot_prompt_template.json")
+  #     prompt.prefix # "Write antonyms for the following words."
+  #
+  # Loading a new prompt template using a YAML file:
+  #
+  #     prompt = Langchain::Prompt.load_from_path(file_path: "spec/fixtures/prompt/prompt_template.yaml")
+  #     prompt.input_variables #=> ["adjective", "content"]
+  #
   class FewShotPromptTemplate < Base
     attr_reader :examples, :example_prompt, :input_variables, :prefix, :suffix, :example_separator
 

data/lib/langchain/prompt/prompt_template.rb

@@ -1,6 +1,37 @@
 # frozen_string_literal: true
 
 module Langchain::Prompt
+  # = Prompt Templates
+  #
+  # Create a prompt with one input variable:
+  #
+  #     prompt = Langchain::Prompt::PromptTemplate.new(template: "Tell me a {adjective} joke.", input_variables: ["adjective"])
+  #     prompt.format(adjective: "funny") # "Tell me a funny joke."
+  #
+  # Create a prompt with multiple input variables:
+  #
+  #     prompt = Langchain::Prompt::PromptTemplate.new(template: "Tell me a {adjective} joke about {content}.", input_variables: ["adjective", "content"])
+  #     prompt.format(adjective: "funny", content: "chickens") # "Tell me a funny joke about chickens."
+  #
+  # Creating a PromptTemplate using just a prompt and no input_variables:
+  #
+  #     prompt = Langchain::Prompt::PromptTemplate.from_template("Tell me a {adjective} joke about {content}.")
+  #     prompt.input_variables # ["adjective", "content"]
+  #     prompt.format(adjective: "funny", content: "chickens") # "Tell me a funny joke about chickens."
+  #
+  # Save prompt template to JSON file:
+  #
+  #     prompt.save(file_path: "spec/fixtures/prompt/prompt_template.json")
+  #
+  # Loading a new prompt template using a JSON file:
+  #
+  #     prompt = Langchain::Prompt.load_from_path(file_path: "spec/fixtures/prompt/prompt_template.json")
+  #     prompt.input_variables # ["adjective", "content"]
+  #
+  # Loading a new prompt template using a YAML file:
+  #     prompt = Langchain::Prompt.load_from_path(file_path: "spec/fixtures/prompt/prompt_template.yaml")
+  #     prompt.input_variables #=> ["adjective", "content"]
+  #
   class PromptTemplate < Base
     attr_reader :template, :input_variables, :validate_template
 

data/lib/langchain/tool/base.rb

@@ -1,16 +1,53 @@
 # frozen_string_literal: true
 
 module Langchain::Tool
+  # = Tools
+  #
+  # Tools are used by Agents to perform specific tasks. Basically anything is possible with enough code!
+  #
+  # == Available Tools
+  #
+  # - {Langchain::Tool::Calculator}: Calculate the result of a math expression
+  # - {Langchain::Tool::RubyCodeInterpretor}: Runs ruby code
+  # - {Langchain::Tool::Search}: search on Google (via SerpAPI)
+  # - {Langchain::Tool::Wikipedia}: search on Wikipedia
+  #
+  # == Usage
+  #
+  # 1. Pick the tools you'd like to pass to an Agent and install the gems listed under **Gem Requirements**
+  #
+  #     # To use all 3 tools:
+  #     gem install eqn
+  #     gem install google_search_results
+  #     gem install wikipedia-client
+  #
+  # 2. Set the environment variables listed under **ENV Requirements**
+  #
+  #     export SERPAPI_API_KEY=paste-your-serpapi-api-key-here
+  #
+  # 3. Pass the tools when Agent is instantiated.
+  #
+  #     agent = Langchain::Agent::ChainOfThoughtAgent.new(
+  #       llm: :openai, # or :cohere, :hugging_face, :google_palm or :replicate
+  #       llm_api_key: ENV["OPENAI_API_KEY"],
+  #       tools: ["search", "calculator", "wikipedia"]
+  #     )
+  #
+  # 4. Confirm that the Agent is using the Tools you passed in:
+  #
+  #     agent.tools
+  #     # => ["search", "calculator", "wikipedia"]
+  #
+  # == Adding Tools
+  #
+  # 1. Create a new file in lib/langchain/tool/your_tool_name.rb
+  # 2. Create a class in the file that inherits from {Langchain::Tool::Base}
+  # 3. Add `NAME=` and `DESCRIPTION=` constants in your Tool class
+  # 4. Implement `execute(input:)` method in your tool class
+  # 5. Add your tool to the {file:README.md}
   class Base
     include Langchain::DependencyHelper
 
-    # How to add additional Tools?
-    # 1. Create a new file in lib/tool/your_tool_name.rb
-    # 2. Create a class in the file that inherits from Langchain::Tool::Base
-    # 3. Add `NAME=` and `DESCRIPTION=` constants in your Tool class
-    # 4. Implement `execute(input:)` method in your tool class
-    # 5. Add your tool to the README.md
-
     #
     # Returns the NAME constant of the tool
     #
@@ -44,7 +81,7 @@ module Langchain::Tool
     #
     # @param input [String] input to the tool
     # @return [String] answer
-    #
+    # @raise NotImplementedError when not implemented
     def execute(input:)
       raise NotImplementedError, "Your tool must implement the `#execute(input:)` method that returns a string"
     end

data/lib/langchain/utils/token_length_validator.rb

@@ -51,13 +51,18 @@ module Langchain
       #
       # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
       #
-      # @param text [String] The text to validate
+      # @param content [String | Array<String>] The text or array of texts to validate
       # @param model_name [String] The model name to validate against
       # @return [Integer] Whether the text is valid or not
       # @raise [TokenLimitExceeded] If the text is too long
       #
-      def self.validate_max_tokens!(text, model_name)
-        text_token_length = token_length(text, model_name)
+      def self.validate_max_tokens!(content, model_name)
+        text_token_length = if content.is_a?(Array)
+          content.sum { |item| token_length(item.to_json, model_name) }
+        else
+          token_length(content, model_name)
+        end
+
         max_tokens = TOKEN_LIMITS[model_name] - text_token_length
 
         # Raise an error even if whole prompt is equal to the model's token limit (max_tokens == 0) since not response will be returned

data/lib/langchain/vectorsearch/base.rb

@@ -3,6 +3,88 @@
 require "forwardable"
 
 module Langchain::Vectorsearch
+  # = Vector Databases
+  # A vector database a type of database that stores data as high-dimensional vectors, which are mathematical representations of features or attributes. Each vector has a certain number of dimensions, which can range from tens to thousands, depending on the complexity and granularity of the data.
+  #
+  # == Available vector databases
+  #
+  # - {Langchain::Vectorsearch::Chroma}
+  # - {Langchain::Vectorsearch::Milvus}
+  # - {Langchain::Vectorsearch::Pinecone}
+  # - {Langchain::Vectorsearch::Qdrant}
+  # - {Langchain::Vectorsearch::Weaviate}
+  # - {Langchain::Vectorsearch::Pgvector}
+  #
+  # == Usage
+  #
+  # 1. Pick a vector database from list.
+  # 2. Review its documentation to install the required gems, and create an account, get an API key, etc
+  # 3. Instantiate the vector database class:
+  #
+  #     weaviate = Langchain::Vectorsearch::Weaviate.new(
+  #       url:         ENV["WEAVIATE_URL"],
+  #       api_key:     ENV["WEAVIATE_API_KEY"],
+  #       index_name:  "Documents",
+  #       llm:         :openai,              # or :cohere, :hugging_face, :google_palm, or :replicate
+  #       llm_api_key: ENV["OPENAI_API_KEY"] # API key for the selected LLM
+  #     )
+  #
+  #     # You can instantiate other supported vector databases the same way:
+  #     milvus   = Langchain::Vectorsearch::Milvus.new(...)
+  #     qdrant   = Langchain::Vectorsearch::Qdrant.new(...)
+  #     pinecone = Langchain::Vectorsearch::Pinecone.new(...)
+  #     chrome   = Langchain::Vectorsearch::Chroma.new(...)
+  #     pgvector = Langchain::Vectorsearch::Pgvector.new(...)
+  #
+  # == Schema Creation
+  #
+  # `create_default_schema()` creates default schema in your vector database.
+  #
+  #     search.create_default_schema
+  #
+  # (We plan on offering customizable schema creation shortly)
+  #
+  # == Adding Data
+  #
+  # You can add data with:
+  # 1. `add_data(path:, paths:)` to add any kind of data type
+  #
+  #     my_pdf = Langchain.root.join("path/to/my.pdf")
+  #     my_text = Langchain.root.join("path/to/my.txt")
+  #     my_docx = Langchain.root.join("path/to/my.docx")
+  #     my_csv = Langchain.root.join("path/to/my.csv")
+  #
+  #     search.add_data(paths: [my_pdf, my_text, my_docx, my_csv])
+  #
+  # 2. `add_texts(texts:)` to only add textual data
+  #
+  #     search.add_texts(
+  #       texts: [
+  #         "Lorem Ipsum is simply dummy text of the printing and typesetting industry.",
+  #         "Lorem Ipsum has been the industry's standard dummy text ever since the 1500s"
+  #       ]
+  #     )
+  #
+  # == Retrieving Data
+  #
+  # `similarity_search_by_vector(embedding:, k:)` searches the vector database for the closest `k` number of embeddings.
+  #
+  #    search.similarity_search_by_vector(
+  #      embedding: ...,
+  #      k: # number of results to be retrieved
+  #    )
+  #
+  # `vector_store.similarity_search(query:, k:)` generates an embedding for the query and searches the vector database for the closest `k` number of embeddings.
+  #
+  # search.similarity_search_by_vector(
+  #   embedding: ...,
+  #   k: # number of results to be retrieved
+  # )
+  #
+  # `ask(question:)` generates an embedding for the passed-in question, searches the vector database for closest embeddings and then passes these as context to the LLM to generate an answer to the question.
+  #
+  #     search.ask(question: "What is lorem ipsum?")
+  #
   class Base
     include Langchain::DependencyHelper
     extend Forwardable

data/lib/langchain/vectorsearch/chroma.rb

@@ -8,7 +8,7 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "chroma-db", "~> 0.3.0"
     #
     # Usage:
-    # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, llm_api_key:, api_key: nil)
+    # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, api_key: nil)
     #
 
     # Initialize the Chroma client

data/lib/langchain/vectorsearch/milvus.rb

@@ -8,9 +8,17 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "milvus", "~> 0.9.0"
     #
     # Usage:
-    # milvus = Langchain::Vectorsearch::Milvus.new(url:, index_name:, llm:, llm_api_key:)
+    # milvus = Langchain::Vectorsearch::Milvus.new(url:, index_name:, llm:)
     #
 
+    #
+    # Initialize the Milvus client
+    #
+    # @param url [String] The URL of the Milvus server
+    # @param api_key [String] The API key to use
+    # @param index_name [String] The name of the index to use
+    # @param llm [Object] The LLM client to use
+    #
     def initialize(url:, index_name:, llm:, api_key: nil)
       depends_on "milvus"
       require "milvus"
@@ -21,6 +29,11 @@ module Langchain::Vectorsearch
       super(llm: llm)
     end
 
+    #
+    # Add a list of texts to the index
+    #
+    # @param texts [Array] The list of texts to add
+    #
     def add_texts(texts:)
       client.entities.insert(
         collection_name: index_name,

data/lib/langchain/vectorsearch/pgvector.rb

@@ -8,7 +8,7 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "pgvector", "~> 0.2"
     #
     # Usage:
-    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:, llm_api_key:)
+    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:)
     #
 
     # The operators supported by the PostgreSQL vector search adapter
@@ -20,10 +20,14 @@ module Langchain::Vectorsearch
 
     attr_reader :operator, :quoted_table_name
 
+    #
+    # Initialize the PostgreSQL client
+    #
     # @param url [String] The URL of the PostgreSQL database
     # @param index_name [String] The name of the table to use for the index
     # @param llm [Object] The LLM client to use
     # @param api_key [String] The API key for the Vectorsearch DB (not used for PostgreSQL)
+    #
     def initialize(url:, index_name:, llm:, api_key: nil)
       require "pg"
       require "pgvector"

data/lib/langchain/vectorsearch/pinecone.rb

@@ -8,14 +8,17 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "pinecone", "~> 0.1.6"
     #
     # Usage:
-    # pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:, llm_api_key:)
+    # pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:)
     #
 
+    #
     # Initialize the Pinecone client
+    #
     # @param environment [String] The environment to use
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
+    #
     def initialize(environment:, api_key:, index_name:, llm:)
       depends_on "pinecone"
       require "pinecone"

data/lib/langchain/vectorsearch/qdrant.rb

@@ -8,14 +8,17 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "qdrant-ruby", "~> 0.9.0"
     #
     # Usage:
-    # qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:, llm_api_key:)
+    # qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:)
     #
 
+    #
     # Initialize the Qdrant client
+    #
     # @param url [String] The URL of the Qdrant server
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
+    #
     def initialize(url:, api_key:, index_name:, llm:)
       depends_on "qdrant-ruby"
       require "qdrant"

data/lib/langchain/vectorsearch/weaviate.rb

@@ -8,14 +8,17 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "weaviate-ruby", "~> 0.8.0"
     #
     # Usage:
-    # weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:, llm_api_key:)
+    # weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:)
     #
 
+    #
     # Initialize the Weaviate adapter
+    #
     # @param url [String] The URL of the Weaviate instance
     # @param api_key [String] The API key to use
     # @param index_name [String] The name of the index to use
     # @param llm [Object] The LLM client to use
+    #
     def initialize(url:, api_key:, index_name:, llm:)
       depends_on "weaviate-ruby"
       require "weaviate"

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.5.2"
+  VERSION = "0.5.3"
 end

data/lib/langchain.rb

@@ -6,10 +6,53 @@ require "colorize"
 
 require_relative "./langchain/version"
 
+# Langchain.rb a is library for building LLM-backed Ruby applications. It is an abstraction layer that sits on top of the emerging AI-related tools that makes it easy for developers to consume and string those services together.
+#
+# = Installation
+# Install the gem and add to the application's Gemfile by executing:
+#
+#     $ bundle add langchainrb
+#
+# If bundler is not being used to manage dependencies, install the gem by executing:
+#
+#     $ gem install langchainrb
+#
+# Require the gem to start using it:
+#
+#     require "langchain"
+#
+# = Concepts
+#
+# == Processors
+# Processors load and parse/process various data types such as CSVs, PDFs, Word documents, HTML pages, and others.
+#
+# == Chunkers
+# Chunkers split data based on various available options such as delimeters, chunk sizes or custom-defined functions. Chunkers are used when data needs to be split up before being imported in vector databases.
+#
+# == Prompts
+# Prompts are structured inputs to the LLMs. Prompts provide instructions, context and other user input that LLMs use to generate responses.
+#
+# == Large Language Models (LLMs)
+# LLM is a language model consisting of a neural network with many parameters (typically billions of weights or more), trained on large quantities of unlabeled text using self-supervised learning or semi-supervised learning.
+#
+# == Vectorsearch Databases
+# Vector database is a type of database that stores data as high-dimensional vectors, which are mathematical representations of features or attributes. Each vector has a certain number of dimensions, which can range from tens to thousands, depending on the complexity and granularity of the data.
+#
+# == Embedding
+# Word embedding or word vector is an approach with which we represent documents and words. It is defined as a numeric vector input that allows words with similar meanings to have the same representation. It can approximate meaning and represent a word in a lower dimensional space.
+#
+#
+# = Logging
+#
+# LangChain.rb uses standard logging mechanisms and defaults to :debug level. Most messages are at info level, but we will add debug or warn statements as needed. To show all log messages:
+#
+#     Langchain.logger.level = :info
 module Langchain
   class << self
+    # @return [Logger]
     attr_accessor :logger
 
+    # @return [Pathname]
     attr_reader :root
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-08 00:00:00.000000000 Z
+date: 2023-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tiktoken_ruby
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.1.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: google_search_results
   requirement: !ruby/object:Gem::Requirement
@@ -495,7 +495,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Build LLM-backed Ruby applications with Ruby's LangChain