langchainrb 0.5.2 → 0.5.4

data/lib/langchain/prompt/base.rb

@@ -5,15 +5,20 @@ require "json"
 require "yaml"
 
 module Langchain::Prompt
+  # Prompts are structured inputs to the LLMs. Prompts provide instructions, context and other user input that LLMs use to generate responses.
+  #
+  # @abstract
   class Base
     def format(**kwargs)
       raise NotImplementedError
     end
 
+    # @return [String] the type of the prompt
     def prompt_type
       raise NotImplementedError
     end
 
+    # @return [Hash] a hash representation of the prompt
     def to_h
       raise NotImplementedError
     end

data/lib/langchain/prompt/few_shot_prompt_template.rb

@@ -1,6 +1,51 @@
 # frozen_string_literal: true
 
 module Langchain::Prompt
+  # = Few Shot Prompt Templates
+  #
+  # Create a prompt with a few shot examples:
+  #
+  #     prompt = Langchain::Prompt::FewShotPromptTemplate.new(
+  #       prefix: "Write antonyms for the following words.",
+  #       suffix: "Input: <code>{adjective}</code>\nOutput:",
+  #       example_prompt: Langchain::Prompt::PromptTemplate.new(
+  #         input_variables: ["input", "output"],
+  #         template: "Input: {input}\nOutput: {output}"
+  #       ),
+  #       examples: [
+  #         { "input": "happy", "output": "sad" },
+  #         { "input": "tall", "output": "short" }
+  #       ],
+  #        input_variables: ["adjective"]
+  #     )
+  #
+  #     prompt.format(adjective: "good")
+  #
+  #     # Write antonyms for the following words.
+  #     #
+  #     # Input: happy
+  #     # Output: sad
+  #     #
+  #     # Input: tall
+  #     # Output: short
+  #     #
+  #     # Input: good
+  #     # Output:
+  #
+  # Save prompt template to JSON file:
+  #
+  #     prompt.save(file_path: "spec/fixtures/prompt/few_shot_prompt_template.json")
+  #
+  # Loading a new prompt template using a JSON file:
+  #
+  #     prompt = Langchain::Prompt.load_from_path(file_path: "spec/fixtures/prompt/few_shot_prompt_template.json")
+  #     prompt.prefix # "Write antonyms for the following words."
+  #
+  # Loading a new prompt template using a YAML file:
+  #
+  #     prompt = Langchain::Prompt.load_from_path(file_path: "spec/fixtures/prompt/prompt_template.yaml")
+  #     prompt.input_variables #=> ["adjective", "content"]
+  #
   class FewShotPromptTemplate < Base
     attr_reader :examples, :example_prompt, :input_variables, :prefix, :suffix, :example_separator
 

data/lib/langchain/prompt/prompt_template.rb

@@ -1,6 +1,37 @@
 # frozen_string_literal: true
 
 module Langchain::Prompt
+  # = Prompt Templates
+  #
+  # Create a prompt with one input variable:
+  #
+  #     prompt = Langchain::Prompt::PromptTemplate.new(template: "Tell me a {adjective} joke.", input_variables: ["adjective"])
+  #     prompt.format(adjective: "funny") # "Tell me a funny joke."
+  #
+  # Create a prompt with multiple input variables:
+  #
+  #     prompt = Langchain::Prompt::PromptTemplate.new(template: "Tell me a {adjective} joke about {content}.", input_variables: ["adjective", "content"])
+  #     prompt.format(adjective: "funny", content: "chickens") # "Tell me a funny joke about chickens."
+  #
+  # Creating a PromptTemplate using just a prompt and no input_variables:
+  #
+  #     prompt = Langchain::Prompt::PromptTemplate.from_template("Tell me a {adjective} joke about {content}.")
+  #     prompt.input_variables # ["adjective", "content"]
+  #     prompt.format(adjective: "funny", content: "chickens") # "Tell me a funny joke about chickens."
+  #
+  # Save prompt template to JSON file:
+  #
+  #     prompt.save(file_path: "spec/fixtures/prompt/prompt_template.json")
+  #
+  # Loading a new prompt template using a JSON file:
+  #
+  #     prompt = Langchain::Prompt.load_from_path(file_path: "spec/fixtures/prompt/prompt_template.json")
+  #     prompt.input_variables # ["adjective", "content"]
+  #
+  # Loading a new prompt template using a YAML file:
+  #     prompt = Langchain::Prompt.load_from_path(file_path: "spec/fixtures/prompt/prompt_template.yaml")
+  #     prompt.input_variables #=> ["adjective", "content"]
+  #
   class PromptTemplate < Base
     attr_reader :template, :input_variables, :validate_template
 

data/lib/langchain/tool/base.rb

@@ -1,16 +1,53 @@
 # frozen_string_literal: true
 
 module Langchain::Tool
+  # = Tools
+  #
+  # Tools are used by Agents to perform specific tasks. Basically anything is possible with enough code!
+  #
+  # == Available Tools
+  #
+  # - {Langchain::Tool::Calculator}: Calculate the result of a math expression
+  # - {Langchain::Tool::RubyCodeInterpretor}: Runs ruby code
+  # - {Langchain::Tool::Search}: search on Google (via SerpAPI)
+  # - {Langchain::Tool::Wikipedia}: search on Wikipedia
+  #
+  # == Usage
+  #
+  # 1. Pick the tools you'd like to pass to an Agent and install the gems listed under **Gem Requirements**
+  #
+  #     # To use all 3 tools:
+  #     gem install eqn
+  #     gem install google_search_results
+  #     gem install wikipedia-client
+  #
+  # 2. Set the environment variables listed under **ENV Requirements**
+  #
+  #     export SERPAPI_API_KEY=paste-your-serpapi-api-key-here
+  #
+  # 3. Pass the tools when Agent is instantiated.
+  #
+  #     agent = Langchain::Agent::ChainOfThoughtAgent.new(
+  #       llm: :openai, # or :cohere, :hugging_face, :google_palm or :replicate
+  #       llm_api_key: ENV["OPENAI_API_KEY"],
+  #       tools: ["search", "calculator", "wikipedia"]
+  #     )
+  #
+  # 4. Confirm that the Agent is using the Tools you passed in:
+  #
+  #     agent.tools
+  #     # => ["search", "calculator", "wikipedia"]
+  #
+  # == Adding Tools
+  #
+  # 1. Create a new file in lib/langchain/tool/your_tool_name.rb
+  # 2. Create a class in the file that inherits from {Langchain::Tool::Base}
+  # 3. Add `NAME=` and `DESCRIPTION=` constants in your Tool class
+  # 4. Implement `execute(input:)` method in your tool class
+  # 5. Add your tool to the {file:README.md}
   class Base
     include Langchain::DependencyHelper
 
-    # How to add additional Tools?
-    # 1. Create a new file in lib/tool/your_tool_name.rb
-    # 2. Create a class in the file that inherits from Langchain::Tool::Base
-    # 3. Add `NAME=` and `DESCRIPTION=` constants in your Tool class
-    # 4. Implement `execute(input:)` method in your tool class
-    # 5. Add your tool to the README.md
-
     #
     # Returns the NAME constant of the tool
     #
@@ -20,6 +57,15 @@ module Langchain::Tool
       self.class.const_get(:NAME)
     end
 
+    #
+    # Returns the DESCRIPTION constant of the tool
+    #
+    # @return [String] tool description
+    #
+    def tool_description
+      self.class.const_get(:DESCRIPTION)
+    end
+
     #
     # Sets the DESCRIPTION constant of the tool
     #
@@ -44,7 +90,7 @@ module Langchain::Tool
     #
     # @param input [String] input to the tool
     # @return [String] answer
-    #
+    # @raise NotImplementedError when not implemented
     def execute(input:)
       raise NotImplementedError, "Your tool must implement the `#execute(input:)` method that returns a string"
     end

data/lib/langchain/utils/token_length/google_palm_validator.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Utils
+    module TokenLength
+      #
+      # This class is meant to validate the length of the text passed in to Google Palm's API.
+      # It is used to validate the token length before the API call is made
+      #
+      class GooglePalmValidator
+        TOKEN_LIMITS = {
+          # Source:
+          # This data can be pulled when `list_models()` method is called: https://github.com/andreibondarev/google_palm_api#usage
+
+          # chat-bison-001 is the only model that currently supports countMessageTokens functions
+          "chat-bison-001" => {
+            "input_token_limit" => 4000, # 4096 is the limit but the countMessageTokens does not return anything higher than 4000
+            "output_token_limit" => 1024
+          }
+          # "text-bison-001" => {
+          #   "input_token_limit" => 8196,
+          #   "output_token_limit" => 1024
+          # },
+          # "embedding-gecko-001" => {
+          #   "input_token_limit" => 1024
+          # }
+        }.freeze
+
+        #
+        # Validate the context length of the text
+        #
+        # @param content [String | Array<String>] The text or array of texts to validate
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] Whether the text is valid or not
+        # @raise [TokenLimitExceeded] If the text is too long
+        #
+        def self.validate_max_tokens!(google_palm_llm, content, model_name)
+          text_token_length = if content.is_a?(Array)
+            content.sum { |item| token_length(google_palm_llm, item.to_json, model_name) }
+          else
+            token_length(google_palm_llm, content, model_name)
+          end
+
+          leftover_tokens = TOKEN_LIMITS.dig(model_name, "input_token_limit") - text_token_length
+
+          # Raise an error even if whole prompt is equal to the model's token limit (leftover_tokens == 0)
+          if leftover_tokens <= 0
+            raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS.dig(model_name, "input_token_limit")} tokens, but the given text is #{text_token_length} tokens long."
+          end
+
+          leftover_tokens
+        end
+
+        #
+        # Calculate token length for a given text and model name
+        #
+        # @param llm [Langchain::LLM:GooglePalm] The Langchain::LLM:GooglePalm instance
+        # @param text [String] The text to calculate the token length for
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] The token length of the text
+        #
+        def self.token_length(llm, text, model_name = "chat-bison-001")
+          response = llm.client.count_message_tokens(model: model_name, prompt: text)
+          response.dig("tokenCount")
+        end
+      end
+    end
+  end
+end

data/lib/langchain/utils/token_length/openai_validator.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "tiktoken_ruby"
+
+module Langchain
+  module Utils
+    module TokenLength
+      #
+      # This class is meant to validate the length of the text passed in to OpenAI's API.
+      # It is used to validate the token length before the API call is made
+      #
+      class OpenAIValidator
+        TOKEN_LIMITS = {
+          # Source:
+          # https://platform.openai.com/docs/api-reference/embeddings
+          # https://platform.openai.com/docs/models/gpt-4
+          "text-embedding-ada-002" => 8191,
+          "gpt-3.5-turbo" => 4096,
+          "gpt-3.5-turbo-0301" => 4096,
+          "text-davinci-003" => 4097,
+          "text-davinci-002" => 4097,
+          "code-davinci-002" => 8001,
+          "gpt-4" => 8192,
+          "gpt-4-0314" => 8192,
+          "gpt-4-32k" => 32768,
+          "gpt-4-32k-0314" => 32768,
+          "text-curie-001" => 2049,
+          "text-babbage-001" => 2049,
+          "text-ada-001" => 2049,
+          "davinci" => 2049,
+          "curie" => 2049,
+          "babbage" => 2049,
+          "ada" => 2049
+        }.freeze
+
+        #
+        # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
+        #
+        # @param content [String | Array<String>] The text or array of texts to validate
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] Whether the text is valid or not
+        # @raise [TokenLimitExceeded] If the text is too long
+        #
+        def self.validate_max_tokens!(content, model_name)
+          text_token_length = if content.is_a?(Array)
+            content.sum { |item| token_length(item.to_json, model_name) }
+          else
+            token_length(content, model_name)
+          end
+
+          max_tokens = TOKEN_LIMITS[model_name] - text_token_length
+
+          # Raise an error even if whole prompt is equal to the model's token limit (max_tokens == 0) since not response will be returned
+          if max_tokens <= 0
+            raise TokenLimitExceeded, "This model's maximum context length is #{TOKEN_LIMITS[model_name]} tokens, but the given text is #{text_token_length} tokens long."
+          end
+
+          max_tokens
+        end
+
+        #
+        # Calculate token length for a given text and model name
+        #
+        # @param text [String] The text to calculate the token length for
+        # @param model_name [String] The model name to validate against
+        # @return [Integer] The token length of the text
+        #
+        def self.token_length(text, model_name)
+          encoder = Tiktoken.encoding_for_model(model_name)
+          encoder.encode(text).length
+        end
+      end
+    end
+  end
+end

data/lib/langchain/vectorsearch/base.rb

@@ -3,6 +3,88 @@
 require "forwardable"
 
 module Langchain::Vectorsearch
+  # = Vector Databases
+  # A vector database a type of database that stores data as high-dimensional vectors, which are mathematical representations of features or attributes. Each vector has a certain number of dimensions, which can range from tens to thousands, depending on the complexity and granularity of the data.
+  #
+  # == Available vector databases
+  #
+  # - {Langchain::Vectorsearch::Chroma}
+  # - {Langchain::Vectorsearch::Milvus}
+  # - {Langchain::Vectorsearch::Pinecone}
+  # - {Langchain::Vectorsearch::Qdrant}
+  # - {Langchain::Vectorsearch::Weaviate}
+  # - {Langchain::Vectorsearch::Pgvector}
+  #
+  # == Usage
+  #
+  # 1. Pick a vector database from list.
+  # 2. Review its documentation to install the required gems, and create an account, get an API key, etc
+  # 3. Instantiate the vector database class:
+  #
+  #     weaviate = Langchain::Vectorsearch::Weaviate.new(
+  #       url:         ENV["WEAVIATE_URL"],
+  #       api_key:     ENV["WEAVIATE_API_KEY"],
+  #       index_name:  "Documents",
+  #       llm:         :openai,              # or :cohere, :hugging_face, :google_palm, or :replicate
+  #       llm_api_key: ENV["OPENAI_API_KEY"] # API key for the selected LLM
+  #     )
+  #
+  #     # You can instantiate other supported vector databases the same way:
+  #     milvus   = Langchain::Vectorsearch::Milvus.new(...)
+  #     qdrant   = Langchain::Vectorsearch::Qdrant.new(...)
+  #     pinecone = Langchain::Vectorsearch::Pinecone.new(...)
+  #     chrome   = Langchain::Vectorsearch::Chroma.new(...)
+  #     pgvector = Langchain::Vectorsearch::Pgvector.new(...)
+  #
+  # == Schema Creation
+  #
+  # `create_default_schema()` creates default schema in your vector database.
+  #
+  #     search.create_default_schema
+  #
+  # (We plan on offering customizable schema creation shortly)
+  #
+  # == Adding Data
+  #
+  # You can add data with:
+  # 1. `add_data(path:, paths:)` to add any kind of data type
+  #
+  #     my_pdf = Langchain.root.join("path/to/my.pdf")
+  #     my_text = Langchain.root.join("path/to/my.txt")
+  #     my_docx = Langchain.root.join("path/to/my.docx")
+  #     my_csv = Langchain.root.join("path/to/my.csv")
+  #
+  #     search.add_data(paths: [my_pdf, my_text, my_docx, my_csv])
+  #
+  # 2. `add_texts(texts:)` to only add textual data
+  #
+  #     search.add_texts(
+  #       texts: [
+  #         "Lorem Ipsum is simply dummy text of the printing and typesetting industry.",
+  #         "Lorem Ipsum has been the industry's standard dummy text ever since the 1500s"
+  #       ]
+  #     )
+  #
+  # == Retrieving Data
+  #
+  # `similarity_search_by_vector(embedding:, k:)` searches the vector database for the closest `k` number of embeddings.
+  #
+  #    search.similarity_search_by_vector(
+  #      embedding: ...,
+  #      k: # number of results to be retrieved
+  #    )
+  #
+  # `vector_store.similarity_search(query:, k:)` generates an embedding for the query and searches the vector database for the closest `k` number of embeddings.
+  #
+  # search.similarity_search_by_vector(
+  #   embedding: ...,
+  #   k: # number of results to be retrieved
+  # )
+  #
+  # `ask(question:)` generates an embedding for the passed-in question, searches the vector database for closest embeddings and then passes these as context to the LLM to generate an answer to the question.
+  #
+  #     search.ask(question: "What is lorem ipsum?")
+  #
   class Base
     include Langchain::DependencyHelper
     extend Forwardable

data/lib/langchain/vectorsearch/hnswlib.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Langchain::Vectorsearch
+  class Hnswlib < Base
+    #
+    # Wrapper around HNSW (Hierarchical Navigable Small World) library.
+    # HNSWLib is an in-memory vectorstore that can be saved to a file on disk.
+    #
+    # Gem requirements:
+    #     gem "hnswlib", "~> 0.8.1"
+    #
+    # Usage:
+    #     hnsw = Langchain::Vectorsearch::Hnswlib.new(llm:, url:, index_name:)
+    #
+
+    attr_reader :client, :path_to_index
+
+    #
+    # Initialize the HNSW vector search
+    #
+    # @param llm [Object] The LLM client to use
+    # @param path_to_index [String] The local path to the index file, e.g.: "/storage/index.ann"
+    # @return [Langchain::Vectorsearch::Hnswlib] Class instance
+    #
+    def initialize(llm:, path_to_index:)
+      depends_on "hnswlib"
+      require "hnswlib"
+
+      super(llm: llm)
+
+      @client = ::Hnswlib::HierarchicalNSW.new(space: DEFAULT_METRIC, dim: llm.default_dimension)
+      @path_to_index = path_to_index
+
+      initialize_index
+    end
+
+    #
+    # Add a list of texts and corresponding IDs to the index
+    #
+    # @param texts [Array] The list of texts to add
+    # @param ids [Array] The list of corresponding IDs (integers) to the texts
+    # @return [Boolean] The response from the HNSW library
+    #
+    def add_texts(texts:, ids:)
+      resize_index(texts.size)
+
+      Array(texts).each_with_index do |text, i|
+        embedding = llm.embed(text: text)
+
+        client.add_point(embedding, ids[i])
+      end
+
+      client.save_index(path_to_index)
+    end
+
+    #
+    # Search for similar texts
+    #
+    # @param query [String] The text to search for
+    # @param k [Integer] The number of results to return
+    # @return [Array] Results in the format `[[id1, distance3], [id2, distance2]]`
+    #
+    def similarity_search(
+      query:,
+      k: 4
+    )
+      embedding = llm.embed(text: query)
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    #
+    # Search for the K nearest neighbors of a given vector
+    #
+    # @param embedding [Array] The embedding to search for
+    # @param k [Integer] The number of results to return
+    # @return [Array] Results in the format `[[id1, distance3], [id2, distance2]]`
+    #
+    def similarity_search_by_vector(
+      embedding:,
+      k: 4
+    )
+      client.search_knn(embedding, k)
+    end
+
+    private
+
+    #
+    # Optionally resizes the index if there's no space for new data
+    #
+    # @param num_of_elements_to_add [Integer] The number of elements to add to the index
+    #
+    def resize_index(num_of_elements_to_add)
+      current_count = client.current_count
+
+      if (current_count + num_of_elements_to_add) > client.max_elements
+        new_size = current_count + num_of_elements_to_add
+
+        client.resize_index(new_size)
+      end
+    end
+
+    #
+    # Loads or initializes the new index
+    #
+    def initialize_index
+      if File.exist?(path_to_index)
+        client.load_index(path_to_index)
+
+        Langchain.logger.info("[#{self.class.name}]".blue + ": Successfully loaded the index at \"#{path_to_index}\"")
+      else
+        # Default max_elements: 100, but we constantly resize the index as new data is written to it
+        client.init_index(max_elements: 100)
+
+        Langchain.logger.info("[#{self.class.name}]".blue + ": Creating a new index at \"#{path_to_index}\"")
+      end
+    end
+  end
+end

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.5.2"
+  VERSION = "0.5.4"
 end

data/lib/langchain.rb

@@ -6,10 +6,53 @@ require "colorize"
 
 require_relative "./langchain/version"
 
+# Langchain.rb a is library for building LLM-backed Ruby applications. It is an abstraction layer that sits on top of the emerging AI-related tools that makes it easy for developers to consume and string those services together.
+#
+# = Installation
+# Install the gem and add to the application's Gemfile by executing:
+#
+#     $ bundle add langchainrb
+#
+# If bundler is not being used to manage dependencies, install the gem by executing:
+#
+#     $ gem install langchainrb
+#
+# Require the gem to start using it:
+#
+#     require "langchain"
+#
+# = Concepts
+#
+# == Processors
+# Processors load and parse/process various data types such as CSVs, PDFs, Word documents, HTML pages, and others.
+#
+# == Chunkers
+# Chunkers split data based on various available options such as delimeters, chunk sizes or custom-defined functions. Chunkers are used when data needs to be split up before being imported in vector databases.
+#
+# == Prompts
+# Prompts are structured inputs to the LLMs. Prompts provide instructions, context and other user input that LLMs use to generate responses.
+#
+# == Large Language Models (LLMs)
+# LLM is a language model consisting of a neural network with many parameters (typically billions of weights or more), trained on large quantities of unlabeled text using self-supervised learning or semi-supervised learning.
+#
+# == Vectorsearch Databases
+# Vector database is a type of database that stores data as high-dimensional vectors, which are mathematical representations of features or attributes. Each vector has a certain number of dimensions, which can range from tens to thousands, depending on the complexity and granularity of the data.
+#
+# == Embedding
+# Word embedding or word vector is an approach with which we represent documents and words. It is defined as a numeric vector input that allows words with similar meanings to have the same representation. It can approximate meaning and represent a word in a lower dimensional space.
+#
+#
+# = Logging
+#
+# LangChain.rb uses standard logging mechanisms and defaults to :debug level. Most messages are at info level, but we will add debug or warn statements as needed. To show all log messages:
+#
+#     Langchain.logger.level = :info
 module Langchain
   class << self
+    # @return [Logger]
     attr_accessor :logger
 
+    # @return [Pathname]
     attr_reader :root
   end
 
@@ -19,6 +62,7 @@ module Langchain
 
   autoload :Loader, "langchain/loader"
   autoload :Data, "langchain/data"
+  autoload :Chat, "langchain/chat"
   autoload :DependencyHelper, "langchain/dependency_helper"
 
   module Agent
@@ -49,12 +93,18 @@ module Langchain
   end
 
   module Utils
-    autoload :TokenLengthValidator, "langchain/utils/token_length_validator"
+    module TokenLength
+      class TokenLimitExceeded < StandardError; end
+
+      autoload :OpenAIValidator, "langchain/utils/token_length/openai_validator"
+      autoload :GooglePalmValidator, "langchain/utils/token_length/google_palm_validator"
+    end
   end
 
   module Vectorsearch
     autoload :Base, "langchain/vectorsearch/base"
     autoload :Chroma, "langchain/vectorsearch/chroma"
+    autoload :Hnswlib, "langchain/vectorsearch/hnswlib"
     autoload :Milvus, "langchain/vectorsearch/milvus"
     autoload :Pinecone, "langchain/vectorsearch/pinecone"
     autoload :Pgvector, "langchain/vectorsearch/pgvector"