langchainrb 0.5.2 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d36de4206b792714ba9b6773c03272e9638b14caf7140e0bc00c3e767aa5fdef
-  data.tar.gz: 819fab9de55a34e4e6dc865febc19bb9979df55fa8fc6a753774cf1961c40103
+  metadata.gz: 87647b8a7e2dc49359f3f6d655eda501dcac26ebdd14247ad6c583be8dc1a71c
+  data.tar.gz: fb7b4321caa4ff026439158f5ccfc2ae9e7b515a69c35cba87385f2cb367fa85
 SHA512:
-  metadata.gz: 6e180b41bbca96bd5523c276923f223bbebe470314086c6a909df440890793bcc70dbd66ecf59bf5d0fd52426650cc5d2684c56cc8fc643209cc1679527cbef4
-  data.tar.gz: af5db76c2b22b5c7bdc1170de437921e8464a16566f46a5cad465d69e6da47c97a82f7331a5ea5747840e58acc71463aa8456b03e9bc8851efda7b734e5d23cc
+  metadata.gz: 4f80677c43c00e6d50e0494aa79cb7648b9f3878ed8d2a5f4f2dc90e308a3639589f8457a4615821b70b44c5a43ae4f26fcf00d7548684740e4c05dbcc165bf8
+  data.tar.gz: 4722233dbed83d21f2dadff19a9b79a30d8fd208d6e30bd057f018060c602b7f00f0526ee0364823597806388e2a8da48e883a5e6fa77b31490199685644b4d2

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## [Unreleased]
 
+## [0.5.4] - 2023-06-10
+- 🔍 Vectorsearch
+  - Introducing support for HNSWlib
+- Improved and new `Langchain::Chat` interface that persists chat history in memory
+
+## [0.5.3] - 2023-06-09
+- 🗣️ LLMs
+  - Chat message history support for Langchain::LLM::GooglePalm and Langchain::LLM::OpenAI
+
 ## [0.5.2] - 2023-06-07
 - 🗣️ LLMs
   - Auto-calculate the max_tokens: setting to be passed on to OpenAI

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.5.2)
+    langchainrb (0.5.4)
       colorize (~> 0.8.1)
       tiktoken_ruby (~> 0.0.5)
 
@@ -122,7 +122,7 @@ GEM
     faraday-retry (1.0.3)
     faraday_middleware (1.2.0)
       faraday (~> 1.0)
-    google_palm_api (0.1.0)
+    google_palm_api (0.1.1)
       faraday (>= 1.0.0)
       faraday_middleware (>= 1.0.0)
     google_search_results (2.0.1)
@@ -135,6 +135,7 @@ GEM
       activesupport (>= 3.0)
       graphql
     hashery (2.1.2)
+    hnswlib (0.8.1)
     httparty (0.21.0)
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
@@ -310,8 +311,9 @@ DEPENDENCIES
   docx (~> 0.8.0)
   dotenv-rails (~> 2.7.6)
   eqn (~> 1.6.5)
-  google_palm_api (~> 0.1.0)
+  google_palm_api (~> 0.1.1)
   google_search_results (~> 2.0.0)
+  hnswlib (~> 0.8.1)
   hugging-face (~> 0.3.4)
   langchainrb!
   milvus (~> 0.9.0)

data/README.md

@@ -1,4 +1,4 @@
-🦜️🔗 LangChain.rb
+💎🔗 LangChain.rb
 ---
 ⚡ Building applications with LLMs through composability ⚡
 
@@ -6,7 +6,10 @@
 
 :warning: UNDER ACTIVE AND RAPID DEVELOPMENT (MAY BE BUGGY AND UNTESTED)
 
-![Tests status](https://github.com/andreibondarev/langchainrb/actions/workflows/ci.yml/badge.svg) [![Gem Version](https://badge.fury.io/rb/langchainrb.svg)](https://badge.fury.io/rb/langchainrb)
+![Tests status](https://github.com/andreibondarev/langchainrb/actions/workflows/ci.yml/badge.svg)
+[![Gem Version](https://badge.fury.io/rb/langchainrb.svg)](https://badge.fury.io/rb/langchainrb)
+[![Docs](http://img.shields.io/badge/yard-docs-blue.svg)](http://rubydoc.info/gems/langchainrb)
+[![License](https://img.shields.io/badge/license-MIT-green.svg)](https://github.com/andreibondarev/langchainrb/blob/main/LICENSE.txt)
 
 Langchain.rb is a library that's an abstraction layer on top many emergent AI, ML and other DS tools. The goal is to abstract complexity and difficult concepts to make building AI/ML-supercharged applications approachable for traditional software engineers.
 
@@ -31,8 +34,10 @@ require "langchain"
 | Database | Querying           | Storage | Schema Management | Backups | Rails Integration |
 | -------- |:------------------:| -------:| -----------------:| -------:| -----------------:|
 | [Chroma](https://trychroma.com/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Hnswlib](https://github.com/nmslib/hnswlib/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Milvus](https://milvus.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Pinecone](https://www.pinecone.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Pgvector](https://github.com/pgvector/pgvector) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Qdrant](https://qdrant.tech/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Weaviate](https://weaviate.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 
@@ -47,14 +52,17 @@ Pick the vector search database you'll be using and instantiate the client:
 client = Langchain::Vectorsearch::Weaviate.new(
     url: ENV["WEAVIATE_URL"],
     api_key: ENV["WEAVIATE_API_KEY"],
+    index: "",
     llm: Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
 )
 
 # You can instantiate any other supported vector search database:
+client = Langchain::Vectorsearch::Chroma.new(...) # `gem "chroma-db", "~> 0.3.0"`
+client = Langchain::Vectorsearch::Hnswlib.new(...) # `gem "hnswlib", "~> 0.8.1"`
 client = Langchain::Vectorsearch::Milvus.new(...) # `gem "milvus", "~> 0.9.0"`
-client = Langchain::Vectorsearch::Qdrant.new(...) # `gem"qdrant-ruby", "~> 0.9.0"`
 client = Langchain::Vectorsearch::Pinecone.new(...) # `gem "pinecone", "~> 0.1.6"`
-client = Langchain::Vectorsearch::Chroma.new(...) # `gem "chroma-db", "~> 0.3.0"`
+client = Langchain::Vectorsearch::Pgvector.new(...) # `gem "pgvector", "~> 0.2"`
+client = Langchain::Vectorsearch::Qdrant.new(...) # `gem"qdrant-ruby", "~> 0.9.0"`
 ```
 
 ```ruby
@@ -135,17 +143,17 @@ cohere.complete(prompt: "What is the meaning of life?")
 #### HuggingFace
 Add `gem "hugging-face", "~> 0.3.2"` to your Gemfile.
 ```ruby
-cohere = Langchain::LLM::HuggingFace.new(api_key: ENV["HUGGING_FACE_API_KEY"])
+hugging_face = Langchain::LLM::HuggingFace.new(api_key: ENV["HUGGING_FACE_API_KEY"])
 ```
 
 #### Replicate
 Add `gem "replicate-ruby", "~> 0.2.2"` to your Gemfile.
 ```ruby
-cohere = Langchain::LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
+replicate = Langchain::LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
 ```
 
 #### Google PaLM (Pathways Language Model)
-Add `"google_palm_api", "~> 0.1.0"` to your Gemfile.
+Add `"google_palm_api", "~> 0.1.1"` to your Gemfile.
 ```ruby
 google_palm = Langchain::LLM::GooglePalm.new(api_key: ENV["GOOGLE_PALM_API_KEY"])
 ```

data/Rakefile

@@ -14,5 +14,4 @@ Rake::Task["spec"].enhance do
 end
 
 YARD::Rake::YardocTask.new do |t|
-  t.options = ["--fail-on-warning"]
 end

data/lib/langchain/agent/base.rb

@@ -1,6 +1,14 @@
 # frozen_string_literal: true
 
 module Langchain::Agent
+  # = Agents
+  #
+  # Agents are semi-autonomous bots that can respond to user questions and use available to them Tools to provide informed replies. They break down problems into series of steps and define Actions (and Action Inputs) along the way that are executed and fed back to them as additional information. Once an Agent decides that it has the Final Answer it responds with it.
+  #
+  # Available:
+  # - {Langchain::Agent::ChainOfThoughtAgent}
+  #
+  # @abstract
   class Base
   end
 end

data/lib/langchain/agent/chain_of_thought_agent/chain_of_thought_agent.rb

@@ -1,6 +1,20 @@
 # frozen_string_literal: true
 
 module Langchain::Agent
+  # = Chain of Thought Agent
+  #
+  #     llm = Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"]) # or your choice of Langchain::LLM::Base implementation
+  #
+  #     agent = Langchain::Agent::ChainOfThoughtAgent.new(
+  #       llm: llm,
+  #       tools: ["search", "calculator", "wikipedia"]
+  #     )
+  #
+  #     agent.tools
+  #     # => ["search", "calculator", "wikipedia"]
+  #
+  #     agent.run(question: "How many full soccer fields would be needed to cover the distance between NYC and DC in a straight line?")
+  #     #=> "Approximately 2,945 soccer fields would be needed to cover the distance between NYC and DC in a straight line."
   class ChainOfThoughtAgent < Base
     attr_reader :llm, :tools
 
@@ -87,7 +101,7 @@ module Langchain::Agent
         tool_names: "[#{tool_list.join(", ")}]",
         tools: tools.map do |tool|
           tool_name = tool.tool_name
-          tool_description = tool.class.const_get(:DESCRIPTION)
+          tool_description = tool.tool_description
           "#{tool_name}: #{tool_description}"
         end.join("\n")
       )

data/lib/langchain/chat.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Langchain
+  class Chat
+    attr_reader :context
+
+    def initialize(llm:, **options)
+      @llm = llm
+      @context = nil
+      @examples = []
+      @messages = []
+    end
+
+    # Set the context of the conversation. Usually used to set the model's persona.
+    # @param message [String] The context of the conversation
+    def set_context(message)
+      @context = message
+    end
+
+    # Add examples to the conversation. Used to give the model a sense of the conversation.
+    # @param examples [Array<Hash>] The examples to add to the conversation
+    def add_examples(examples)
+      @examples.concat examples
+    end
+
+    # Message the model with a prompt and return the response.
+    # @param message [String] The prompt to message the model with
+    # @return [String] The response from the model
+    def message(message)
+      append_user_message(message)
+      response = llm_response(message)
+      append_ai_message(response)
+      response
+    end
+
+    private
+
+    def llm_response(prompt)
+      @llm.chat(messages: @messages, context: @context, examples: @examples)
+    end
+
+    def append_ai_message(message)
+      @messages << {role: "ai", content: message}
+    end
+
+    def append_user_message(message)
+      @messages << {role: "user", content: message}
+    end
+  end
+end

data/lib/langchain/llm/ai21.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  #
+  # Wrapper around AI21 Studio APIs.
+  #
+  # Gem requirements:
+  #   gem "ai21", "~> 0.2.0"
+  #
+  # Usage:
+  #     ai21 = Langchain::LLM::AI21.new(api_key:)
+  #
   class AI21 < Base
-    #
-    # Wrapper around AI21 Studio APIs.
-    #
-    # Gem requirements: gem "ai21", "~> 0.2.0"
-    #
-    # Usage:
-    # ai21 = Langchain::LLM::AI21.new(api_key:)
-    #
-
     def initialize(api_key:)
       depends_on "ai21"
       require "ai21"

data/lib/langchain/llm/base.rb

@@ -1,31 +1,58 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  # A LLM is a language model consisting of a neural network with many parameters (typically billions of weights or more), trained on large quantities of unlabeled text using self-supervised learning or semi-supervised learning.
+  #
+  # Langchain.rb provides a common interface to interact with all supported LLMs:
+  #
+  # - {Langchain::LLM::AI21}
+  # - {Langchain::LLM::Cohere}
+  # - {Langchain::LLM::GooglePalm}
+  # - {Langchain::LLM::HuggingFace}
+  # - {Langchain::LLM::OpenAI}
+  # - {Langchain::LLM::Replicate}
+  #
+  # @abstract
   class Base
     include Langchain::DependencyHelper
 
+    # A client for communicating with the LLM
     attr_reader :client
 
     def default_dimension
       self.class.const_get(:DEFAULTS).dig(:dimension)
     end
 
-    # Method supported by an LLM that generates a response for a given chat-style prompt
+    #
+    # Generate a chat completion for a given prompt. Parameters will depend on the LLM
+    #
+    # @raise NotImplementedError if not supported by the LLM
     def chat(...)
       raise NotImplementedError, "#{self.class.name} does not support chat"
     end
 
-    # Method supported by an LLM that completes a given prompt
+    #
+    # Generate a completion for a given prompt. Parameters will depend on the LLM.
+    #
+    # @raise NotImplementedError if not supported by the LLM
     def complete(...)
       raise NotImplementedError, "#{self.class.name} does not support completion"
     end
 
-    # Method supported by an LLM that generates an embedding for a given text or array of texts
+    #
+    # Generate an embedding for a given text. Parameters depends on the LLM.
+    #
+    # @raise NotImplementedError if not supported by the LLM
+    #
     def embed(...)
       raise NotImplementedError, "#{self.class.name} does not support generating embeddings"
     end
 
-    # Method supported by an LLM that summarizes a given text
+    #
+    # Generate a summary for a given text. Parameters depends on the LLM.
+    #
+    # @raise NotImplementedError if not supported by the LLM
+    #
     def summarize(...)
       raise NotImplementedError, "#{self.class.name} does not support summarization"
     end

data/lib/langchain/llm/cohere.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  #
+  # Wrapper around the Cohere API.
+  #
+  # Gem requirements:
+  #     gem "cohere-ruby", "~> 0.9.4"
+  #
+  # Usage:
+  #     cohere = Langchain::LLM::Cohere.new(api_key: "YOUR_API_KEY")
+  #
   class Cohere < Base
-    #
-    # Wrapper around the Cohere API.
-    #
-    # Gem requirements: gem "cohere-ruby", "~> 0.9.4"
-    #
-    # Usage:
-    # cohere = Langchain::LLM::Cohere.new(api_key: "YOUR_API_KEY")
-    #
-
     DEFAULTS = {
       temperature: 0.0,
       completion_model_name: "base",
@@ -43,6 +43,7 @@ module Langchain::LLM
     # Generate a completion for a given prompt
     #
     # @param prompt [String] The prompt to generate a completion for
+    # @param params[:stop_sequences]
     # @return [Hash] The completion
     #
     def complete(prompt:, **params)

data/lib/langchain/llm/google_palm.rb

@@ -1,11 +1,20 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  #
+  # Wrapper around the Google PaLM (Pathways Language Model) APIs: https://ai.google/build/machine-learning/
+  #
+  # Gem requirements:
+  #     gem "google_palm_api", "~> 0.1.0"
+  #
+  # Usage:
+  #     google_palm = Langchain::LLM::GooglePalm.new(api_key: "YOUR_API_KEY")
+  #
   class GooglePalm < Base
     #
     # Wrapper around the Google PaLM (Pathways Language Model) APIs.
     #
-    # Gem requirements: gem "google_palm_api", "~> 0.1.0"
+    # Gem requirements: gem "google_palm_api", "~> 0.1.1"
     #
     # Usage:
     # google_palm = Langchain::LLM::GooglePalm.new(api_key: "YOUR_API_KEY")
@@ -40,6 +49,7 @@ module Langchain::LLM
     # Generate a completion for a given prompt
     #
     # @param prompt [String] The prompt to generate a completion for
+    # @param params extra parameters passed to GooglePalmAPI::Client#generate_text
     # @return [String] The completion
     #
     def complete(prompt:, **params)
@@ -66,26 +76,35 @@ module Langchain::LLM
     # Generate a chat completion for a given prompt
     #
     # @param prompt [String] The prompt to generate a chat completion for
+    # @param messages [Array] The messages that have been sent in the conversation
+    # @param params extra parameters passed to GooglePalmAPI::Client#generate_chat_message
     # @return [String] The chat completion
     #
-    def chat(prompt:, **params)
-      # TODO: Figure out how to introduce persisted conversations
+    def chat(prompt: "", messages: [], context: "", examples: [], **options)
+      raise ArgumentError.new(":prompt or :messages argument is expected") if prompt.empty? && messages.empty?
+
       default_params = {
-        prompt: prompt,
-        temperature: DEFAULTS[:temperature]
+        temperature: DEFAULTS[:temperature],
+        context: context,
+        messages: compose_chat_messages(prompt: prompt, messages: messages),
+        examples: compose_examples(examples)
       }
 
-      if params[:stop_sequences]
-        default_params[:stop] = params.delete(:stop_sequences)
+      Langchain::Utils::TokenLength::GooglePalmValidator.validate_max_tokens!(self, default_params[:messages], "chat-bison-001")
+
+      if options[:stop_sequences]
+        default_params[:stop] = options.delete(:stop_sequences)
       end
 
-      if params[:max_tokens]
-        default_params[:max_output_tokens] = params.delete(:max_tokens)
+      if options[:max_tokens]
+        default_params[:max_output_tokens] = options.delete(:max_tokens)
       end
 
-      default_params.merge!(params)
+      default_params.merge!(options)
 
       response = client.generate_chat_message(**default_params)
+      raise "GooglePalm API returned an error: #{response}" if response.dig("error")
+
       response.dig("candidates", 0, "content")
     end
 
@@ -108,5 +127,39 @@ module Langchain::LLM
         max_tokens: 2048
       )
     end
+
+    private
+
+    def compose_chat_messages(prompt:, messages:)
+      history = []
+      history.concat transform_messages(messages) unless messages.empty?
+
+      unless prompt.empty?
+        if history.last && history.last[:role] == "user"
+          history.last[:content] += "\n#{prompt}"
+        else
+          history.append({author: "user", content: prompt})
+        end
+      end
+      history
+    end
+
+    def compose_examples(examples)
+      examples.each_slice(2).map do |example|
+        {
+          input: {content: example.first[:content]},
+          output: {content: example.last[:content]}
+        }
+      end
+    end
+
+    def transform_messages(messages)
+      messages.map do |message|
+        {
+          author: message[:role],
+          content: message[:content]
+        }
+      end
+    end
   end
 end

data/lib/langchain/llm/hugging_face.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  #
+  # Wrapper around the HuggingFace Inference API: https://huggingface.co/inference-api
+  #
+  # Gem requirements:
+  #     gem "hugging-face", "~> 0.3.4"
+  #
+  # Usage:
+  #     hf = Langchain::LLM::HuggingFace.new(api_key: "YOUR_API_KEY")
+  #
   class HuggingFace < Base
-    #
-    # Wrapper around the HuggingFace Inference API.
-    #
-    # Gem requirements: gem "hugging-face", "~> 0.3.4"
-    #
-    # Usage:
-    # hf = Langchain::LLM::HuggingFace.new(api_key: "YOUR_API_KEY")
-    #
-
     # The gem does not currently accept other models:
     # https://github.com/alchaplinsky/hugging-face/blob/main/lib/hugging_face/inference_api.rb#L32-L34
     DEFAULTS = {

data/lib/langchain/llm/openai.rb

@@ -1,16 +1,15 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  # LLM interface for OpenAI APIs: https://platform.openai.com/overview
+  #
+  # Gem requirements:
+  #    gem "ruby-openai", "~> 4.0.0"
+  #
+  # Usage:
+  #    openai = Langchain::LLM::OpenAI.new(api_key:, llm_options: {})
+  #
   class OpenAI < Base
-    #
-    # Wrapper around OpenAI APIs.
-    #
-    # Gem requirements: gem "ruby-openai", "~> 4.0.0"
-    #
-    # Usage:
-    # openai = Langchain::LLM::OpenAI.new(api_key:, llm_options: {})
-    #
-
     DEFAULTS = {
       temperature: 0.0,
       completion_model_name: "text-davinci-003",
@@ -30,12 +29,13 @@ module Langchain::LLM
     # Generate an embedding for a given text
     #
     # @param text [String] The text to generate an embedding for
+    # @param params extra parameters passed to OpenAI::Client#embeddings
     # @return [Array] The embedding
     #
     def embed(text:, **params)
       parameters = {model: DEFAULTS[:embeddings_model_name], input: text}
 
-      Langchain::Utils::TokenLengthValidator.validate_max_tokens!(text, parameters[:model])
+      Langchain::Utils::TokenLength::OpenAIValidator.validate_max_tokens!(text, parameters[:model])
 
       response = client.embeddings(parameters: parameters.merge(params))
       response.dig("data").first.dig("embedding")
@@ -45,13 +45,14 @@ module Langchain::LLM
     # Generate a completion for a given prompt
     #
     # @param prompt [String] The prompt to generate a completion for
+    # @param params  extra parameters passed to OpenAI::Client#complete
     # @return [String] The completion
     #
     def complete(prompt:, **params)
       parameters = compose_parameters DEFAULTS[:completion_model_name], params
 
       parameters[:prompt] = prompt
-      parameters[:max_tokens] = Langchain::Utils::TokenLengthValidator.validate_max_tokens!(prompt, parameters[:model])
+      parameters[:max_tokens] = Langchain::Utils::TokenLength::OpenAIValidator.validate_max_tokens!(prompt, parameters[:model])
 
       response = client.completions(parameters: parameters)
       response.dig("choices", 0, "text")
@@ -61,15 +62,23 @@ module Langchain::LLM
     # Generate a chat completion for a given prompt
     #
     # @param prompt [String] The prompt to generate a chat completion for
+    # @param messages [Array] The messages that have been sent in the conversation
+    # @param context [String] The context of the conversation
+    # @param examples [Array] Examples of messages provide model with
+    # @param options extra parameters passed to OpenAI::Client#chat
     # @return [String] The chat completion
     #
-    def chat(prompt:, **params)
-      parameters = compose_parameters DEFAULTS[:chat_completion_model_name], params
+    def chat(prompt: "", messages: [], context: "", examples: [], **options)
+      raise ArgumentError.new(":prompt or :messages argument is expected") if prompt.empty? && messages.empty?
 
-      parameters[:messages] = [{role: "user", content: prompt}]
-      parameters[:max_tokens] = Langchain::Utils::TokenLengthValidator.validate_max_tokens!(prompt, parameters[:model])
+      parameters = compose_parameters DEFAULTS[:chat_completion_model_name], options
+      parameters[:messages] = compose_chat_messages(prompt: prompt, messages: messages, context: context, examples: examples)
+      parameters[:max_tokens] = validate_max_tokens(parameters[:messages], parameters[:model])
 
       response = client.chat(parameters: parameters)
+
+      raise "Chat completion failed: #{response}" if response.dig("error")
+
       response.dig("choices", 0, "message", "content")
     end
 
@@ -97,5 +106,41 @@ module Langchain::LLM
 
       default_params.merge(params)
     end
+
+    def compose_chat_messages(prompt:, messages:, context:, examples:)
+      history = []
+
+      history.concat transform_messages(examples) unless examples.empty?
+
+      history.concat transform_messages(messages) unless messages.empty?
+
+      unless context.nil? || context.empty?
+        history.reject! { |message| message[:role] == "system" }
+        history.prepend({role: "system", content: context})
+      end
+
+      unless prompt.empty?
+        if history.last && history.last[:role] == "user"
+          history.last[:content] += "\n#{prompt}"
+        else
+          history.append({role: "user", content: prompt})
+        end
+      end
+
+      history
+    end
+
+    def transform_messages(messages)
+      messages.map do |message|
+        {
+          content: message[:content],
+          role: (message[:role] == "ai") ? "assistant" : message[:role]
+        }
+      end
+    end
+
+    def validate_max_tokens(messages, model)
+      Langchain::Utils::TokenLength::OpenAIValidator.validate_max_tokens!(messages, model)
+    end
   end
 end

data/lib/langchain/llm/replicate.rb

@@ -1,22 +1,23 @@
 # frozen_string_literal: true
 
 module Langchain::LLM
+  #
+  # Wrapper around Replicate.com LLM provider
+  #
+  # Gem requirements:
+  #     gem "replicate-ruby", "~> 0.2.2"
+  #
+  # Use it directly:
+  #     replicate = Langchain::LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
+  #
+  # Or pass it to be used by a vector search DB:
+  #     chroma = Langchain::Vectorsearch::Chroma.new(
+  #       url: ENV["CHROMA_URL"],
+  #       index_name: "...",
+  #       llm: replicate
+  #     )
+  #
   class Replicate < Base
-    #
-    # Wrapper around Replicate.com LLM provider
-    #
-    # Gem requirements: gem "replicate-ruby", "~> 0.2.2"
-    #
-    # Use it directly:
-    # replicate = LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
-    #
-    # Or pass it to be instantiated by a vector search DB:
-    # chroma = Vectorsearch::Chroma.new(
-    #   url: ENV["CHROMA_URL"],
-    #   index_name: "...",
-    #   llm: Langchain::LLM::Replicate(api_key: ENV["REPLICATE_API_KEY"])
-    # )
-
     DEFAULTS = {
       # TODO: Figure out how to send the temperature to the API
       temperature: 0.01, # Minimum accepted value

data/lib/langchain/processors/base.rb

@@ -2,6 +2,7 @@
 
 module Langchain
   module Processors
+    # Processors load and parse/process various data types such as CSVs, PDFs, Word documents, HTML pages, and others.
     class Base
       include Langchain::DependencyHelper