langchainrb 0.3.13 → 0.3.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ee811b2bac8fadea4d90c4212363a901829a4aac219da0f2a2dcbe7c6f59c5b
-  data.tar.gz: 8fa32e6df4aaf69cb6d29977913c1b8a30d6f65b777b1f90c8a7f504d869ca8f
+  metadata.gz: 4f855e3c0e1f0d7b59e0255004a1a806c7048da6d3fe0a8ddf10be68e36ed9ba
+  data.tar.gz: 5758c90205c3e2bea420cf7fa0dec07638917beced60bd0482e1d803ced96c07
 SHA512:
-  metadata.gz: cbb7e0c975333248c01082a47f7096fb9d6807c3b7619424eb9348238008d7b4257518287d9358114bf4e3a589349520ebf71ace00bf1fe8906afd27e8b1418a
-  data.tar.gz: 759444abe0b17518c6ef31fed6980f6bc0d3d096606860c4d6fddb8baeda4e0a23fc3909e42eba0f32912a786abec76cac54384533db2787e05d741f0907fa1d
+  metadata.gz: ec26f8c4257a6949d829d7f68d2175943b80c2837739bdbead8e6b61891a6738cf84ec1934caff777b63dee39f3d0111e8f29a81c37c125be1cd3ae8137b6968
+  data.tar.gz: 31bb1aa0296dbbc8e1a1c6c2bc7236c92bf3e627935c752bd527c7f7d769da69f11c6bb63de6e0b4e0d1d8d0c739744045669220c35e253ae83eaa0e23482e6a

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 ## [Unreleased]
 
+## [0.3.15] - 2023-05-30
+- Drop Ruby 2.7 support. It had reached EOD.
+- Bump pgvector-ruby to 0.2
+- 🚚 Loaders
+  - Support for options and block to be passed to CSV processor
+
+## [0.3.14] - 2023-05-28
+- 🔍 Vectorsearch
+  - Not relying on Weaviate modules anymore
+  - Adding missing specs for Qdrant and Milvus classes
+- 🚚 Loaders
+  - Add Langchain::Data result object for data loaders
+- 🗣️ LLMs
+  - Add `summarize()` method to the LLMs
+
 ## [0.3.13] - 2023-05-26
 - 🔍 Vectorsearch
   - Pgvector support

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.3.13)
+    langchainrb (0.3.15)
 
 GEM
   remote: https://rubygems.org/
@@ -171,7 +171,7 @@ GEM
       ruby-rc4
       ttfunk
     pg (1.5.3)
-    pgvector (0.1.1)
+    pgvector (0.2.0)
     pinecone (0.1.71)
       dry-struct (~> 1.6.0)
       dry-validation (~> 1.10.0)
@@ -298,7 +298,7 @@ DEPENDENCIES
   nokogiri (~> 1.13)
   pdf-reader (~> 1.4)
   pg (~> 1.5)
-  pgvector (< 0.2)
+  pgvector (~> 0.2)
   pinecone (~> 0.1.6)
   pry-byebug (~> 3.10.0)
   qdrant-ruby (~> 0.9.0)

data/README.md

@@ -321,6 +321,7 @@ Langchain.logger.level = :info
 [<img style="border-radius:50%" alt="Andrei Bondarev" src="https://avatars.githubusercontent.com/u/541665?v=4" width="80" height="80" class="avatar">](https://github.com/andreibondarev)
 [<img style="border-radius:50%" alt="Rafael Figueiredo" src="https://avatars.githubusercontent.com/u/35845775?v=4" width="80" height="80" class="avatar">](https://github.com/rafaelqfigueiredo)
 [<img style="border-radius:50%" alt="Ricky Chilcott" src="https://avatars.githubusercontent.com/u/445759?v=4" width="80" height="80" class="avatar">](https://github.com/rickychilcott)
+[<img style="border-radius:50%" alt="Alex Chaplinsky" src="https://avatars.githubusercontent.com/u/695947?v=4" width="80" height="80" class="avatar">](https://github.com/alchaplinsky)
 
 (Criteria for becoming an Honorary Contributor or Core Contributor is pending...)
 

data/lib/langchain/data.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Langchain
+  class Data
+    attr_reader :source
+
+    def initialize(data, options = {})
+      @source = options[:source]
+      @data = data
+    end
+
+    def value
+      @data
+    end
+  end
+end

data/lib/langchain/loader.rb

@@ -14,14 +14,15 @@ module Langchain
     # Equivalent to Langchain::Loader.new(path).load
     # @param path [String | Pathname] path to file or url
     # @return [String] file content
-    def self.load(path)
-      new(path).load
+    def self.load(path, options = {}, &block)
+      new(path, options).load(&block)
     end
 
     # Initialize Langchain::Loader
     # @param path [String | Pathname] path to file or url
     # @return [Langchain::Loader] loader instance
-    def initialize(path)
+    def initialize(path, options = {})
+      @options = options
       @path = path
     end
 
@@ -35,38 +36,38 @@ module Langchain
 
     # Load data from a file or url
     # @return [String] file content
-    def load
-      url? ? from_url(@path) : from_path(@path)
+    def load(&block)
+      @raw_data = url? ? load_from_url : load_from_path
+
+      data = if block
+        yield @raw_data.read, @options
+      else
+        processor_klass.new(@options).parse(@raw_data)
+      end
+
+      Langchain::Data.new(data, source: @path)
     end
 
     private
 
-    def from_url(url)
-      process do
-        data = URI.parse(url).open
-        processor = find_processor(:CONTENT_TYPES, data.content_type)
-        [data, processor]
-      end
+    def load_from_url
+      URI.parse(@path).open
     end
 
-    def from_path(path)
-      raise FileNotFound unless File.exist?(path)
+    def load_from_path
+      raise FileNotFound unless File.exist?(@path)
 
-      process do
-        [File.open(path), find_processor(:EXTENSIONS, File.extname(path))]
-      end
+      File.open(@path)
     end
 
-    def process(&block)
-      data, processor = yield
+    def processor_klass
+      raise UnknownFormatError unless (kind = find_processor)
 
-      raise UnknownFormatError unless processor
-
-      Langchain::Processors.const_get(processor).new.parse(data)
+      Langchain::Processors.const_get(kind)
     end
 
-    def find_processor(constant, value)
-      processors.find { |klass| processor_matches? "#{klass}::#{constant}", value }
+    def find_processor
+      processors.find { |klass| processor_matches? "#{klass}::#{lookup_constant}", source_type }
     end
 
     def processor_matches?(constant, value)
@@ -76,5 +77,13 @@ module Langchain
     def processors
       Langchain::Processors.constants
     end
+
+    def source_type
+      url? ? @raw_data.content_type : File.extname(@path)
+    end
+
+    def lookup_constant
+      url? ? :CONTENT_TYPES : :EXTENSIONS
+    end
   end
 end

data/lib/langchain/processors/base.rb

@@ -6,6 +6,10 @@ module Langchain
       EXTENSIONS = []
       CONTENT_TYPES = []
 
+      def initialize(options = {})
+        @options = options
+      end
+
       def parse(data)
         raise NotImplementedError
       end

data/lib/langchain/processors/csv.rb

@@ -12,10 +12,16 @@ module Langchain
       # @param [File] data
       # @return [Array of Hash]
       def parse(data)
-        ::CSV.new(data.read).map do |row|
+        ::CSV.new(data.read, col_sep: separator).map do |row|
           row.map(&:strip)
         end
       end
+
+      private
+
+      def separator
+        @options[:col_sep] || ","
+      end
     end
   end
 end

data/lib/langchain/processors/docx.rb

@@ -6,7 +6,7 @@ module Langchain
       EXTENSIONS = [".docx"]
       CONTENT_TYPES = ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]
 
-      def initialize
+      def initialize(*)
         depends_on "docx"
         require "docx"
       end

data/lib/langchain/processors/html.rb

@@ -9,7 +9,7 @@ module Langchain
       # We only look for headings and paragraphs
       TEXT_CONTENT_TAGS = %w[h1 h2 h3 h4 h5 h6 p]
 
-      def initialize
+      def initialize(*)
         depends_on "nokogiri"
         require "nokogiri"
       end

data/lib/langchain/processors/pdf.rb

@@ -6,7 +6,7 @@ module Langchain
       EXTENSIONS = [".pdf"]
       CONTENT_TYPES = ["application/pdf"]
 
-      def initialize
+      def initialize(*)
         depends_on "pdf-reader"
         require "pdf-reader"
       end

data/lib/langchain.rb

@@ -17,6 +17,7 @@ module Langchain
   @root = Pathname.new(__dir__)
 
   autoload :Loader, "langchain/loader"
+  autoload :Data, "langchain/data"
 
   module Processors
     autoload :Base, "langchain/processors/base"

data/lib/llm/base.rb

@@ -33,6 +33,11 @@ module LLM
       raise NotImplementedError, "#{self.class.name} does not support generating embeddings"
     end
 
+    # Method supported by an LLM that summarizes a given text
+    def summarize(...)
+      raise NotImplementedError, "#{self.class.name} does not support summarization"
+    end
+
     # Ensure that the LLM value passed in is supported
     # @param llm [Symbol] The LLM to use
     def self.validate_llm!(llm:)

data/lib/llm/cohere.rb

@@ -16,9 +16,12 @@ module LLM
       @client = ::Cohere::Client.new(api_key: api_key)
     end
 
+    #
     # Generate an embedding for a given text
+    #
     # @param text [String] The text to generate an embedding for
     # @return [Hash] The embedding
+    #
     def embed(text:)
       response = client.embed(
         texts: [text],
@@ -27,9 +30,12 @@ module LLM
       response.dig("embeddings").first
     end
 
+    #
     # Generate a completion for a given prompt
+    #
     # @param prompt [String] The prompt to generate a completion for
     # @return [Hash] The completion
+    #
     def complete(prompt:, **params)
       default_params = {
         prompt: prompt,
@@ -51,5 +57,16 @@ module LLM
     def chat(...)
       complete(...)
     end
+
+    # Generate a summary in English for a given text
+    #
+    # More parameters available to extend this method with: https://github.com/andreibondarev/cohere-ruby/blob/0.9.4/lib/cohere/client.rb#L107-L115
+    #
+    # @param text [String] The text to generate a summary for
+    # @return [String] The summary
+    def summarize(text:)
+      response = client.summarize(text: text)
+      response.dig("summary")
+    end
   end
 end

data/lib/llm/google_palm.rb

@@ -81,5 +81,25 @@ module LLM
       response = client.generate_chat_message(**default_params)
       response.dig("candidates", 0, "content")
     end
+
+    #
+    # Generate a summarization for a given text
+    #
+    # @param text [String] The text to generate a summarization for
+    # @return [String] The summarization
+    #
+    def summarize(text:)
+      prompt_template = Prompt.load_from_path(
+        file_path: Langchain.root.join("llm/prompts/summarize_template.json")
+      )
+      prompt = prompt_template.format(text: text)
+
+      complete(
+        prompt: prompt,
+        temperature: DEFAULTS[:temperature],
+        # Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
+        max_tokens: 2048
+      )
+    end
   end
 end

data/lib/llm/hugging_face.rb

@@ -12,6 +12,7 @@ module LLM
 
     #
     # Intialize the HuggingFace LLM
+    #
     # @param api_key [String] The API key to use
     #
     def initialize(api_key:)
@@ -21,9 +22,12 @@ module LLM
       @client = ::HuggingFace::InferenceApi.new(api_token: api_key)
     end
 
+    #
     # Generate an embedding for a given text
+    #
     # @param text [String] The text to embed
     # @return [Array] The embedding
+    #
     def embed(text:)
       client.embedding(
         input: text,

data/lib/llm/openai.rb

@@ -18,9 +18,12 @@ module LLM
       @client = ::OpenAI::Client.new(access_token: api_key)
     end
 
+    #
     # Generate an embedding for a given text
+    #
     # @param text [String] The text to generate an embedding for
     # @return [Array] The embedding
+    #
     def embed(text:)
       response = client.embeddings(
         parameters: {
@@ -31,9 +34,12 @@ module LLM
       response.dig("data").first.dig("embedding")
     end
 
+    #
     # Generate a completion for a given prompt
+    #
     # @param prompt [String] The prompt to generate a completion for
     # @return [String] The completion
+    #
     def complete(prompt:, **params)
       default_params = {
         model: DEFAULTS[:completion_model_name],
@@ -51,9 +57,12 @@ module LLM
       response.dig("choices", 0, "text")
     end
 
+    #
     # Generate a chat completion for a given prompt
+    #
     # @param prompt [String] The prompt to generate a chat completion for
     # @return [String] The chat completion
+    #
     def chat(prompt:, **params)
       default_params = {
         model: DEFAULTS[:chat_completion_model_name],
@@ -71,5 +80,25 @@ module LLM
       response = client.chat(parameters: default_params)
       response.dig("choices", 0, "message", "content")
     end
+
+    #
+    # Generate a summary for a given text
+    #
+    # @param text [String] The text to generate a summary for
+    # @return [String] The summary
+    #
+    def summarize(text:)
+      prompt_template = Prompt.load_from_path(
+        file_path: Langchain.root.join("llm/prompts/summarize_template.json")
+      )
+      prompt = prompt_template.format(text: text)
+
+      complete(
+        prompt: prompt,
+        temperature: DEFAULTS[:temperature],
+        # Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
+        max_tokens: 2048
+      )
+    end
   end
 end

data/lib/llm/prompts/summarize_template.json

@@ -0,0 +1,5 @@
+{
+  "_type": "prompt",
+  "input_variables": ["text"],
+  "template": "Write a concise summary of the following:\n\n{text}\n\nCONCISE SUMMARY:"
+}

data/lib/llm/replicate.rb

@@ -23,8 +23,11 @@ module LLM
       dimension: 384
     }.freeze
 
+    #
     # Intialize the Replicate LLM
+    #
     # @param api_key [String] The API key to use
+    #
     def initialize(api_key:)
       depends_on "replicate-ruby"
       require "replicate"
@@ -36,9 +39,12 @@ module LLM
       @client = ::Replicate.client
     end
 
+    #
     # Generate an embedding for a given text
+    #
     # @param text [String] The text to generate an embedding for
     # @return [Hash] The embedding
+    #
     def embed(text:)
       response = embeddings_model.predict(input: text)
 
@@ -50,9 +56,12 @@ module LLM
       response.output
     end
 
+    #
     # Generate a completion for a given prompt
+    #
     # @param prompt [String] The prompt to generate a completion for
     # @return [Hash] The completion
+    #
     def complete(prompt:, **params)
       response = completion_model.predict(prompt: prompt)
 
@@ -73,6 +82,26 @@ module LLM
       complete(...)
     end
 
+    #
+    # Generate a summary for a given text
+    #
+    # @param text [String] The text to generate a summary for
+    # @return [String] The summary
+    #
+    def summarize(text:)
+      prompt_template = Prompt.load_from_path(
+        file_path: Langchain.root.join("llm/prompts/summarize_template.json")
+      )
+      prompt = prompt_template.format(text: text)
+
+      complete(
+        prompt: prompt,
+        temperature: DEFAULTS[:temperature],
+        # Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
+        max_tokens: 2048
+      )
+    end
+
     alias_method :generate_embedding, :embed
 
     private

data/lib/vectorsearch/base.rb

@@ -74,7 +74,7 @@ module Vectorsearch
 
       texts = Array(path || paths)
         .flatten
-        .map { |path| Langchain::Loader.new(path)&.load }
+        .map { |path| Langchain::Loader.new(path)&.load&.value }
         .compact
 
       add_texts(texts: texts)

data/lib/vectorsearch/weaviate.rb

@@ -14,9 +14,7 @@ module Vectorsearch
 
       @client = ::Weaviate::Client.new(
         url: url,
-        api_key: api_key,
-        model_service: llm,
-        model_service_api_key: llm_api_key
+        api_key: api_key
       )
       @index_name = index_name
 
@@ -30,7 +28,8 @@ module Vectorsearch
       objects = Array(texts).map do |text|
         {
           class: index_name,
-          properties: {content: text}
+          properties: {content: text},
+          vector: llm_client.embed(text: text)
         }
       end
 
@@ -43,11 +42,7 @@ module Vectorsearch
     def create_default_schema
       client.schema.create(
         class_name: index_name,
-        vectorizer: "text2vec-#{llm}",
-        # TODO: Figure out a way to optionally enable it
-        # "module_config": {
-        #   "qna-openai": {}
-        # },
+        vectorizer: "none",
         properties: [
           # TODO: Allow passing in your own IDs
           {
@@ -63,14 +58,9 @@ module Vectorsearch
     # @param k [Integer|String] The number of results to return
     # @return [Hash] The search results
     def similarity_search(query:, k: 4)
-      near_text = "{ concepts: [\"#{query}\"] }"
+      embedding = llm_client.embed(text: query)
 
-      client.query.get(
-        class_name: index_name,
-        near_text: near_text,
-        limit: k.to_s,
-        fields: "content _additional { id }"
-      )
+      similarity_search_by_vector(embedding: embedding, k: k)
     end
 
     # Return documents similar to the vector
@@ -92,29 +82,16 @@ module Vectorsearch
     # @param question [String] The question to ask
     # @return [Hash] The answer
     def ask(question:)
-      # Weaviate currently supports the `ask:` parameter only for the OpenAI LLM (with `qna-openai` module enabled).
-      # The Cohere support is on the way: https://github.com/weaviate/weaviate/pull/2600
-      if llm == :openai
-        ask_object = "{ question: \"#{question}\" }"
-
-        client.query.get(
-          class_name: index_name,
-          ask: ask_object,
-          limit: "1",
-          fields: "_additional { answer { result } }"
-        )
-      elsif llm == :cohere
-        search_results = similarity_search(query: question)
+      search_results = similarity_search(query: question)
 
-        context = search_results.map do |result|
-          result.dig("content").to_s
-        end
-        context = context.join("\n---\n")
+      context = search_results.map do |result|
+        result.dig("content").to_s
+      end
+      context = context.join("\n---\n")
 
-        prompt = generate_prompt(question: question, context: context)
+      prompt = generate_prompt(question: question, context: context)
 
-        llm_client.chat(prompt: prompt)
-      end
+      llm_client.chat(prompt: prompt)
     end
   end
 end

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.3.13"
+  VERSION = "0.3.15"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.3.15
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-26 00:00:00.000000000 Z
+date: 2023-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv-rails
@@ -182,14 +182,14 @@ dependencies:
   name: pgvector
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
 - !ruby/object:Gem::Dependency
@@ -290,7 +290,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.17.0
-description: Build ML/AI-powered applications with Ruby's LangChain
+description: Build LLM-backed Ruby applications with Ruby's LangChain
 email:
 - andrei.bondarev13@gmail.com
 executables: []
@@ -316,6 +316,7 @@ files:
 - lib/agent/chain_of_thought_agent/chain_of_thought_agent_prompt.json
 - lib/dependency_helper.rb
 - lib/langchain.rb
+- lib/langchain/data.rb
 - lib/langchain/loader.rb
 - lib/langchain/processors/base.rb
 - lib/langchain/processors/csv.rb
@@ -331,6 +332,7 @@ files:
 - lib/llm/google_palm.rb
 - lib/llm/hugging_face.rb
 - lib/llm/openai.rb
+- lib/llm/prompts/summarize_template.json
 - lib/llm/replicate.rb
 - lib/prompt/base.rb
 - lib/prompt/few_shot_prompt_template.rb
@@ -364,15 +366,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
-summary: Build ML/AI-powered applications with Ruby's LangChain
+summary: Build LLM-backed Ruby applications with Ruby's LangChain
 test_files: []