langchainrb 0.8.1 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5dd13c5aae47af13fe248636ed88bd40d0e241291ab5c3dc2d5925dcc742af37
-  data.tar.gz: b190f73403a77b4ea4d1f9869423546d584df32785ae342a01d9a72ee5fe04fd
+  metadata.gz: 13eec34cc529732ddfb8994956659bd4307a79ebfd76ff883fe3b6644d647c24
+  data.tar.gz: ce04acfe42a6a8da5a5951734651dd0083f7d2efc43cf4b3367710c8221ee96a
 SHA512:
-  metadata.gz: 81dd80f49173e3d711a713b6dd365addf04129cb0f6c015d6909200a709780e30c39888f0bccba72035e03c17a0b01a4d1456e6431473149d9969907435f18c1
-  data.tar.gz: 748f841cf01b802e81bc6f6ecf8aaea5ab13593363afadc7c9634446c169812064dd41af3e58e87068a224972be85f00b1e3c2669a99e1406819507c86b1a15c
+  metadata.gz: 2094d99610311a1583d890f8c6898605bcd3e76d2fb72deb1ccd4b250f2b98f7a883401faf2e161b97b82fb29f6e64ead8843d8af22f0bd3e8a4c872c150c134
+  data.tar.gz: d7ce155cbb992e651aa8dc468ed1ee39bd96d1457f50faa11a32d7caac87086f5d8a381fc2b50aaba10ac934486ed415d5e609f47ee0426b4187540e2436b2e9

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [0.8.2]
+- Introducing new `Langchain::Chunker::Markdown` chunker (thanks @spikex)
+- Fixes
+
 ## [0.8.1]
 - Support for Epsilla vector DB
 - Fully functioning Google Vertex AI LLM

data/lib/langchain/chunker/markdown.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "baran"
+
+module Langchain
+  module Chunker
+    #
+    # Simple text chunker
+    #
+    # Usage:
+    #     Langchain::Chunker::Markdown.new(text).chunks
+    #
+    class Markdown < Base
+      attr_reader :text, :chunk_size, :chunk_overlap
+
+      # @param [String] text
+      # @param [Integer] chunk_size
+      # @param [Integer] chunk_overlap
+      # @param [String] separator
+      def initialize(text, chunk_size: 1000, chunk_overlap: 200)
+        @text = text
+        @chunk_size = chunk_size
+        @chunk_overlap = chunk_overlap
+      end
+
+      # @return [Array<Langchain::Chunk>]
+      def chunks
+        splitter = Baran::MarkdownSplitter.new(
+          chunk_size: chunk_size,
+          chunk_overlap: chunk_overlap
+        )
+
+        splitter.chunks(text).map do |chunk|
+          Langchain::Chunk.new(text: chunk[:text])
+        end
+      end
+    end
+  end
+end

data/lib/langchain/data.rb

@@ -9,9 +9,10 @@ module Langchain
 
     # @param data [String] data that was loaded
     # @option options [String] :source URL or Path of the data source
-    def initialize(data, options = {})
-      @source = options[:source]
+    def initialize(data, source: nil, chunker: Langchain::Chunker::Text)
+      @source = source
       @data = data
+      @chunker_klass = chunker
     end
 
     # @return [String]
@@ -22,7 +23,7 @@ module Langchain
     # @param opts [Hash] options passed to the chunker
     # @return [Array<String>]
     def chunks(opts = {})
-      Langchain::Chunker::Text.new(@data, **opts).chunks
+      @chunker_klass.new(@data, **opts).chunks
     end
   end
 end

data/lib/langchain/llm/response/ollama_response.rb

@@ -8,7 +8,7 @@ module Langchain::LLM
     end
 
     def completion
-      raw_response.first
+      completions.first
     end
 
     def completions

data/lib/langchain/loader.rb

@@ -37,9 +37,10 @@ module Langchain
     # @param path [String | Pathname] path to file or URL
     # @param options [Hash] options passed to the processor class used to process the data
     # @return [Langchain::Loader] loader instance
-    def initialize(path, options = {})
+    def initialize(path, options = {}, chunker: Langchain::Chunker::Text)
       @options = options
       @path = path
+      @chunker = chunker
     end
 
     # Is the path a URL?
@@ -112,7 +113,7 @@ module Langchain
         processor_klass.new(@options).parse(@raw_data)
       end
 
-      Langchain::Data.new(result)
+      Langchain::Data.new(result, source: @options[:source], chunker: @chunker)
     end
 
     def processor_klass

data/lib/langchain/processors/markdown.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Processors
+    class Markdown < Base
+      EXTENSIONS = [".markdown", ".md"]
+      CONTENT_TYPES = ["text/markdown"]
+
+      # Parse the document and return the text
+      # @param [File] data
+      # @return [String]
+      def parse(data)
+        data.read
+      end
+    end
+  end
+end

data/lib/langchain/utils/token_length/ai21_validator.rb

@@ -31,6 +31,10 @@ module Langchain
           TOKEN_LIMITS[model_name]
         end
         singleton_class.alias_method :completion_token_limit, :token_limit
+
+        def self.token_length_from_messages(messages, model_name, options)
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/base_validator.rb

@@ -14,7 +14,7 @@ module Langchain
       class BaseValidator
         def self.validate_max_tokens!(content, model_name, options = {})
           text_token_length = if content.is_a?(Array)
-            content.sum { |item| token_length(item.to_json, model_name, options) }
+            token_length_from_messages(content, model_name, options)
           else
             token_length(content, model_name, options)
           end

data/lib/langchain/utils/token_length/cohere_validator.rb

@@ -39,6 +39,10 @@ module Langchain
           TOKEN_LIMITS[model_name]
         end
         singleton_class.alias_method :completion_token_limit, :token_limit
+
+        def self.token_length_from_messages(messages, model_name, options)
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
       end
     end
   end

data/lib/langchain/utils/token_length/google_palm_validator.rb

@@ -43,6 +43,10 @@ module Langchain
           response.dig("tokenCount")
         end
 
+        def self.token_length_from_messages(messages, model_name, options)
+          messages.sum { |message| token_length(message.to_json, model_name, options) }
+        end
+
         def self.token_limit(model_name)
           TOKEN_LIMITS.dig(model_name, "input_token_limit")
         end

data/lib/langchain/utils/token_length/openai_validator.rb

@@ -75,6 +75,47 @@ module Langchain
           max_tokens = super(content, model_name, options)
           [options[:max_tokens], max_tokens].reject(&:nil?).min
         end
+
+        # Copied from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+        # Return the number of tokens used by a list of messages
+        #
+        # @param messages [Array<Hash>] The messages to calculate the token length for
+        # @param model [String] The model name to validate against
+        # @return [Integer] The token length of the messages
+        #
+        def self.token_length_from_messages(messages, model_name, options = {})
+          encoding = Tiktoken.encoding_for_model(model_name)
+
+          if ["gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613", "gpt-4-0314", "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613"].include?(model_name)
+            tokens_per_message = 3
+            tokens_per_name = 1
+          elsif model_name == "gpt-3.5-turbo-0301"
+            tokens_per_message = 4  # every message follows {role/name}\n{content}\n
+            tokens_per_name = -1  # if there's a name, the role is omitted
+          elsif model_name.include?("gpt-3.5-turbo")
+            puts "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613."
+            return token_length_from_messages(messages, "gpt-3.5-turbo-0613", options)
+          elsif model_name.include?("gpt-4")
+            puts "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613."
+            return token_length_from_messages(messages, "gpt-4-0613", options)
+          else
+            raise NotImplementedError.new(
+              "token_length_from_messages() is not implemented for model #{model_name}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."
+            )
+          end
+
+          num_tokens = 0
+          messages.each do |message|
+            num_tokens += tokens_per_message
+            message.each do |key, value|
+              num_tokens += encoding.encode(value).length
+              num_tokens += tokens_per_name if ["name", :name].include?(key)
+            end
+          end
+
+          num_tokens += 3  # every reply is primed with assistant
+          num_tokens
+        end
       end
     end
   end

data/lib/langchain/vectorsearch/base.rb

@@ -175,13 +175,13 @@ module Langchain::Vectorsearch
       prompt_template.format(question: question, context: context)
     end
 
-    def add_data(paths:)
+    def add_data(paths:, options: {}, chunker: Langchain::Chunker::Text)
       raise ArgumentError, "Paths must be provided" if Array(paths).empty?
 
       texts = Array(paths)
         .flatten
         .map do |path|
-          data = Langchain::Loader.new(path)&.load&.chunks
+          data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
           data.map { |chunk| chunk.text }
         end
 

data/lib/langchain/vectorsearch/epsilla.rb

@@ -36,7 +36,11 @@ module Langchain::Vectorsearch
         status_code, response = @client.database.load_db(db_name, db_path)
 
         if status_code != 200
-          if status_code == 500 && response["message"].include?("already loaded")
+          if status_code == 409 || (status_code == 500 && response["message"].include?("already loaded"))
+            # When db is already loaded, Epsilla may return HTTP 409 Conflict.
+            # This behavior is changed in https://github.com/epsilla-cloud/vectordb/pull/95
+            # Old behavior (HTTP 500) is preserved for backwards compatibility.
+            # It does not prevent us from using the db.
             Langchain.logger.info("Database already loaded")
           else
             raise "Failed to load database: #{response}"

data/lib/langchain/vectorsearch/pinecone.rb

@@ -64,13 +64,13 @@ module Langchain::Vectorsearch
       index.upsert(vectors: vectors, namespace: namespace)
     end
 
-    def add_data(paths:, namespace: "")
+    def add_data(paths:, namespace: "", options: {}, chunker: Langchain::Chunker::Text)
       raise ArgumentError, "Paths must be provided" if Array(paths).empty?
 
       texts = Array(paths)
         .flatten
         .map do |path|
-          data = Langchain::Loader.new(path)&.load&.chunks
+          data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
           data.map { |chunk| chunk.text }
         end
 

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.8.1"
+  VERSION = "0.8.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.2
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-12-07 00:00:00.000000000 Z
+date: 2023-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: baran
@@ -618,6 +618,7 @@ files:
 - lib/langchain/agent/sql_query_agent/sql_query_agent_sql_prompt.yaml
 - lib/langchain/chunk.rb
 - lib/langchain/chunker/base.rb
+- lib/langchain/chunker/markdown.rb
 - lib/langchain/chunker/prompts/semantic_prompt_template.yml
 - lib/langchain/chunker/recursive_text.rb
 - lib/langchain/chunker/semantic.rb
@@ -677,6 +678,7 @@ files:
 - lib/langchain/processors/html.rb
 - lib/langchain/processors/json.rb
 - lib/langchain/processors/jsonl.rb
+- lib/langchain/processors/markdown.rb
 - lib/langchain/processors/pdf.rb
 - lib/langchain/processors/text.rb
 - lib/langchain/processors/xlsx.rb