langchainrb 0.3.14 → 0.3.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33c9436ac8d6a73dc06d30f63c11e4f246b3705aa8934765a53ee59325c3a9cd
-  data.tar.gz: 9cc85603694f9367dd162e25379029a345aa0b5c88cccf303c2af114d43a4010
+  metadata.gz: 4f855e3c0e1f0d7b59e0255004a1a806c7048da6d3fe0a8ddf10be68e36ed9ba
+  data.tar.gz: 5758c90205c3e2bea420cf7fa0dec07638917beced60bd0482e1d803ced96c07
 SHA512:
-  metadata.gz: ca5e81638625939d11999a64d44c92fc57c762a934aa8fd5b110c3f5aacc9a736ab5f02da4366e7a1b9b9ec0335dd1eb1683f5b9d90bd97c81914ea0a698dc7c
-  data.tar.gz: aff49ef9451bcbc9a97d181757a5b913737cbfbb4fc3ca49d423cbd2e59a4a71091816e98c2996ff7f8292cb6e0c0d69931a4f3e4e36e2a69fdd7f745640e266
+  metadata.gz: ec26f8c4257a6949d829d7f68d2175943b80c2837739bdbead8e6b61891a6738cf84ec1934caff777b63dee39f3d0111e8f29a81c37c125be1cd3ae8137b6968
+  data.tar.gz: 31bb1aa0296dbbc8e1a1c6c2bc7236c92bf3e627935c752bd527c7f7d769da69f11c6bb63de6e0b4e0d1d8d0c739744045669220c35e253ae83eaa0e23482e6a

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## [Unreleased]
 
+## [0.3.15] - 2023-05-30
+- Drop Ruby 2.7 support. It had reached EOD.
+- Bump pgvector-ruby to 0.2
+- 🚚 Loaders
+  - Support for options and block to be passed to CSV processor
+
 ## [0.3.14] - 2023-05-28
 - 🔍 Vectorsearch
   - Not relying on Weaviate modules anymore

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.3.14)
+    langchainrb (0.3.15)
 
 GEM
   remote: https://rubygems.org/
@@ -171,7 +171,7 @@ GEM
       ruby-rc4
       ttfunk
     pg (1.5.3)
-    pgvector (0.1.1)
+    pgvector (0.2.0)
     pinecone (0.1.71)
       dry-struct (~> 1.6.0)
       dry-validation (~> 1.10.0)
@@ -298,7 +298,7 @@ DEPENDENCIES
   nokogiri (~> 1.13)
   pdf-reader (~> 1.4)
   pg (~> 1.5)
-  pgvector (< 0.2)
+  pgvector (~> 0.2)
   pinecone (~> 0.1.6)
   pry-byebug (~> 3.10.0)
   qdrant-ruby (~> 0.9.0)

data/lib/langchain/loader.rb

@@ -14,14 +14,15 @@ module Langchain
     # Equivalent to Langchain::Loader.new(path).load
     # @param path [String | Pathname] path to file or url
     # @return [String] file content
-    def self.load(path)
-      new(path).load
+    def self.load(path, options = {}, &block)
+      new(path, options).load(&block)
     end
 
     # Initialize Langchain::Loader
     # @param path [String | Pathname] path to file or url
     # @return [Langchain::Loader] loader instance
-    def initialize(path)
+    def initialize(path, options = {})
+      @options = options
       @path = path
     end
 
@@ -35,39 +36,38 @@ module Langchain
 
     # Load data from a file or url
     # @return [String] file content
-    def load
-      url? ? from_url(@path) : from_path(@path)
+    def load(&block)
+      @raw_data = url? ? load_from_url : load_from_path
+
+      data = if block
+        yield @raw_data.read, @options
+      else
+        processor_klass.new(@options).parse(@raw_data)
+      end
+
+      Langchain::Data.new(data, source: @path)
     end
 
     private
 
-    def from_url(url)
-      process do
-        data = URI.parse(url).open
-        processor = find_processor(:CONTENT_TYPES, data.content_type)
-        [data, processor]
-      end
+    def load_from_url
+      URI.parse(@path).open
     end
 
-    def from_path(path)
-      raise FileNotFound unless File.exist?(path)
+    def load_from_path
+      raise FileNotFound unless File.exist?(@path)
 
-      process do
-        [File.open(path), find_processor(:EXTENSIONS, File.extname(path))]
-      end
+      File.open(@path)
     end
 
-    def process(&block)
-      raw_data, kind = yield
+    def processor_klass
+      raise UnknownFormatError unless (kind = find_processor)
 
-      raise UnknownFormatError unless kind
-
-      processor = Langchain::Processors.const_get(kind).new
-      Langchain::Data.new(processor.parse(raw_data), source: @path)
+      Langchain::Processors.const_get(kind)
     end
 
-    def find_processor(constant, value)
-      processors.find { |klass| processor_matches? "#{klass}::#{constant}", value }
+    def find_processor
+      processors.find { |klass| processor_matches? "#{klass}::#{lookup_constant}", source_type }
     end
 
     def processor_matches?(constant, value)
@@ -77,5 +77,13 @@ module Langchain
     def processors
       Langchain::Processors.constants
     end
+
+    def source_type
+      url? ? @raw_data.content_type : File.extname(@path)
+    end
+
+    def lookup_constant
+      url? ? :CONTENT_TYPES : :EXTENSIONS
+    end
   end
 end

data/lib/langchain/processors/base.rb

@@ -6,6 +6,10 @@ module Langchain
       EXTENSIONS = []
       CONTENT_TYPES = []
 
+      def initialize(options = {})
+        @options = options
+      end
+
       def parse(data)
         raise NotImplementedError
       end

data/lib/langchain/processors/csv.rb

@@ -12,10 +12,16 @@ module Langchain
       # @param [File] data
       # @return [Array of Hash]
       def parse(data)
-        ::CSV.new(data.read).map do |row|
+        ::CSV.new(data.read, col_sep: separator).map do |row|
           row.map(&:strip)
         end
       end
+
+      private
+
+      def separator
+        @options[:col_sep] || ","
+      end
     end
   end
 end

data/lib/langchain/processors/docx.rb

@@ -6,7 +6,7 @@ module Langchain
       EXTENSIONS = [".docx"]
       CONTENT_TYPES = ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]
 
-      def initialize
+      def initialize(*)
         depends_on "docx"
         require "docx"
       end

data/lib/langchain/processors/html.rb

@@ -9,7 +9,7 @@ module Langchain
       # We only look for headings and paragraphs
       TEXT_CONTENT_TAGS = %w[h1 h2 h3 h4 h5 h6 p]
 
-      def initialize
+      def initialize(*)
         depends_on "nokogiri"
         require "nokogiri"
       end

data/lib/langchain/processors/pdf.rb

@@ -6,7 +6,7 @@ module Langchain
       EXTENSIONS = [".pdf"]
       CONTENT_TYPES = ["application/pdf"]
 
-      def initialize
+      def initialize(*)
         depends_on "pdf-reader"
         require "pdf-reader"
       end

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.3.14"
+  VERSION = "0.3.15"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.3.14
+  version: 0.3.15
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-28 00:00:00.000000000 Z
+date: 2023-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv-rails
@@ -182,14 +182,14 @@ dependencies:
   name: pgvector
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
 - !ruby/object:Gem::Dependency
@@ -290,7 +290,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.17.0
-description: Build ML/AI-powered applications with Ruby's LangChain
+description: Build LLM-backed Ruby applications with Ruby's LangChain
 email:
 - andrei.bondarev13@gmail.com
 executables: []
@@ -366,15 +366,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
-summary: Build ML/AI-powered applications with Ruby's LangChain
+summary: Build LLM-backed Ruby applications with Ruby's LangChain
 test_files: []