kreuzberg 4.1.2 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6550daabf58e5e396576e5a83c6a53f226e677f9c129920c9990bba309fbd7ba
-  data.tar.gz: 9595aa468666391d08a0962db589bbbc50d5bd1c8532e101efa234f6c523d7c5
+  metadata.gz: 9a1c9adffca7d75c142bd661f1d481b1aee00d97c6f62dcc70292f37978bcc17
+  data.tar.gz: 227af2ed45bff1dfa9afebd69220d15a41b2e476bf97f8a83173d21aab8b88e1
 SHA512:
-  metadata.gz: 0dea911deebe061515dd4cbff2b76b3a7947c68f196fcc576001d42d80386f6c53f8ed63e0e4acb8e719ad6f95c21e689df7aef5f6cbbbc0d1c92ef96ddb673c
-  data.tar.gz: 0df091f80f7c73dda0c17d89d4aa0571cd01f0f2b697b187fd9bae28f8dbcf96cd2e3a269f9831a442b8cf46ce40608586d3d6a242d84bb394fe6056cba3b492
+  metadata.gz: 0d1b0081f89a73f5422e68a714fc415f6d290dd8be7cf0ba6d454cfdf1938ebdac4919358b25d6e5a0bc1a209e1b165062a0341d28cde1b3fa0595bffec837f5
+  data.tar.gz: fc5a5f29309c29fbbf63ba035cf5e462e78b15c2afc239fb333bd8b6e70ef061219822ed4d533f81bb35cdd62db84da8c01e8f172561b0f7fb802b848b491c0a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kreuzberg (4.1.2)
+    kreuzberg (4.2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -207,7 +207,7 @@ CHECKSUMS
   i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
   json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
-  kreuzberg (4.1.2)
+  kreuzberg (4.2.0)
   language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
   lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
   listen (3.10.0) sha256=c6e182db62143aeccc2e1960033bebe7445309c7272061979bb098d03760c9d2

data/README.md

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
   </a>
   <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
-    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.1.2" alt="Go">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.2.0" alt="Go">
   </a>
   <a href="https://www.nuget.org/packages/Kreuzberg/">
     <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">

data/kreuzberg.gemspec

@@ -130,10 +130,22 @@ vendor_files = Dir.chdir(__dir__) do
   kreuzberg_files + kreuzberg_ffi_files + kreuzberg_tesseract_files + rb_sys_files + workspace_toml
 end
 
+# When vendor files exist, get ext/ files from filesystem (to include modified Cargo.toml
+# with vendor paths) instead of from git (which has original 5-level crate paths)
+ext_files_from_fs = Dir.chdir(__dir__) do
+  Dir.glob('ext/**/*', File::FNM_DOTMATCH)
+     .reject { |f| File.directory?(f) }
+     .reject { |f| f.include?('/target/') }
+     .grep_v(/\.(swp|bak|tmp)$/)
+     .grep_v(/~$/)
+end
+
 files = if (ruby_files + core_files + ffi_files).empty?
           fallback_files
         elsif vendor_files.any?
-          ruby_files + vendor_files
+          # Use ext/ files from filesystem (modified by vendor script) + non-ext ruby files from git
+          non_ext_ruby_files = ruby_files.reject { |f| f.start_with?('ext/') }
+          non_ext_ruby_files + ext_files_from_fs + vendor_files
         else
           ruby_files + core_files + ffi_files
         end

data/lib/kreuzberg/config.rb

@@ -717,7 +717,7 @@ module Kreuzberg
                   :ocr, :chunking, :language_detection, :pdf_options,
                   :image_extraction, :image_preprocessing, :postprocessor,
                   :token_reduction, :keywords, :html_options, :pages,
-                  :max_concurrent_extractions
+                  :max_concurrent_extractions, :output_format, :result_format
 
       # Load configuration from a file.
       #
@@ -738,7 +738,7 @@ module Kreuzberg
         use_cache enable_quality_processing force_ocr ocr chunking
         language_detection pdf_options image_extraction image_preprocessing
         postprocessor token_reduction keywords html_options pages
-        max_concurrent_extractions
+        max_concurrent_extractions output_format result_format
       ].freeze
 
       # Aliases for backward compatibility
@@ -789,41 +789,67 @@ module Kreuzberg
         new(**normalize_hash_keys(hash))
       end
 
-      def initialize(
-        use_cache: true,
-        enable_quality_processing: false,
-        force_ocr: false,
-        ocr: nil,
-        chunking: nil,
-        language_detection: nil,
-        pdf_options: nil,
-        image_extraction: nil,
-        image_preprocessing: nil,
-        postprocessor: nil,
-        token_reduction: nil,
-        keywords: nil,
-        html_options: nil,
-        pages: nil,
-        max_concurrent_extractions: nil
-      )
-        @use_cache = use_cache ? true : false
-        @enable_quality_processing = enable_quality_processing ? true : false
-        @force_ocr = force_ocr ? true : false
-        @ocr = normalize_config(ocr, OCR)
-        @chunking = normalize_config(chunking, Chunking)
-        @language_detection = normalize_config(language_detection, LanguageDetection)
-        @pdf_options = normalize_config(pdf_options, PDF)
-        @image_extraction = normalize_config(image_extraction, ImageExtraction)
-        @image_preprocessing = normalize_config(image_preprocessing, ImagePreprocessing)
-        @postprocessor = normalize_config(postprocessor, PostProcessor)
-        @token_reduction = normalize_config(token_reduction, TokenReduction)
-        @keywords = normalize_config(keywords, Keywords)
-        @html_options = normalize_config(html_options, HtmlOptions)
-        @pages = normalize_config(pages, PageConfig)
-        @max_concurrent_extractions = max_concurrent_extractions&.to_i
+      def initialize(hash = nil,
+                     use_cache: true,
+                     enable_quality_processing: false,
+                     force_ocr: false,
+                     ocr: nil,
+                     chunking: nil,
+                     language_detection: nil,
+                     pdf_options: nil,
+                     image_extraction: nil,
+                     image_preprocessing: nil,
+                     postprocessor: nil,
+                     token_reduction: nil,
+                     keywords: nil,
+                     html_options: nil,
+                     pages: nil,
+                     max_concurrent_extractions: nil,
+                     output_format: nil,
+                     result_format: nil)
+        kwargs = {
+          use_cache: use_cache, enable_quality_processing: enable_quality_processing,
+          force_ocr: force_ocr, ocr: ocr, chunking: chunking, language_detection: language_detection,
+          pdf_options: pdf_options, image_extraction: image_extraction,
+          image_preprocessing: image_preprocessing, postprocessor: postprocessor,
+          token_reduction: token_reduction, keywords: keywords, html_options: html_options,
+          pages: pages, max_concurrent_extractions: max_concurrent_extractions,
+          output_format: output_format, result_format: result_format
+        }
+        extracted = extract_from_hash(hash, kwargs)
+
+        assign_attributes(extracted)
+      end
+
+      def extract_from_hash(hash, defaults)
+        return defaults unless hash.is_a?(Hash)
+
+        hash = hash.transform_keys(&:to_sym)
+        defaults.merge(hash.slice(*defaults.keys))
+      end
+
+      def assign_attributes(params)
+        @use_cache = params[:use_cache] ? true : false
+        @enable_quality_processing = params[:enable_quality_processing] ? true : false
+        @force_ocr = params[:force_ocr] ? true : false
+        @ocr = normalize_config(params[:ocr], OCR)
+        @chunking = normalize_config(params[:chunking], Chunking)
+        @language_detection = normalize_config(params[:language_detection], LanguageDetection)
+        @pdf_options = normalize_config(params[:pdf_options], PDF)
+        @image_extraction = normalize_config(params[:image_extraction], ImageExtraction)
+        @image_preprocessing = normalize_config(params[:image_preprocessing], ImagePreprocessing)
+        @postprocessor = normalize_config(params[:postprocessor], PostProcessor)
+        @token_reduction = normalize_config(params[:token_reduction], TokenReduction)
+        @keywords = normalize_config(params[:keywords], Keywords)
+        @html_options = normalize_config(params[:html_options], HtmlOptions)
+        @pages = normalize_config(params[:pages], PageConfig)
+        @max_concurrent_extractions = params[:max_concurrent_extractions]&.to_i
+        @output_format = params[:output_format]&.to_s
+        @result_format = params[:result_format]&.to_s
       end
 
       # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/MethodLength
       def to_h
         {
           use_cache: @use_cache,
@@ -840,9 +866,12 @@ module Kreuzberg
           keywords: @keywords&.to_h,
           html_options: @html_options&.to_h,
           pages: @pages&.to_h,
-          max_concurrent_extractions: @max_concurrent_extractions
+          max_concurrent_extractions: @max_concurrent_extractions,
+          output_format: @output_format,
+          result_format: @result_format
         }.compact
       end
+      # rubocop:enable Metrics/MethodLength
       # rubocop:enable Metrics/CyclomaticComplexity
 
       # Serialize configuration to JSON string
@@ -981,6 +1010,10 @@ module Kreuzberg
           @pages = normalize_config(value, PageConfig)
         when :max_concurrent_extractions
           @max_concurrent_extractions = value&.to_i
+        when :output_format
+          @output_format = value&.to_s
+        when :result_format
+          @result_format = value&.to_s
         else
           raise ArgumentError, "Unknown configuration key: #{key}"
         end
@@ -1028,6 +1061,8 @@ module Kreuzberg
         @html_options = merged.html_options
         @pages = merged.pages
         @max_concurrent_extractions = merged.max_concurrent_extractions
+        @output_format = merged.output_format
+        @result_format = merged.result_format
       end
     end
   end

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.1.2'
+  VERSION = '4.2.0'
 end

data/sig/kreuzberg.rbs

@@ -202,6 +202,8 @@ module Kreuzberg
       attr_reader html_options: HtmlOptions?
       attr_reader pages: PageConfig?
       attr_reader max_concurrent_extractions: Integer?
+      attr_reader output_format: String?
+      attr_reader result_format: String?
 
       def self.from_file: (String path) -> Extraction
       def initialize: (
@@ -219,7 +221,9 @@ module Kreuzberg
         ?keywords: (Keywords | Hash[Symbol, untyped])?,
         ?html_options: (HtmlOptions | Hash[Symbol, untyped])?,
         ?pages: (PageConfig | Hash[Symbol, untyped])?,
-        ?max_concurrent_extractions: Integer?
+        ?max_concurrent_extractions: Integer?,
+        ?output_format: String?,
+        ?result_format: String?
       ) -> void
       def to_h: () -> Hash[Symbol, untyped]
 

data/spec/binding/batch_operations_spec.rb

@@ -592,4 +592,84 @@ RSpec.describe 'Batch Operations' do
       paths.each { |p| FileUtils.rm_f(p) }
     end
   end
+
+  describe 'batch with output and result formats' do
+    it 'batch processes with output_format' do
+      paths = []
+      file = Tempfile.new(['format_test', '.txt']).tap do |f|
+        f.write('Test content for output format')
+        f.close
+      end
+      paths << file.path
+
+      config = Kreuzberg::Config::Extraction.new(output_format: 'markdown')
+      results = Kreuzberg.batch_extract_files_sync(paths: paths, config: config)
+
+      expect(results).to be_an Array
+      expect(results.length).to eq 1
+      expect(results[0]).to be_a Kreuzberg::Result
+
+      paths.each { |p| FileUtils.rm_f(p) }
+    end
+
+    it 'batch processes with result_format' do
+      paths = []
+      file = Tempfile.new(['format_test', '.txt']).tap do |f|
+        f.write('Test content for result format')
+        f.close
+      end
+      paths << file.path
+
+      config = Kreuzberg::Config::Extraction.new(result_format: 'unified')
+      results = Kreuzberg.batch_extract_files_sync(paths: paths, config: config)
+
+      expect(results).to be_an Array
+      expect(results.length).to eq 1
+      expect(results[0]).to be_a Kreuzberg::Result
+
+      paths.each { |p| FileUtils.rm_f(p) }
+    end
+
+    it 'batch processes with both output and result formats' do
+      paths = []
+      file = Tempfile.new(['format_test', '.txt']).tap do |f|
+        f.write('Test content for both formats')
+        f.close
+      end
+      paths << file.path
+
+      config = Kreuzberg::Config::Extraction.new(
+        output_format: 'plain',
+        result_format: 'element_based'
+      )
+      results = Kreuzberg.batch_extract_files_sync(paths: paths, config: config)
+
+      expect(results).to be_an Array
+      expect(results.length).to eq 1
+      expect(results[0]).to be_a Kreuzberg::Result
+
+      paths.each { |p| FileUtils.rm_f(p) }
+    end
+
+    it 'batch processes with chunking and output_format' do
+      paths = []
+      file = Tempfile.new(['format_test', '.txt']).tap do |f|
+        f.write('Test content ' * 100)
+        f.close
+      end
+      paths << file.path
+
+      config = Kreuzberg::Config::Extraction.new(
+        output_format: 'markdown',
+        chunking: { max_chars: 1000 }
+      )
+      results = Kreuzberg.batch_extract_files_sync(paths: paths, config: config)
+
+      expect(results).to be_an Array
+      expect(results.length).to eq 1
+      expect(results[0]).to be_a Kreuzberg::Result
+
+      paths.each { |p| FileUtils.rm_f(p) }
+    end
+  end
 end

data/spec/binding/metadata_types_spec.rb

@@ -1154,64 +1154,11 @@ RSpec.describe 'Kreuzberg Metadata Types' do
 
   describe 'Thread Safety: Concurrent Extraction' do
     it 'handles concurrent extraction safely' do
-      test_files = []
-      results = []
-      errors = []
+      test_files = create_concurrent_test_files
+      results, errors = run_concurrent_extractions(test_files)
 
-      5.times do |i|
-        html_content = <<~HTML
-          <html>
-          <head>
-            <title>Concurrent Test #{i}</title>
-            <meta name="description" content="Test document #{i}">
-            <meta name="keywords" content="test#{i}, concurrent, thread-safe">
-          </head>
-          <body>
-            <h1>Test Document #{i}</h1>
-            <p>Content for test #{i}</p>
-            <a href="/page-#{i}">Link #{i}</a>
-            <img src="image-#{i}.jpg" alt="Image #{i}">
-          </body>
-          </html>
-        HTML
-        test_files << create_test_html_file(html_content)
-      end
-
-      begin
-        threads = test_files.map do |file|
-          Thread.new do
-            result = Kreuzberg.extract_file_sync(path: file)
-            results << result
-          rescue StandardError => e
-            errors << e
-          end
-        end
-
-        threads.each(&:join)
-
-        expect(errors).to be_empty
-
-        expect(results.length).to eq(5)
-        results.each do |result|
-          expect(result).to be_a(Kreuzberg::Result)
-          expect(result.metadata).not_to be_nil
-
-          metadata = result.metadata
-          next unless metadata.is_a?(Kreuzberg::HtmlMetadata)
-
-          expect(metadata.title).not_to be_nil
-          expect(metadata.description).not_to be_nil
-          expect(metadata.keywords).to be_a(Array)
-          expect(metadata.headers).to be_a(Array)
-          expect(metadata.links).to be_a(Array)
-          expect(metadata.images).to be_a(Array)
-        end
-
-        titles = results.map { |r| r.metadata.is_a?(Kreuzberg::HtmlMetadata) ? r.metadata.title : r.metadata['title'] }
-        expect(titles.uniq.length).to eq(5)
-      ensure
-        test_files.each { |f| FileUtils.rm_f(f) }
-      end
+      expect(results).not_to be_empty
+      verify_concurrent_results(results, errors, test_files)
     end
   end
 
@@ -1225,4 +1172,77 @@ RSpec.describe 'Kreuzberg Metadata Types' do
     file.close
     file.path
   end
+
+  def create_concurrent_test_files
+    test_files = []
+    5.times do |i|
+      html_content = <<~HTML
+        <html>
+        <head>
+          <title>Concurrent Test #{i}</title>
+          <meta name="description" content="Test document #{i}">
+          <meta name="keywords" content="test#{i}, concurrent, thread-safe">
+        </head>
+        <body>
+          <h1>Test Document #{i}</h1>
+          <p>Content for test #{i}</p>
+          <a href="/page-#{i}">Link #{i}</a>
+          <img src="image-#{i}.jpg" alt="Image #{i}">
+        </body>
+        </html>
+      HTML
+      test_files << create_test_html_file(html_content)
+    end
+    test_files
+  end
+
+  def run_concurrent_extractions(test_files)
+    results = []
+    errors = []
+
+    threads = test_files.map do |file|
+      Thread.new do
+        result = Kreuzberg.extract_file_sync(path: file)
+        results << result
+      rescue StandardError => e
+        errors << e
+      end
+    end
+
+    threads.each(&:join)
+    [results, errors]
+  end
+
+  def verify_concurrent_results(results, errors, test_files)
+    expect(errors).to be_empty
+    expect(results.length).to eq(5)
+
+    results.each do |result|
+      expect(result).to be_a(Kreuzberg::Result)
+      expect(result.metadata).not_to be_nil
+
+      metadata = result.metadata
+      next unless metadata.is_a?(Kreuzberg::HtmlMetadata)
+
+      verify_metadata_fields(metadata)
+    end
+
+    titles = extract_titles(results)
+    expect(titles.uniq.length).to eq(5)
+  ensure
+    test_files.each { |f| FileUtils.rm_f(f) }
+  end
+
+  def verify_metadata_fields(metadata)
+    expect(metadata.title).not_to be_nil
+    expect(metadata.description).not_to be_nil
+    expect(metadata.keywords).to be_a(Array)
+    expect(metadata.headers).to be_a(Array)
+    expect(metadata.links).to be_a(Array)
+    expect(metadata.images).to be_a(Array)
+  end
+
+  def extract_titles(results)
+    results.map { |r| r.metadata.is_a?(Kreuzberg::HtmlMetadata) ? r.metadata.title : r.metadata['title'] }
+  end
 end

data/spec/serialization_spec.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+# Cross-language serialization tests for Ruby bindings
+#
+# Validates that ExtractionConfig serializes consistently with other language bindings
+
+require 'json'
+require 'spec_helper'
+
+RSpec.describe Kreuzberg::ExtractionConfig do
+  describe '#to_h' do
+    it 'serializes minimal config to hash' do
+      config = described_class.new
+      hash = config.to_h
+
+      expect(hash).to be_a(Hash)
+      expect(hash).to have_key(:use_cache)
+      expect(hash).to have_key(:enable_quality_processing)
+      expect(hash).to have_key(:force_ocr)
+    end
+
+    it 'serializes config with all fields' do
+      config = described_class.new(
+        use_cache: true,
+        enable_quality_processing: true,
+        force_ocr: false
+      )
+
+      hash = config.to_h
+
+      expect(hash[:use_cache]).to be(true)
+      expect(hash[:enable_quality_processing]).to be(true)
+      expect(hash[:force_ocr]).to be(false)
+    end
+
+    it 'preserves field values after serialization' do
+      original = described_class.new(
+        use_cache: false,
+        enable_quality_processing: true
+      )
+
+      hash = original.to_h
+
+      expect(hash[:use_cache]).to be(false)
+      expect(hash[:enable_quality_processing]).to be(true)
+    end
+  end
+
+  describe '#to_json' do
+    it 'serializes to JSON' do
+      config = described_class.new(use_cache: true)
+      json = config.to_json
+
+      expect(json).to be_a(String)
+
+      parsed = JSON.parse(json, symbolize_names: true)
+      expect(parsed).to have_key(:use_cache)
+      expect(parsed[:use_cache]).to be(true)
+    end
+
+    it 'produces valid JSON' do
+      config = described_class.new
+      json = config.to_json
+
+      expect { JSON.parse(json) }.not_to raise_error
+    end
+
+    it 'uses snake_case field names' do
+      config = described_class.new(use_cache: true)
+      json = config.to_json
+
+      expect(json).to include('use_cache')
+      expect(json).not_to include('useCache')
+    end
+  end
+
+  describe 'round-trip serialization' do
+    it 'survives serialization -> deserialization -> serialization' do
+      config1 = described_class.new(
+        use_cache: true,
+        enable_quality_processing: false
+      )
+
+      json1 = config1.to_json
+      hash1 = JSON.parse(json1, symbolize_names: true)
+
+      config2 = described_class.new(hash1)
+      json2 = config2.to_json
+
+      # JSON strings should be equivalent
+      expect(JSON.parse(json1)).to eq(JSON.parse(json2))
+    end
+  end
+
+  describe 'field consistency' do
+    it 'includes all mandatory fields' do
+      config = described_class.new
+      hash = config.to_h
+
+      mandatory_fields = %i[use_cache enable_quality_processing force_ocr]
+      mandatory_fields.each do |field|
+        expect(hash).to have_key(field)
+      end
+    end
+
+    it 'handles nested ocr config' do
+      config = described_class.new(
+        ocr: {
+          backend: 'tesseract',
+          language: 'eng'
+        }
+      )
+
+      hash = config.to_h
+
+      expect(hash).to have_key(:ocr)
+      expect(hash[:ocr][:backend]).to eq('tesseract')
+      expect(hash[:ocr][:language]).to eq('eng')
+    end
+  end
+
+  describe 'immutability' do
+    it 'does not modify original config during serialization' do
+      config = described_class.new(use_cache: true)
+
+      json1 = config.to_json
+      json2 = config.to_json
+      json3 = config.to_json
+
+      expect(json1).to eq(json2)
+      expect(json2).to eq(json3)
+    end
+  end
+end