kreuzberg 4.1.1 → 4.2.0

data/spec/serialization_spec.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+# Cross-language serialization tests for Ruby bindings
+#
+# Validates that ExtractionConfig serializes consistently with other language bindings
+
+require 'json'
+require 'spec_helper'
+
+RSpec.describe Kreuzberg::ExtractionConfig do
+  describe '#to_h' do
+    it 'serializes minimal config to hash' do
+      config = described_class.new
+      hash = config.to_h
+
+      expect(hash).to be_a(Hash)
+      expect(hash).to have_key(:use_cache)
+      expect(hash).to have_key(:enable_quality_processing)
+      expect(hash).to have_key(:force_ocr)
+    end
+
+    it 'serializes config with all fields' do
+      config = described_class.new(
+        use_cache: true,
+        enable_quality_processing: true,
+        force_ocr: false
+      )
+
+      hash = config.to_h
+
+      expect(hash[:use_cache]).to be(true)
+      expect(hash[:enable_quality_processing]).to be(true)
+      expect(hash[:force_ocr]).to be(false)
+    end
+
+    it 'preserves field values after serialization' do
+      original = described_class.new(
+        use_cache: false,
+        enable_quality_processing: true
+      )
+
+      hash = original.to_h
+
+      expect(hash[:use_cache]).to be(false)
+      expect(hash[:enable_quality_processing]).to be(true)
+    end
+  end
+
+  describe '#to_json' do
+    it 'serializes to JSON' do
+      config = described_class.new(use_cache: true)
+      json = config.to_json
+
+      expect(json).to be_a(String)
+
+      parsed = JSON.parse(json, symbolize_names: true)
+      expect(parsed).to have_key(:use_cache)
+      expect(parsed[:use_cache]).to be(true)
+    end
+
+    it 'produces valid JSON' do
+      config = described_class.new
+      json = config.to_json
+
+      expect { JSON.parse(json) }.not_to raise_error
+    end
+
+    it 'uses snake_case field names' do
+      config = described_class.new(use_cache: true)
+      json = config.to_json
+
+      expect(json).to include('use_cache')
+      expect(json).not_to include('useCache')
+    end
+  end
+
+  describe 'round-trip serialization' do
+    it 'survives serialization -> deserialization -> serialization' do
+      config1 = described_class.new(
+        use_cache: true,
+        enable_quality_processing: false
+      )
+
+      json1 = config1.to_json
+      hash1 = JSON.parse(json1, symbolize_names: true)
+
+      config2 = described_class.new(hash1)
+      json2 = config2.to_json
+
+      # JSON strings should be equivalent
+      expect(JSON.parse(json1)).to eq(JSON.parse(json2))
+    end
+  end
+
+  describe 'field consistency' do
+    it 'includes all mandatory fields' do
+      config = described_class.new
+      hash = config.to_h
+
+      mandatory_fields = %i[use_cache enable_quality_processing force_ocr]
+      mandatory_fields.each do |field|
+        expect(hash).to have_key(field)
+      end
+    end
+
+    it 'handles nested ocr config' do
+      config = described_class.new(
+        ocr: {
+          backend: 'tesseract',
+          language: 'eng'
+        }
+      )
+
+      hash = config.to_h
+
+      expect(hash).to have_key(:ocr)
+      expect(hash[:ocr][:backend]).to eq('tesseract')
+      expect(hash[:ocr][:language]).to eq('eng')
+    end
+  end
+
+  describe 'immutability' do
+    it 'does not modify original config during serialization' do
+      config = described_class.new(use_cache: true)
+
+      json1 = config.to_json
+      json2 = config.to_json
+      json3 = config.to_json
+
+      expect(json1).to eq(json2)
+      expect(json2).to eq(json3)
+    end
+  end
+end

data/spec/unit/config/output_format_spec.rb

@@ -0,0 +1,380 @@
+# frozen_string_literal: true
+
+# rubocop:disable RSpec/RepeatedExample
+RSpec.describe 'Output Format and Result Format Configuration' do
+  describe Kreuzberg::Config::Extraction do
+    describe 'output_format' do
+      it 'accepts output_format as initialization parameter' do
+        config = described_class.new(output_format: 'markdown')
+
+        expect(config.output_format).to eq 'markdown'
+      end
+
+      it 'defaults to nil when not specified' do
+        config = described_class.new
+
+        expect(config.output_format).to be_nil
+      end
+
+      it 'accepts plain format' do
+        config = described_class.new(output_format: 'plain')
+
+        expect(config.output_format).to eq 'plain'
+      end
+
+      it 'accepts markdown format' do
+        config = described_class.new(output_format: 'markdown')
+
+        expect(config.output_format).to eq 'markdown'
+      end
+
+      it 'accepts djot format' do
+        config = described_class.new(output_format: 'djot')
+
+        expect(config.output_format).to eq 'djot'
+      end
+
+      it 'accepts html format' do
+        config = described_class.new(output_format: 'html')
+
+        expect(config.output_format).to eq 'html'
+      end
+
+      it 'converts output_format to string' do
+        config = described_class.new(output_format: :markdown)
+
+        expect(config.output_format).to eq 'markdown'
+        expect(config.output_format).to be_a String
+      end
+
+      it 'includes output_format in to_h' do
+        config = described_class.new(output_format: 'markdown')
+        hash = config.to_h
+
+        expect(hash[:output_format]).to eq 'markdown'
+      end
+
+      it 'excludes nil output_format from to_h' do
+        config = described_class.new(output_format: nil)
+        hash = config.to_h
+
+        expect(hash.key?(:output_format)).to be false
+      end
+
+      it 'includes output_format in JSON' do
+        config = described_class.new(output_format: 'markdown')
+        json = config.to_json
+        parsed = JSON.parse(json)
+
+        expect(parsed['output_format']).to eq 'markdown'
+      end
+
+      it 'retrieves output_format with get_field' do
+        config = described_class.new(output_format: 'djot')
+
+        expect(config.get_field('output_format')).to eq 'djot'
+      end
+
+      it 'can be set with []=' do
+        config = described_class.new
+        config[:output_format] = 'html'
+
+        expect(config.output_format).to eq 'html'
+      end
+
+      it 'can be set with []= using symbol' do
+        config = described_class.new
+        config[:output_format] = :plain
+
+        expect(config.output_format).to eq 'plain'
+      end
+
+      it 'can be retrieved with []' do
+        config = described_class.new(output_format: 'markdown')
+
+        expect(config[:output_format]).to eq 'markdown'
+      end
+    end
+
+    describe 'result_format' do
+      it 'accepts result_format as initialization parameter' do
+        config = described_class.new(result_format: 'unified')
+
+        expect(config.result_format).to eq 'unified'
+      end
+
+      it 'defaults to nil when not specified' do
+        config = described_class.new
+
+        expect(config.result_format).to be_nil
+      end
+
+      it 'accepts unified format' do
+        config = described_class.new(result_format: 'unified')
+
+        expect(config.result_format).to eq 'unified'
+      end
+
+      it 'accepts element_based format' do
+        config = described_class.new(result_format: 'element_based')
+
+        expect(config.result_format).to eq 'element_based'
+      end
+
+      it 'converts result_format to string' do
+        config = described_class.new(result_format: :unified)
+
+        expect(config.result_format).to eq 'unified'
+        expect(config.result_format).to be_a String
+      end
+
+      it 'includes result_format in to_h' do
+        config = described_class.new(result_format: 'element_based')
+        hash = config.to_h
+
+        expect(hash[:result_format]).to eq 'element_based'
+      end
+
+      it 'excludes nil result_format from to_h' do
+        config = described_class.new(result_format: nil)
+        hash = config.to_h
+
+        expect(hash.key?(:result_format)).to be false
+      end
+
+      it 'includes result_format in JSON' do
+        config = described_class.new(result_format: 'element_based')
+        json = config.to_json
+        parsed = JSON.parse(json)
+
+        expect(parsed['result_format']).to eq 'element_based'
+      end
+
+      it 'retrieves result_format with get_field' do
+        config = described_class.new(result_format: 'unified')
+
+        expect(config.get_field('result_format')).to eq 'unified'
+      end
+
+      it 'can be set with []=' do
+        config = described_class.new
+        config[:result_format] = 'unified'
+
+        expect(config.result_format).to eq 'unified'
+      end
+
+      it 'can be set with []= using symbol' do
+        config = described_class.new
+        config[:result_format] = :element_based
+
+        expect(config.result_format).to eq 'element_based'
+      end
+
+      it 'can be retrieved with []' do
+        config = described_class.new(result_format: 'element_based')
+
+        expect(config[:result_format]).to eq 'element_based'
+      end
+    end
+
+    describe 'combined output and result formats' do
+      it 'accepts both output_format and result_format' do
+        config = described_class.new(
+          output_format: 'markdown',
+          result_format: 'unified'
+        )
+
+        expect(config.output_format).to eq 'markdown'
+        expect(config.result_format).to eq 'unified'
+      end
+
+      it 'serializes both formats in to_h' do
+        config = described_class.new(
+          output_format: 'djot',
+          result_format: 'element_based'
+        )
+        hash = config.to_h
+
+        expect(hash[:output_format]).to eq 'djot'
+        expect(hash[:result_format]).to eq 'element_based'
+      end
+
+      it 'serializes both formats in JSON' do
+        config = described_class.new(
+          output_format: 'html',
+          result_format: 'unified'
+        )
+        json = config.to_json
+        parsed = JSON.parse(json)
+
+        expect(parsed['output_format']).to eq 'html'
+        expect(parsed['result_format']).to eq 'unified'
+      end
+
+      it 'merges both formats correctly' do
+        base = described_class.new(
+          output_format: 'markdown',
+          result_format: 'unified'
+        )
+        override = described_class.new(output_format: 'html')
+        merged = base.merge(override)
+
+        expect(merged.output_format).to eq 'html'
+        expect(merged.result_format).to eq 'unified'
+      end
+
+      it 'merges both formats with merge!' do
+        config = described_class.new(
+          output_format: 'markdown',
+          result_format: 'unified'
+        )
+        override = described_class.new(
+          output_format: 'djot',
+          result_format: 'element_based'
+        )
+        config.merge!(override)
+
+        expect(config.output_format).to eq 'djot'
+        expect(config.result_format).to eq 'element_based'
+      end
+
+      it 'handles merge with hash containing both formats' do
+        config = described_class.new(
+          output_format: 'plain',
+          result_format: 'unified'
+        )
+        merged = config.merge({ output_format: 'markdown' })
+
+        expect(merged.output_format).to eq 'markdown'
+        expect(merged.result_format).to eq 'unified'
+      end
+    end
+
+    describe 'format persistence across operations' do
+      it 'persists output_format through multiple conversions' do
+        config = described_class.new(output_format: 'markdown')
+        hash = config.to_h
+        new_config = described_class.new(**hash)
+
+        expect(new_config.output_format).to eq 'markdown'
+      end
+
+      it 'persists result_format through multiple conversions' do
+        config = described_class.new(result_format: 'element_based')
+        hash = config.to_h
+        new_config = described_class.new(**hash)
+
+        expect(new_config.result_format).to eq 'element_based'
+      end
+
+      it 'round-trips through JSON' do
+        config = described_class.new(
+          output_format: 'djot',
+          result_format: 'unified'
+        )
+        json = config.to_json
+        parsed = JSON.parse(json)
+        new_config = described_class.new(**parsed.transform_keys(&:to_sym))
+
+        expect(new_config.output_format).to eq 'djot'
+        expect(new_config.result_format).to eq 'unified'
+      end
+    end
+
+    describe 'format validation and edge cases' do
+      it 'handles empty string output_format' do
+        config = described_class.new(output_format: '')
+
+        expect(config.output_format).to eq ''
+      end
+
+      it 'handles empty string result_format' do
+        config = described_class.new(result_format: '')
+
+        expect(config.result_format).to eq ''
+      end
+
+      it 'handles whitespace in output_format' do
+        config = described_class.new(output_format: '  plain  ')
+
+        expect(config.output_format).to eq '  plain  '
+      end
+
+      it 'handles case sensitivity in output_format' do
+        config = described_class.new(output_format: 'MarkDown')
+
+        expect(config.output_format).to eq 'MarkDown'
+      end
+
+      it 'handles custom string in result_format' do
+        config = described_class.new(result_format: 'custom_format')
+
+        expect(config.result_format).to eq 'custom_format'
+      end
+    end
+
+    describe 'integration with other config fields' do
+      it 'works with output_format and chunking together' do
+        config = described_class.new(
+          output_format: 'markdown',
+          chunking: { max_chars: 500 }
+        )
+
+        expect(config.output_format).to eq 'markdown'
+        expect(config.chunking.max_chars).to eq 500
+      end
+
+      it 'works with result_format and OCR together' do
+        config = described_class.new(
+          result_format: 'element_based',
+          ocr: { backend: 'tesseract' }
+        )
+
+        expect(config.result_format).to eq 'element_based'
+        expect(config.ocr.backend).to eq 'tesseract'
+      end
+
+      it 'works with both formats and language detection' do
+        config = described_class.new(
+          output_format: 'html',
+          result_format: 'unified',
+          language_detection: { enabled: true }
+        )
+
+        expect(config.output_format).to eq 'html'
+        expect(config.result_format).to eq 'unified'
+        expect(config.language_detection.enabled).to be true
+      end
+
+      it 'preserves formats in complex config merge' do
+        base = described_class.new(
+          output_format: 'markdown',
+          result_format: 'unified',
+          chunking: { max_chars: 500 },
+          ocr: { backend: 'tesseract' }
+        )
+        override = described_class.new(
+          output_format: 'djot',
+          chunking: { max_chars: 750 }
+        )
+        merged = base.merge(override)
+
+        expect(merged.output_format).to eq 'djot'
+        expect(merged.result_format).to eq 'unified'
+        expect(merged.chunking.max_chars).to eq 750
+        expect(merged.ocr.backend).to eq 'tesseract'
+      end
+    end
+
+    describe 'allowed keys integration' do
+      it 'includes output_format in ALLOWED_KEYS' do
+        expect(Kreuzberg::Config::Extraction::ALLOWED_KEYS).to include(:output_format)
+      end
+
+      it 'includes result_format in ALLOWED_KEYS' do
+        expect(Kreuzberg::Config::Extraction::ALLOWED_KEYS).to include(:result_format)
+      end
+    end
+  end
+end
+# rubocop:enable RSpec/RepeatedExample

data/vendor/Cargo.toml

@@ -3,7 +3,7 @@ members = ["kreuzberg", "kreuzberg-tesseract", "kreuzberg-ffi"]
 resolver = "2"
 
 [workspace.package]
-version = "4.1.1"
+version = "4.2.0"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.1.1"
+version = "4.2.0"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -142,7 +142,7 @@ mime_guess = "2.0"
 rmp-serde = "1.3"
 thiserror = { workspace = true }
 tokio = { workspace = true, optional = true }
-uuid = { version = "1.19.0", features = ["v4", "js"] }
+uuid = { version = "1.20.0", features = ["v4", "js"] }
 indexmap = "2.13.0"
 tracing = { workspace = true }
 pdfium-render = { package = "kreuzberg-pdfium-render", version = "0.9.0", features = [
@@ -198,7 +198,7 @@ rake = { version = "0.3.6", optional = true }
 axum = { version = "0.8", features = ["macros", "json", "multipart"], optional = true }
 tower = { version = "0.5", optional = true }
 tower-http = { version = "0.6", features = ["cors", "trace", "limit"], optional = true }
-rmcp = { version = "0.13.0", features = [
+rmcp = { version = "0.14.0", features = [
     "server",
     "macros",
     "base64",

data/vendor/kreuzberg/README.md

@@ -17,7 +17,7 @@ High-performance document intelligence library for Rust. Extract text, metadata,
 
 This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
 
-> **🚀 Version 4.1.1 Release**
+> **🚀 Version 4.2.0 Release**
 > This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
 >
 > **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.

data/vendor/kreuzberg/src/embeddings.rs

@@ -219,10 +219,10 @@ pub fn get_or_init_model(
         // This prevents panics that cannot unwind through FFI boundaries
         fn ensure_onnx_available() -> Result<(), String> {
             // Check if ORT_DYLIB_PATH is already set and valid
-            if let Ok(path) = std::env::var("ORT_DYLIB_PATH") {
-                if std::path::Path::new(&path).exists() {
-                    return Ok(());
-                }
+            if let Ok(path) = std::env::var("ORT_DYLIB_PATH")
+                && std::path::Path::new(&path).exists()
+            {
+                return Ok(());
             }
 
             // Check common installation paths and set ORT_DYLIB_PATH if found