kreuzberg 4.0.0.rc2 → 4.0.1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kreuzberg
 version: !ruby/object:Gem::Version
-  version: 4.0.0.rc2
+  version: 4.0.1
 platform: ruby
 authors:
 - Na'aman Hirschfeld
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-12-09 00:00:00.000000000 Z
+date: 2026-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -56,14 +56,14 @@ dependencies:
   name: rb_sys
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
         version: 0.9.119
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
         version: 0.9.119
 - !ruby/object:Gem::Dependency
@@ -80,6 +80,76 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: rbs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.66'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.66'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: steep
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -116,6 +186,7 @@ files:
 - Steepfile
 - examples/async_patterns.rb
 - ext/kreuzberg_rb/extconf.rb
+- ext/kreuzberg_rb/native/.cargo/config.toml
 - ext/kreuzberg_rb/native/Cargo.lock
 - ext/kreuzberg_rb/native/Cargo.toml
 - ext/kreuzberg_rb/native/README.md
@@ -141,29 +212,114 @@ files:
 - lib/kreuzberg/post_processor_protocol.rb
 - lib/kreuzberg/result.rb
 - lib/kreuzberg/setup_lib_path.rb
+- lib/kreuzberg/types.rb
 - lib/kreuzberg/validator_protocol.rb
 - lib/kreuzberg/version.rb
+- lib/libpdfium.so
 - sig/kreuzberg.rbs
 - sig/kreuzberg/internal.rbs
+- spec/binding/async_operations_spec.rb
+- spec/binding/batch_operations_spec.rb
+- spec/binding/batch_spec.rb
 - spec/binding/cache_spec.rb
 - spec/binding/cli_proxy_spec.rb
 - spec/binding/cli_spec.rb
+- spec/binding/config_result_spec.rb
 - spec/binding/config_spec.rb
 - spec/binding/config_validation_spec.rb
+- spec/binding/embeddings_spec.rb
 - spec/binding/error_handling_spec.rb
+- spec/binding/error_recovery_spec.rb
 - spec/binding/errors_spec.rb
+- spec/binding/font_config_spec.rb
+- spec/binding/images_spec.rb
+- spec/binding/keywords_extraction_spec.rb
+- spec/binding/metadata_types_spec.rb
+- spec/binding/pages_extraction_spec.rb
 - spec/binding/plugins/ocr_backend_spec.rb
 - spec/binding/plugins/postprocessor_spec.rb
 - spec/binding/plugins/validator_spec.rb
+- spec/binding/tables_spec.rb
 - spec/fixtures/config.toml
 - spec/fixtures/config.yaml
 - spec/fixtures/invalid_config.toml
 - spec/smoke/package_spec.rb
 - spec/spec_helper.rb
+- spec/unit/config/chunking_config_spec.rb
+- spec/unit/config/embedding_config_spec.rb
+- spec/unit/config/extraction_config_spec.rb
+- spec/unit/config/font_config_spec.rb
+- spec/unit/config/hierarchy_config_spec.rb
+- spec/unit/config/image_extraction_config_spec.rb
+- spec/unit/config/image_preprocessing_config_spec.rb
+- spec/unit/config/keyword_config_spec.rb
+- spec/unit/config/language_detection_config_spec.rb
+- spec/unit/config/ocr_config_spec.rb
+- spec/unit/config/page_config_spec.rb
+- spec/unit/config/pdf_config_spec.rb
+- spec/unit/config/postprocessor_config_spec.rb
+- spec/unit/config/tesseract_config_spec.rb
+- spec/unit/config/token_reduction_config_spec.rb
+- test/metadata_types_test.rb
+- vendor/Cargo.toml
+- vendor/kreuzberg-ffi/Cargo.toml
+- vendor/kreuzberg-ffi/README.md
+- vendor/kreuzberg-ffi/benches/result_view_benchmark.rs
+- vendor/kreuzberg-ffi/build.rs
+- vendor/kreuzberg-ffi/cbindgen.toml
+- vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in
+- vendor/kreuzberg-ffi/kreuzberg.h
+- vendor/kreuzberg-ffi/src/batch_streaming.rs
+- vendor/kreuzberg-ffi/src/config.rs
+- vendor/kreuzberg-ffi/src/error.rs
+- vendor/kreuzberg-ffi/src/extraction.rs
+- vendor/kreuzberg-ffi/src/helpers.rs
+- vendor/kreuzberg-ffi/src/lib.rs
+- vendor/kreuzberg-ffi/src/memory.rs
+- vendor/kreuzberg-ffi/src/mime.rs
+- vendor/kreuzberg-ffi/src/panic_shield.rs
+- vendor/kreuzberg-ffi/src/plugins/document_extractor.rs
+- vendor/kreuzberg-ffi/src/plugins/mod.rs
+- vendor/kreuzberg-ffi/src/plugins/ocr_backend.rs
+- vendor/kreuzberg-ffi/src/plugins/post_processor.rs
+- vendor/kreuzberg-ffi/src/plugins/validator.rs
+- vendor/kreuzberg-ffi/src/result.rs
+- vendor/kreuzberg-ffi/src/result_pool.rs
+- vendor/kreuzberg-ffi/src/result_view.rs
+- vendor/kreuzberg-ffi/src/string_intern.rs
+- vendor/kreuzberg-ffi/src/types.rs
+- vendor/kreuzberg-ffi/src/util.rs
+- vendor/kreuzberg-ffi/src/validation.rs
+- vendor/kreuzberg-ffi/tests.disabled/README.md
+- vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs
+- vendor/kreuzberg-ffi/tests.disabled/config_tests.rs
+- vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs
+- vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs
+- vendor/kreuzberg-tesseract/.commitlintrc.json
+- vendor/kreuzberg-tesseract/.crate-ignore
+- vendor/kreuzberg-tesseract/Cargo.lock
+- vendor/kreuzberg-tesseract/Cargo.toml
+- vendor/kreuzberg-tesseract/LICENSE
+- vendor/kreuzberg-tesseract/README.md
+- vendor/kreuzberg-tesseract/build.rs
+- vendor/kreuzberg-tesseract/patches/README.md
+- vendor/kreuzberg-tesseract/patches/tesseract.diff
+- vendor/kreuzberg-tesseract/src/api.rs
+- vendor/kreuzberg-tesseract/src/choice_iterator.rs
+- vendor/kreuzberg-tesseract/src/enums.rs
+- vendor/kreuzberg-tesseract/src/error.rs
+- vendor/kreuzberg-tesseract/src/lib.rs
+- vendor/kreuzberg-tesseract/src/monitor.rs
+- vendor/kreuzberg-tesseract/src/mutable_iterator.rs
+- vendor/kreuzberg-tesseract/src/page_iterator.rs
+- vendor/kreuzberg-tesseract/src/result_iterator.rs
+- vendor/kreuzberg-tesseract/src/result_renderer.rs
+- vendor/kreuzberg-tesseract/tests/integration_test.rs
 - vendor/kreuzberg/Cargo.toml
 - vendor/kreuzberg/README.md
-- vendor/kreuzberg/benches/otel_overhead.rs
 - vendor/kreuzberg/build.rs
+- vendor/kreuzberg/examples/bench_fixes.rs
+- vendor/kreuzberg/examples/test_pdfium_fork.rs
 - vendor/kreuzberg/src/api/error.rs
 - vendor/kreuzberg/src/api/handlers.rs
 - vendor/kreuzberg/src/api/mod.rs
@@ -171,16 +327,22 @@ files:
 - vendor/kreuzberg/src/api/types.rs
 - vendor/kreuzberg/src/cache/mod.rs
 - vendor/kreuzberg/src/chunking/mod.rs
+- vendor/kreuzberg/src/chunking/processor.rs
 - vendor/kreuzberg/src/core/batch_mode.rs
+- vendor/kreuzberg/src/core/batch_optimizations.rs
 - vendor/kreuzberg/src/core/config.rs
+- vendor/kreuzberg/src/core/config_validation.rs
 - vendor/kreuzberg/src/core/extractor.rs
+- vendor/kreuzberg/src/core/formats.rs
 - vendor/kreuzberg/src/core/io.rs
 - vendor/kreuzberg/src/core/mime.rs
 - vendor/kreuzberg/src/core/mod.rs
 - vendor/kreuzberg/src/core/pipeline.rs
+- vendor/kreuzberg/src/core/server_config.rs
 - vendor/kreuzberg/src/embeddings.rs
 - vendor/kreuzberg/src/error.rs
 - vendor/kreuzberg/src/extraction/archive.rs
+- vendor/kreuzberg/src/extraction/capacity.rs
 - vendor/kreuzberg/src/extraction/docx.rs
 - vendor/kreuzberg/src/extraction/email.rs
 - vendor/kreuzberg/src/extraction/excel.rs
@@ -207,7 +369,6 @@ files:
 - vendor/kreuzberg/src/extractors/epub.rs
 - vendor/kreuzberg/src/extractors/excel.rs
 - vendor/kreuzberg/src/extractors/fictionbook.rs
-- vendor/kreuzberg/src/extractors/fictionbook.rs.backup2
 - vendor/kreuzberg/src/extractors/html.rs
 - vendor/kreuzberg/src/extractors/image.rs
 - vendor/kreuzberg/src/extractors/jats.rs
@@ -239,12 +400,14 @@ files:
 - vendor/kreuzberg/src/keywords/types.rs
 - vendor/kreuzberg/src/keywords/yake.rs
 - vendor/kreuzberg/src/language_detection/mod.rs
+- vendor/kreuzberg/src/language_detection/processor.rs
 - vendor/kreuzberg/src/lib.rs
 - vendor/kreuzberg/src/mcp/mod.rs
 - vendor/kreuzberg/src/mcp/server.rs
 - vendor/kreuzberg/src/ocr/cache.rs
 - vendor/kreuzberg/src/ocr/error.rs
 - vendor/kreuzberg/src/ocr/hocr.rs
+- vendor/kreuzberg/src/ocr/language_registry.rs
 - vendor/kreuzberg/src/ocr/mod.rs
 - vendor/kreuzberg/src/ocr/processor.rs
 - vendor/kreuzberg/src/ocr/table/mod.rs
@@ -254,7 +417,11 @@ files:
 - vendor/kreuzberg/src/ocr/utils.rs
 - vendor/kreuzberg/src/ocr/validation.rs
 - vendor/kreuzberg/src/panic_context.rs
+- vendor/kreuzberg/src/pdf/bindings.rs
+- vendor/kreuzberg/src/pdf/bundled.rs
 - vendor/kreuzberg/src/pdf/error.rs
+- vendor/kreuzberg/src/pdf/fonts.rs
+- vendor/kreuzberg/src/pdf/hierarchy.rs
 - vendor/kreuzberg/src/pdf/images.rs
 - vendor/kreuzberg/src/pdf/metadata.rs
 - vendor/kreuzberg/src/pdf/mod.rs
@@ -271,6 +438,7 @@ files:
 - vendor/kreuzberg/src/stopwords/mod.rs
 - vendor/kreuzberg/src/text/mod.rs
 - vendor/kreuzberg/src/text/quality.rs
+- vendor/kreuzberg/src/text/quality_processor.rs
 - vendor/kreuzberg/src/text/string_utils.rs
 - vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs
 - vendor/kreuzberg/src/text/token_reduction/config.rs
@@ -279,9 +447,13 @@ files:
 - vendor/kreuzberg/src/text/token_reduction/mod.rs
 - vendor/kreuzberg/src/text/token_reduction/semantic.rs
 - vendor/kreuzberg/src/text/token_reduction/simd_text.rs
+- vendor/kreuzberg/src/text/utf8_validation.rs
 - vendor/kreuzberg/src/types.rs
 - vendor/kreuzberg/src/utils/mod.rs
+- vendor/kreuzberg/src/utils/pool.rs
+- vendor/kreuzberg/src/utils/pool_sizing.rs
 - vendor/kreuzberg/src/utils/quality.rs
+- vendor/kreuzberg/src/utils/string_pool.rs
 - vendor/kreuzberg/src/utils/string_utils.rs
 - vendor/kreuzberg/stopwords/af_stopwords.json
 - vendor/kreuzberg/stopwords/ar_stopwords.json
@@ -347,17 +519,23 @@ files:
 - vendor/kreuzberg/stopwords/yo_stopwords.json
 - vendor/kreuzberg/stopwords/zh_stopwords.json
 - vendor/kreuzberg/stopwords/zu_stopwords.json
+- vendor/kreuzberg/tests/api_embed.rs
 - vendor/kreuzberg/tests/api_extract_multipart.rs
+- vendor/kreuzberg/tests/api_large_pdf_extraction.rs
+- vendor/kreuzberg/tests/api_large_pdf_extraction_diagnostics.rs
 - vendor/kreuzberg/tests/api_tests.rs
 - vendor/kreuzberg/tests/archive_integration.rs
 - vendor/kreuzberg/tests/batch_orchestration.rs
+- vendor/kreuzberg/tests/batch_pooling_benchmark.rs
 - vendor/kreuzberg/tests/batch_processing.rs
 - vendor/kreuzberg/tests/bibtex_parity_test.rs
 - vendor/kreuzberg/tests/concurrency_stress.rs
 - vendor/kreuzberg/tests/config_features.rs
+- vendor/kreuzberg/tests/config_integration_test.rs
 - vendor/kreuzberg/tests/config_loading_tests.rs
 - vendor/kreuzberg/tests/core_integration.rs
 - vendor/kreuzberg/tests/csv_integration.rs
+- vendor/kreuzberg/tests/data/hierarchy_ground_truth.json
 - vendor/kreuzberg/tests/docbook_extractor_tests.rs
 - vendor/kreuzberg/tests/docx_metadata_extraction_test.rs
 - vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs
@@ -379,12 +557,19 @@ files:
 - vendor/kreuzberg/tests/mime_detection.rs
 - vendor/kreuzberg/tests/ocr_configuration.rs
 - vendor/kreuzberg/tests/ocr_errors.rs
+- vendor/kreuzberg/tests/ocr_language_registry.rs
 - vendor/kreuzberg/tests/ocr_quality.rs
 - vendor/kreuzberg/tests/ocr_stress.rs
 - vendor/kreuzberg/tests/odt_extractor_tests.rs
 - vendor/kreuzberg/tests/opml_extractor_tests.rs
 - vendor/kreuzberg/tests/orgmode_extractor_tests.rs
+- vendor/kreuzberg/tests/page_markers.rs
+- vendor/kreuzberg/tests/pdf_hierarchy_detection.rs
+- vendor/kreuzberg/tests/pdf_hierarchy_quality.rs
 - vendor/kreuzberg/tests/pdf_integration.rs
+- vendor/kreuzberg/tests/pdf_ocr_triggering.rs
+- vendor/kreuzberg/tests/pdf_text_merging.rs
+- vendor/kreuzberg/tests/pdfium_linking.rs
 - vendor/kreuzberg/tests/pipeline_integration.rs
 - vendor/kreuzberg/tests/plugin_ocr_backend_test.rs
 - vendor/kreuzberg/tests/plugin_postprocessor_test.rs
@@ -399,51 +584,17 @@ files:
 - vendor/kreuzberg/tests/typst_behavioral_tests.rs
 - vendor/kreuzberg/tests/typst_extractor_tests.rs
 - vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs
-- vendor/rb-sys/.cargo-ok
-- vendor/rb-sys/.cargo_vcs_info.json
-- vendor/rb-sys/Cargo.lock
-- vendor/rb-sys/Cargo.toml
-- vendor/rb-sys/Cargo.toml.orig
-- vendor/rb-sys/LICENSE-APACHE
-- vendor/rb-sys/LICENSE-MIT
-- vendor/rb-sys/bin/release.sh
-- vendor/rb-sys/build/features.rs
-- vendor/rb-sys/build/main.rs
-- vendor/rb-sys/build/stable_api_config.rs
-- vendor/rb-sys/build/version.rs
-- vendor/rb-sys/readme.md
-- vendor/rb-sys/src/bindings.rs
-- vendor/rb-sys/src/hidden.rs
-- vendor/rb-sys/src/lib.rs
-- vendor/rb-sys/src/macros.rs
-- vendor/rb-sys/src/memory.rs
-- vendor/rb-sys/src/ruby_abi_version.rs
-- vendor/rb-sys/src/special_consts.rs
-- vendor/rb-sys/src/stable_api.rs
-- vendor/rb-sys/src/stable_api/compiled.c
-- vendor/rb-sys/src/stable_api/compiled.rs
-- vendor/rb-sys/src/stable_api/ruby_2_6.rs
-- vendor/rb-sys/src/stable_api/ruby_2_7.rs
-- vendor/rb-sys/src/stable_api/ruby_3_0.rs
-- vendor/rb-sys/src/stable_api/ruby_3_1.rs
-- vendor/rb-sys/src/stable_api/ruby_3_2.rs
-- vendor/rb-sys/src/stable_api/ruby_3_3.rs
-- vendor/rb-sys/src/stable_api/ruby_3_4.rs
-- vendor/rb-sys/src/symbol.rs
-- vendor/rb-sys/src/tracking_allocator.rs
-- vendor/rb-sys/src/utils.rs
-- vendor/rb-sys/src/value_type.rs
 homepage: https://github.com/kreuzberg-dev/kreuzberg
 licenses:
 - MIT
 metadata:
-  bug_tracker_uri: https://github.com/kreuzberg-dev/kreuzberg/issues
+  homepage_uri: https://github.com/kreuzberg-dev/kreuzberg
+  source_code_uri: https://github.com/kreuzberg-dev/kreuzberg
   changelog_uri: https://github.com/kreuzberg-dev/kreuzberg/blob/main/CHANGELOG.md
   documentation_uri: https://docs.kreuzberg.dev
-  homepage_uri: https://github.com/kreuzberg-dev/kreuzberg
-  keywords: document-intelligence,document-extraction,ocr,rust,bindings
+  bug_tracker_uri: https://github.com/kreuzberg-dev/kreuzberg/issues
   rubygems_mfa_required: 'true'
-  source_code_uri: https://github.com/kreuzberg-dev/kreuzberg
+  keywords: document-intelligence,document-extraction,ocr,rust,bindings
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -459,7 +610,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.20
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: High-performance document intelligence framework

data/vendor/kreuzberg/benches/otel_overhead.rs

@@ -1,48 +0,0 @@
-use criterion::{Criterion, criterion_group, criterion_main};
-use std::hint::black_box;
-
-fn bench_text_extraction(c: &mut Criterion) {
-    let runtime = tokio::runtime::Runtime::new().unwrap();
-
-    c.bench_function("extract_text_no_otel", |b| {
-        b.iter(|| {
-            runtime.block_on(async {
-                use kreuzberg::core::config::ExtractionConfig;
-                use kreuzberg::core::extractor::extract_bytes;
-
-                let test_content = black_box(b"Hello, World! This is a test document.");
-                let config = ExtractionConfig::default();
-
-                extract_bytes(test_content, "text/plain", &config).await
-            })
-        });
-    });
-}
-
-fn bench_cache_operations(c: &mut Criterion) {
-    use kreuzberg::cache::GenericCache;
-    use tempfile::tempdir;
-
-    let temp_dir = tempdir().unwrap();
-    let cache = GenericCache::new(
-        "bench".to_string(),
-        Some(temp_dir.path().to_str().unwrap().to_string()),
-        30.0,
-        500.0,
-        1000.0,
-    )
-    .unwrap();
-
-    c.bench_function("cache_set_get", |b| {
-        b.iter(|| {
-            let key = black_box("bench_key");
-            let data = black_box(b"benchmark data".to_vec());
-
-            cache.set(key, data.clone(), None).unwrap();
-            cache.get(key, None).unwrap()
-        });
-    });
-}
-
-criterion_group!(benches, bench_text_extraction, bench_cache_operations);
-criterion_main!(benches);