RubyGems - kreuzberg - Versions diffs - 4.0.0.pre.rc.6 - Mend

kreuzberg 4.0.0.pre.rc.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (330) hide show

checksums.yaml +7 -0
data/.gitignore +14 -0
data/.rspec +3 -0
data/.rubocop.yaml +1 -0
data/.rubocop.yml +538 -0
data/Gemfile +8 -0
data/Gemfile.lock +157 -0
data/README.md +426 -0
data/Rakefile +25 -0
data/Steepfile +47 -0
data/examples/async_patterns.rb +341 -0
data/ext/kreuzberg_rb/extconf.rb +45 -0
data/ext/kreuzberg_rb/native/Cargo.lock +6535 -0
data/ext/kreuzberg_rb/native/Cargo.toml +44 -0
data/ext/kreuzberg_rb/native/README.md +425 -0
data/ext/kreuzberg_rb/native/build.rs +15 -0
data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -0
data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -0
data/ext/kreuzberg_rb/native/include/strings.h +20 -0
data/ext/kreuzberg_rb/native/include/unistd.h +47 -0
data/ext/kreuzberg_rb/native/src/lib.rs +2998 -0
data/extconf.rb +28 -0
data/kreuzberg.gemspec +148 -0
data/lib/kreuzberg/api_proxy.rb +142 -0
data/lib/kreuzberg/cache_api.rb +46 -0
data/lib/kreuzberg/cli.rb +55 -0
data/lib/kreuzberg/cli_proxy.rb +127 -0
data/lib/kreuzberg/config.rb +691 -0
data/lib/kreuzberg/error_context.rb +32 -0
data/lib/kreuzberg/errors.rb +118 -0
data/lib/kreuzberg/extraction_api.rb +85 -0
data/lib/kreuzberg/mcp_proxy.rb +186 -0
data/lib/kreuzberg/ocr_backend_protocol.rb +113 -0
data/lib/kreuzberg/post_processor_protocol.rb +86 -0
data/lib/kreuzberg/result.rb +216 -0
data/lib/kreuzberg/setup_lib_path.rb +80 -0
data/lib/kreuzberg/validator_protocol.rb +89 -0
data/lib/kreuzberg/version.rb +5 -0
data/lib/kreuzberg.rb +103 -0
data/sig/kreuzberg/internal.rbs +184 -0
data/sig/kreuzberg.rbs +520 -0
data/spec/binding/cache_spec.rb +227 -0
data/spec/binding/cli_proxy_spec.rb +85 -0
data/spec/binding/cli_spec.rb +55 -0
data/spec/binding/config_spec.rb +345 -0
data/spec/binding/config_validation_spec.rb +283 -0
data/spec/binding/error_handling_spec.rb +213 -0
data/spec/binding/errors_spec.rb +66 -0
data/spec/binding/plugins/ocr_backend_spec.rb +307 -0
data/spec/binding/plugins/postprocessor_spec.rb +269 -0
data/spec/binding/plugins/validator_spec.rb +274 -0
data/spec/fixtures/config.toml +39 -0
data/spec/fixtures/config.yaml +41 -0
data/spec/fixtures/invalid_config.toml +4 -0
data/spec/smoke/package_spec.rb +178 -0
data/spec/spec_helper.rb +42 -0
data/vendor/kreuzberg/Cargo.toml +204 -0
data/vendor/kreuzberg/README.md +175 -0
data/vendor/kreuzberg/benches/otel_overhead.rs +48 -0
data/vendor/kreuzberg/build.rs +474 -0
data/vendor/kreuzberg/src/api/error.rs +81 -0
data/vendor/kreuzberg/src/api/handlers.rs +199 -0
data/vendor/kreuzberg/src/api/mod.rs +79 -0
data/vendor/kreuzberg/src/api/server.rs +353 -0
data/vendor/kreuzberg/src/api/types.rs +170 -0
data/vendor/kreuzberg/src/cache/mod.rs +1167 -0
data/vendor/kreuzberg/src/chunking/mod.rs +677 -0
data/vendor/kreuzberg/src/core/batch_mode.rs +95 -0
data/vendor/kreuzberg/src/core/config.rs +1032 -0
data/vendor/kreuzberg/src/core/extractor.rs +1024 -0
data/vendor/kreuzberg/src/core/io.rs +329 -0
data/vendor/kreuzberg/src/core/mime.rs +605 -0
data/vendor/kreuzberg/src/core/mod.rs +45 -0
data/vendor/kreuzberg/src/core/pipeline.rs +984 -0
data/vendor/kreuzberg/src/embeddings.rs +432 -0
data/vendor/kreuzberg/src/error.rs +431 -0
data/vendor/kreuzberg/src/extraction/archive.rs +954 -0
data/vendor/kreuzberg/src/extraction/docx.rs +40 -0
data/vendor/kreuzberg/src/extraction/email.rs +854 -0
data/vendor/kreuzberg/src/extraction/excel.rs +688 -0
data/vendor/kreuzberg/src/extraction/html.rs +553 -0
data/vendor/kreuzberg/src/extraction/image.rs +368 -0
data/vendor/kreuzberg/src/extraction/libreoffice.rs +563 -0
data/vendor/kreuzberg/src/extraction/markdown.rs +213 -0
data/vendor/kreuzberg/src/extraction/mod.rs +81 -0
data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -0
data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -0
data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -0
data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +130 -0
data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +287 -0
data/vendor/kreuzberg/src/extraction/pptx.rs +3000 -0
data/vendor/kreuzberg/src/extraction/structured.rs +490 -0
data/vendor/kreuzberg/src/extraction/table.rs +328 -0
data/vendor/kreuzberg/src/extraction/text.rs +269 -0
data/vendor/kreuzberg/src/extraction/xml.rs +333 -0
data/vendor/kreuzberg/src/extractors/archive.rs +446 -0
data/vendor/kreuzberg/src/extractors/bibtex.rs +469 -0
data/vendor/kreuzberg/src/extractors/docbook.rs +502 -0
data/vendor/kreuzberg/src/extractors/docx.rs +367 -0
data/vendor/kreuzberg/src/extractors/email.rs +143 -0
data/vendor/kreuzberg/src/extractors/epub.rs +707 -0
data/vendor/kreuzberg/src/extractors/excel.rs +343 -0
data/vendor/kreuzberg/src/extractors/fictionbook.rs +491 -0
data/vendor/kreuzberg/src/extractors/fictionbook.rs.backup2 +738 -0
data/vendor/kreuzberg/src/extractors/html.rs +393 -0
data/vendor/kreuzberg/src/extractors/image.rs +198 -0
data/vendor/kreuzberg/src/extractors/jats.rs +1051 -0
data/vendor/kreuzberg/src/extractors/jupyter.rs +367 -0
data/vendor/kreuzberg/src/extractors/latex.rs +652 -0
data/vendor/kreuzberg/src/extractors/markdown.rs +700 -0
data/vendor/kreuzberg/src/extractors/mod.rs +365 -0
data/vendor/kreuzberg/src/extractors/odt.rs +628 -0
data/vendor/kreuzberg/src/extractors/opml.rs +634 -0
data/vendor/kreuzberg/src/extractors/orgmode.rs +528 -0
data/vendor/kreuzberg/src/extractors/pdf.rs +493 -0
data/vendor/kreuzberg/src/extractors/pptx.rs +248 -0
data/vendor/kreuzberg/src/extractors/rst.rs +576 -0
data/vendor/kreuzberg/src/extractors/rtf.rs +810 -0
data/vendor/kreuzberg/src/extractors/security.rs +484 -0
data/vendor/kreuzberg/src/extractors/security_tests.rs +367 -0
data/vendor/kreuzberg/src/extractors/structured.rs +140 -0
data/vendor/kreuzberg/src/extractors/text.rs +260 -0
data/vendor/kreuzberg/src/extractors/typst.rs +650 -0
data/vendor/kreuzberg/src/extractors/xml.rs +135 -0
data/vendor/kreuzberg/src/image/dpi.rs +164 -0
data/vendor/kreuzberg/src/image/mod.rs +6 -0
data/vendor/kreuzberg/src/image/preprocessing.rs +417 -0
data/vendor/kreuzberg/src/image/resize.rs +89 -0
data/vendor/kreuzberg/src/keywords/config.rs +154 -0
data/vendor/kreuzberg/src/keywords/mod.rs +237 -0
data/vendor/kreuzberg/src/keywords/processor.rs +267 -0
data/vendor/kreuzberg/src/keywords/rake.rs +293 -0
data/vendor/kreuzberg/src/keywords/types.rs +68 -0
data/vendor/kreuzberg/src/keywords/yake.rs +163 -0
data/vendor/kreuzberg/src/language_detection/mod.rs +942 -0
data/vendor/kreuzberg/src/lib.rs +105 -0
data/vendor/kreuzberg/src/mcp/mod.rs +32 -0
data/vendor/kreuzberg/src/mcp/server.rs +1968 -0
data/vendor/kreuzberg/src/ocr/cache.rs +469 -0
data/vendor/kreuzberg/src/ocr/error.rs +37 -0
data/vendor/kreuzberg/src/ocr/hocr.rs +216 -0
data/vendor/kreuzberg/src/ocr/mod.rs +58 -0
data/vendor/kreuzberg/src/ocr/processor.rs +863 -0
data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -0
data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -0
data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +450 -0
data/vendor/kreuzberg/src/ocr/types.rs +393 -0
data/vendor/kreuzberg/src/ocr/utils.rs +47 -0
data/vendor/kreuzberg/src/ocr/validation.rs +206 -0
data/vendor/kreuzberg/src/panic_context.rs +154 -0
data/vendor/kreuzberg/src/pdf/error.rs +122 -0
data/vendor/kreuzberg/src/pdf/images.rs +139 -0
data/vendor/kreuzberg/src/pdf/metadata.rs +346 -0
data/vendor/kreuzberg/src/pdf/mod.rs +50 -0
data/vendor/kreuzberg/src/pdf/rendering.rs +369 -0
data/vendor/kreuzberg/src/pdf/table.rs +393 -0
data/vendor/kreuzberg/src/pdf/text.rs +158 -0
data/vendor/kreuzberg/src/plugins/extractor.rs +1013 -0
data/vendor/kreuzberg/src/plugins/mod.rs +209 -0
data/vendor/kreuzberg/src/plugins/ocr.rs +620 -0
data/vendor/kreuzberg/src/plugins/processor.rs +642 -0
data/vendor/kreuzberg/src/plugins/registry.rs +1337 -0
data/vendor/kreuzberg/src/plugins/traits.rs +258 -0
data/vendor/kreuzberg/src/plugins/validator.rs +956 -0
data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -0
data/vendor/kreuzberg/src/text/mod.rs +19 -0
data/vendor/kreuzberg/src/text/quality.rs +697 -0
data/vendor/kreuzberg/src/text/string_utils.rs +217 -0
data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -0
data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -0
data/vendor/kreuzberg/src/text/token_reduction/core.rs +796 -0
data/vendor/kreuzberg/src/text/token_reduction/filters.rs +902 -0
data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -0
data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -0
data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +147 -0
data/vendor/kreuzberg/src/types.rs +903 -0
data/vendor/kreuzberg/src/utils/mod.rs +17 -0
data/vendor/kreuzberg/src/utils/quality.rs +959 -0
data/vendor/kreuzberg/src/utils/string_utils.rs +381 -0
data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -0
data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -0
data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -0
data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -0
data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -0
data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -0
data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -0
data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -0
data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -0
data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -0
data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -0
data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -0
data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -0
data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -0
data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -0
data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -0
data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -0
data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -0
data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -0
data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -0
data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -0
data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -0
data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -0
data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -0
data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -0
data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -0
data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -0
data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -0
data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -0
data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -0
data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -0
data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -0
data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -0
data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -0
data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -0
data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -0
data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -0
data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -0
data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -0
data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -0
data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -0
data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -0
data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -0
data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -0
data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -0
data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -0
data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -0
data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -0
data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -0
data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -0
data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -0
data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -0
data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -0
data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -0
data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -0
data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -0
data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -0
data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -0
data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -0
data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -0
data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -0
data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -0
data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -0
data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -0
data/vendor/kreuzberg/tests/api_extract_multipart.rs +52 -0
data/vendor/kreuzberg/tests/api_tests.rs +966 -0
data/vendor/kreuzberg/tests/archive_integration.rs +543 -0
data/vendor/kreuzberg/tests/batch_orchestration.rs +556 -0
data/vendor/kreuzberg/tests/batch_processing.rs +316 -0
data/vendor/kreuzberg/tests/bibtex_parity_test.rs +421 -0
data/vendor/kreuzberg/tests/concurrency_stress.rs +525 -0
data/vendor/kreuzberg/tests/config_features.rs +598 -0
data/vendor/kreuzberg/tests/config_loading_tests.rs +415 -0
data/vendor/kreuzberg/tests/core_integration.rs +510 -0
data/vendor/kreuzberg/tests/csv_integration.rs +414 -0
data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +498 -0
data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +122 -0
data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +370 -0
data/vendor/kreuzberg/tests/email_integration.rs +325 -0
data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +275 -0
data/vendor/kreuzberg/tests/error_handling.rs +393 -0
data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +228 -0
data/vendor/kreuzberg/tests/format_integration.rs +159 -0
data/vendor/kreuzberg/tests/helpers/mod.rs +142 -0
data/vendor/kreuzberg/tests/html_table_test.rs +551 -0
data/vendor/kreuzberg/tests/image_integration.rs +253 -0
data/vendor/kreuzberg/tests/instrumentation_test.rs +139 -0
data/vendor/kreuzberg/tests/jats_extractor_tests.rs +639 -0
data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +704 -0
data/vendor/kreuzberg/tests/keywords_integration.rs +479 -0
data/vendor/kreuzberg/tests/keywords_quality.rs +509 -0
data/vendor/kreuzberg/tests/latex_extractor_tests.rs +496 -0
data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +490 -0
data/vendor/kreuzberg/tests/mime_detection.rs +428 -0
data/vendor/kreuzberg/tests/ocr_configuration.rs +510 -0
data/vendor/kreuzberg/tests/ocr_errors.rs +676 -0
data/vendor/kreuzberg/tests/ocr_quality.rs +627 -0
data/vendor/kreuzberg/tests/ocr_stress.rs +469 -0
data/vendor/kreuzberg/tests/odt_extractor_tests.rs +695 -0
data/vendor/kreuzberg/tests/opml_extractor_tests.rs +616 -0
data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +822 -0
data/vendor/kreuzberg/tests/pdf_integration.rs +43 -0
data/vendor/kreuzberg/tests/pipeline_integration.rs +1411 -0
data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +771 -0
data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +560 -0
data/vendor/kreuzberg/tests/plugin_system.rs +921 -0
data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -0
data/vendor/kreuzberg/tests/registry_integration_tests.rs +586 -0
data/vendor/kreuzberg/tests/rst_extractor_tests.rs +692 -0
data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +776 -0
data/vendor/kreuzberg/tests/security_validation.rs +415 -0
data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -0
data/vendor/kreuzberg/tests/test_fastembed.rs +609 -0
data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +1259 -0
data/vendor/kreuzberg/tests/typst_extractor_tests.rs +647 -0
data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -0
data/vendor/rb-sys/.cargo-ok +1 -0
data/vendor/rb-sys/.cargo_vcs_info.json +6 -0
data/vendor/rb-sys/Cargo.lock +393 -0
data/vendor/rb-sys/Cargo.toml +70 -0
data/vendor/rb-sys/Cargo.toml.orig +57 -0
data/vendor/rb-sys/LICENSE-APACHE +190 -0
data/vendor/rb-sys/LICENSE-MIT +21 -0
data/vendor/rb-sys/bin/release.sh +21 -0
data/vendor/rb-sys/build/features.rs +108 -0
data/vendor/rb-sys/build/main.rs +246 -0
data/vendor/rb-sys/build/stable_api_config.rs +153 -0
data/vendor/rb-sys/build/version.rs +48 -0
data/vendor/rb-sys/readme.md +36 -0
data/vendor/rb-sys/src/bindings.rs +21 -0
data/vendor/rb-sys/src/hidden.rs +11 -0
data/vendor/rb-sys/src/lib.rs +34 -0
data/vendor/rb-sys/src/macros.rs +371 -0
data/vendor/rb-sys/src/memory.rs +53 -0
data/vendor/rb-sys/src/ruby_abi_version.rs +38 -0
data/vendor/rb-sys/src/special_consts.rs +31 -0
data/vendor/rb-sys/src/stable_api/compiled.c +179 -0
data/vendor/rb-sys/src/stable_api/compiled.rs +257 -0
data/vendor/rb-sys/src/stable_api/ruby_2_6.rs +316 -0
data/vendor/rb-sys/src/stable_api/ruby_2_7.rs +316 -0
data/vendor/rb-sys/src/stable_api/ruby_3_0.rs +324 -0
data/vendor/rb-sys/src/stable_api/ruby_3_1.rs +317 -0
data/vendor/rb-sys/src/stable_api/ruby_3_2.rs +315 -0
data/vendor/rb-sys/src/stable_api/ruby_3_3.rs +326 -0
data/vendor/rb-sys/src/stable_api/ruby_3_4.rs +327 -0
data/vendor/rb-sys/src/stable_api.rs +261 -0
data/vendor/rb-sys/src/symbol.rs +31 -0
data/vendor/rb-sys/src/tracking_allocator.rs +332 -0
data/vendor/rb-sys/src/utils.rs +89 -0
data/vendor/rb-sys/src/value_type.rs +7 -0
metadata +536 -0

data/vendor/kreuzberg/tests/core_integration.rs ADDED Viewed

@@ -0,0 +1,510 @@
+//! Integration tests for core extraction functionality.
+//!
+//! These tests verify the end-to-end behavior of the extraction pipeline,
+//! config loading, MIME detection, and batch processing.
+use kreuzberg::{
+    ExtractionConfig, batch_extract_bytes, batch_extract_bytes_sync, batch_extract_file, batch_extract_file_sync,
+    detect_mime_type, extract_bytes, extract_bytes_sync, extract_file, extract_file_sync, validate_mime_type,
+};
+use std::fs::{self, File};
+use std::io::Write;
+use tempfile::tempdir;
+fn trim_trailing_newlines(value: &str) -> &str {
+    value.trim_end_matches(['\n', '\r'])
+}
+fn assert_text_content(actual: &str, expected: &str) {
+    assert_eq!(
+        trim_trailing_newlines(actual),
+        expected,
+        "Content mismatch after trimming trailing newlines"
+    );
+}
+/// Test basic file extraction with MIME detection.
+#[tokio::test]
+async fn test_extract_file_basic() {
+    let dir = tempdir().unwrap();
+    let file_path = dir.path().join("test.txt");
+    let mut file = File::create(&file_path).unwrap();
+    file.write_all(b"Hello, Kreuzberg!").unwrap();
+    let config = ExtractionConfig::default();
+    let result = extract_file(&file_path, None, &config).await;
+    assert!(result.is_ok(), "Basic file extraction should succeed");
+    let result = result.unwrap();
+    assert_text_content(&result.content, "Hello, Kreuzberg!");
+    assert_eq!(result.mime_type, "text/plain");
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+    assert!(result.tables.is_empty(), "Text file should not have tables");
+}
+/// Test extraction with explicit MIME type override.
+#[tokio::test]
+async fn test_extract_file_with_mime_override() {
+    let dir = tempdir().unwrap();
+    let file_path = dir.path().join("data.bin");
+    let mut file = File::create(&file_path).unwrap();
+    file.write_all(b"Binary content").unwrap();
+    let config = ExtractionConfig::default();
+    let result = extract_file(&file_path, Some("text/plain"), &config).await;
+    assert!(result.is_ok(), "MIME override should work");
+    let result = result.unwrap();
+    assert_eq!(result.mime_type, "text/plain");
+    assert!(!result.content.is_empty(), "Should extract content");
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+}
+/// Test extraction of multiple file types.
+#[tokio::test]
+async fn test_extract_multiple_file_types() {
+    let dir = tempdir().unwrap();
+    let config = ExtractionConfig::default();
+    let test_files: Vec<(&str, &[u8], &str)> = vec![
+        ("test.txt", b"text content", "text/plain"),
+        ("test.json", b"{\"key\": \"value\"}", "application/json"),
+        #[cfg(feature = "xml")]
+        ("test.xml", b"<root>data</root>", "application/xml"),
+        #[cfg(feature = "html")]
+        ("test.html", b"<html><body>test</body></html>", "text/html"),
+    ];
+    for (filename, content, expected_mime) in test_files {
+        let file_path = dir.path().join(filename);
+        fs::write(&file_path, content).unwrap();
+        let result = extract_file(&file_path, None, &config).await.unwrap();
+        assert_eq!(result.mime_type, expected_mime, "MIME type mismatch for {}", filename);
+        assert!(
+            !result.content.is_empty(),
+            "Content should not be empty for {}",
+            filename
+        );
+        assert!(result.chunks.is_none(), "Chunks should be None for {}", filename);
+        assert!(
+            result.detected_languages.is_none(),
+            "Language detection not enabled for {}",
+            filename
+        );
+    }
+}
+/// Test extract_bytes with various MIME types.
+#[tokio::test]
+async fn test_extract_bytes_various_mime_types() {
+    let config = ExtractionConfig::default();
+    let test_cases: Vec<(&[u8], &str)> = vec![
+        (b"text content", "text/plain"),
+        (b"{\"key\": \"value\"}", "application/json"),
+        #[cfg(feature = "xml")]
+        (b"<root>data</root>", "application/xml"),
+    ];
+    for (content, mime_type) in test_cases {
+        let result = extract_bytes(content, mime_type, &config).await;
+        assert!(result.is_ok(), "Extract bytes failed for MIME type: {}", mime_type);
+        let result = result.unwrap();
+        assert_eq!(result.mime_type, mime_type, "MIME type mismatch");
+        assert!(
+            !result.content.is_empty(),
+            "Content should not be empty for {}",
+            mime_type
+        );
+        assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+        assert!(result.detected_languages.is_none(), "Language detection not enabled");
+    }
+}
+/// Test batch extraction with concurrent processing.
+#[tokio::test]
+async fn test_batch_extract_file_concurrency() {
+    let dir = tempdir().unwrap();
+    let config = ExtractionConfig::default();
+    let num_files = 10;
+    let mut paths = Vec::new();
+    for i in 0..num_files {
+        let file_path = dir.path().join(format!("test_{}.txt", i));
+        fs::write(&file_path, format!("Content {}", i)).unwrap();
+        paths.push(file_path);
+    }
+    let results = batch_extract_file(paths.clone(), &config).await;
+    assert!(results.is_ok());
+    let results = results.unwrap();
+    assert_eq!(results.len(), num_files);
+    for (i, result) in results.iter().enumerate() {
+        assert!(
+            result.content.contains(&i.to_string()),
+            "Content should contain file number"
+        );
+        assert_eq!(result.mime_type, "text/plain", "MIME type should be text/plain");
+        assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+        assert!(result.detected_languages.is_none(), "Language detection not enabled");
+        assert!(result.metadata.error.is_none(), "Should not have errors");
+    }
+}
+/// Test batch extraction with empty input.
+#[tokio::test]
+async fn test_batch_extract_empty() {
+    let config = ExtractionConfig::default();
+    let paths: Vec<std::path::PathBuf> = vec![];
+    let results = batch_extract_file(paths, &config).await;
+    assert!(results.is_ok());
+    assert_eq!(results.unwrap().len(), 0);
+}
+/// Test batch_extract_bytes with concurrent processing.
+#[tokio::test]
+async fn test_batch_extract_bytes_concurrency() {
+    let config = ExtractionConfig::default();
+    let contents = vec![
+        (b"content 1".as_slice(), "text/plain"),
+        (b"content 2".as_slice(), "text/plain"),
+        (b"content 3".as_slice(), "text/plain"),
+        (b"content 4".as_slice(), "text/plain"),
+        (b"content 5".as_slice(), "text/plain"),
+    ];
+    let results = batch_extract_bytes(contents, &config).await;
+    assert!(results.is_ok());
+    let results = results.unwrap();
+    assert_eq!(results.len(), 5);
+    for (i, result) in results.iter().enumerate() {
+        let expected_content = format!("content {}", i + 1);
+        assert_eq!(
+            trim_trailing_newlines(&result.content),
+            expected_content,
+            "Content mismatch for item {}",
+            i
+        );
+        assert_eq!(result.mime_type, "text/plain", "MIME type should be text/plain");
+        assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+        assert!(result.detected_languages.is_none(), "Language detection not enabled");
+        assert!(result.metadata.error.is_none(), "Should not have errors");
+    }
+}
+/// Test sync wrappers for extraction functions.
+#[test]
+fn test_sync_wrappers() {
+    let dir = tempdir().unwrap();
+    let file_path = dir.path().join("sync_test.txt");
+    fs::write(&file_path, "sync content").unwrap();
+    let config = ExtractionConfig::default();
+    let result = extract_file_sync(&file_path, None, &config);
+    assert!(result.is_ok(), "Sync file extraction should succeed");
+    let extraction = result.unwrap();
+    assert_text_content(&extraction.content, "sync content");
+    assert!(extraction.chunks.is_none(), "Chunks should be None");
+    let result = extract_bytes_sync(b"test bytes", "text/plain", &config);
+    assert!(result.is_ok(), "Sync bytes extraction should succeed");
+    let extraction = result.unwrap();
+    assert_text_content(&extraction.content, "test bytes");
+    assert!(extraction.chunks.is_none(), "Chunks should be None");
+    let paths = vec![file_path];
+    let results = batch_extract_file_sync(paths, &config);
+    assert!(results.is_ok(), "Batch sync file should succeed");
+    let results = results.unwrap();
+    assert_eq!(results.len(), 1);
+    assert_text_content(&results[0].content, "sync content");
+    assert!(results[0].chunks.is_none(), "Chunks should be None");
+    let contents = vec![(b"test".as_slice(), "text/plain")];
+    let results = batch_extract_bytes_sync(contents, &config);
+    assert!(results.is_ok(), "Batch bytes sync should succeed");
+    let results = results.unwrap();
+    assert_eq!(results.len(), 1);
+    assert_text_content(&results[0].content, "test");
+    assert!(results[0].chunks.is_none(), "Chunks should be None");
+}
+/// Test MIME type detection for various extensions.
+#[test]
+fn test_mime_detection_comprehensive() {
+    let dir = tempdir().unwrap();
+    let test_cases = vec![
+        ("test.txt", "text/plain"),
+        ("test.md", "text/markdown"),
+        ("test.html", "text/html"),
+        ("test.json", "application/json"),
+        ("test.yaml", "application/x-yaml"),
+        ("test.toml", "application/toml"),
+        ("test.xml", "application/xml"),
+        ("test.pdf", "application/pdf"),
+        (
+            "test.xlsx",
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        ),
+        (
+            "test.docx",
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        ),
+        (
+            "test.pptx",
+            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        ),
+        ("test.png", "image/png"),
+        ("test.jpg", "image/jpeg"),
+        ("test.gif", "image/gif"),
+        ("test.eml", "message/rfc822"),
+    ];
+    for (filename, expected_mime) in test_cases {
+        let file_path = dir.path().join(filename);
+        File::create(&file_path).unwrap();
+        let detected = detect_mime_type(&file_path, true).unwrap();
+        assert_eq!(detected, expected_mime, "Failed for {}", filename);
+        let validated = validate_mime_type(&detected);
+        assert!(validated.is_ok(), "Validation failed for {}", expected_mime);
+    }
+}
+/// Test MIME type validation.
+#[test]
+fn test_mime_validation() {
+    assert!(validate_mime_type("application/pdf").is_ok());
+    assert!(validate_mime_type("text/plain").is_ok());
+    assert!(validate_mime_type("image/png").is_ok());
+    assert!(validate_mime_type("image/custom-format").is_ok());
+    assert!(validate_mime_type("video/mp4").is_err());
+    assert!(validate_mime_type("application/unknown").is_err());
+}
+/// Test case-insensitive extension handling.
+#[test]
+fn test_case_insensitive_extensions() {
+    let dir = tempdir().unwrap();
+    let test_cases = vec![
+        ("test.PDF", "application/pdf"),
+        ("test.TXT", "text/plain"),
+        ("test.Json", "application/json"),
+        (
+            "test.XLSX",
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        ),
+    ];
+    for (filename, expected_mime) in test_cases {
+        let file_path = dir.path().join(filename);
+        File::create(&file_path).unwrap();
+        let detected = detect_mime_type(&file_path, true).unwrap();
+        assert_eq!(detected, expected_mime, "Failed for {}", filename);
+    }
+}
+/// Test config loading from TOML file.
+#[test]
+fn test_config_loading() {
+    let dir = tempdir().unwrap();
+    let config_path = dir.path().join("kreuzberg.toml");
+    fs::write(
+        &config_path,
+        r#"
+use_cache = false
+enable_quality_processing = true
+force_ocr = false
+[ocr]
+backend = "tesseract"
+language = "deu"
+[chunking]
+max_chars = 2000
+max_overlap = 300
+    "#,
+    )
+    .unwrap();
+    let config = ExtractionConfig::from_toml_file(&config_path).unwrap();
+    assert!(!config.use_cache);
+    assert!(config.enable_quality_processing);
+    assert!(!config.force_ocr);
+    let ocr_config = config.ocr.unwrap();
+    assert_eq!(ocr_config.backend, "tesseract");
+    assert_eq!(ocr_config.language, "deu");
+    let chunking_config = config.chunking.unwrap();
+    assert_eq!(chunking_config.max_chars, 2000);
+    assert_eq!(chunking_config.max_overlap, 300);
+}
+/// Test config discovery in parent directories.
+#[test]
+fn test_config_discovery() {
+    let dir = tempdir().unwrap();
+    let subdir = dir.path().join("subdir");
+    fs::create_dir(&subdir).unwrap();
+    let config_path = dir.path().join("kreuzberg.toml");
+    fs::write(
+        &config_path,
+        r#"
+use_cache = false
+enable_quality_processing = true
+    "#,
+    )
+    .unwrap();
+    let original_dir = std::env::current_dir().unwrap();
+    std::env::set_current_dir(&subdir).unwrap();
+    let config = ExtractionConfig::discover().unwrap();
+    assert!(config.is_some());
+    assert!(!config.unwrap().use_cache);
+    std::env::set_current_dir(original_dir).unwrap();
+}
+/// Test error handling for nonexistent files.
+#[tokio::test]
+async fn test_nonexistent_file_error() {
+    let config = ExtractionConfig::default();
+    let result = extract_file("/nonexistent/file.txt", None, &config).await;
+    assert!(result.is_err());
+    assert!(matches!(
+        result.unwrap_err(),
+        kreuzberg::KreuzbergError::Validation { .. }
+    ));
+}
+/// Test error handling for unsupported MIME types.
+#[tokio::test]
+async fn test_unsupported_mime_type_error() {
+    let config = ExtractionConfig::default();
+    let result = extract_bytes(b"test", "video/mp4", &config).await;
+    assert!(result.is_err());
+    assert!(matches!(
+        result.unwrap_err(),
+        kreuzberg::KreuzbergError::UnsupportedFormat(_)
+    ));
+}
+/// Test pipeline execution (currently stub, will be expanded in Phase 2).
+#[tokio::test]
+async fn test_pipeline_execution() {
+    let dir = tempdir().unwrap();
+    let file_path = dir.path().join("pipeline_test.txt");
+    fs::write(&file_path, "pipeline content").unwrap();
+    let config = ExtractionConfig {
+        enable_quality_processing: true,
+        ..Default::default()
+    };
+    let result = extract_file(&file_path, None, &config).await;
+    assert!(result.is_ok(), "Pipeline execution should succeed");
+    let result = result.unwrap();
+    assert_text_content(&result.content, "pipeline content");
+    assert_eq!(result.mime_type, "text/plain");
+    assert!(result.chunks.is_none(), "Chunks should be None without chunking config");
+    assert!(result.detected_languages.is_none(), "Language detection not enabled");
+}
+/// Test extraction with OCR config (placeholder test for Phase 2).
+#[tokio::test]
+async fn test_extraction_with_ocr_config() {
+    let dir = tempdir().unwrap();
+    let file_path = dir.path().join("ocr_test.txt");
+    fs::write(&file_path, "ocr content").unwrap();
+    let config = ExtractionConfig {
+        ocr: Some(kreuzberg::OcrConfig {
+            tesseract_config: None,
+            backend: "tesseract".to_string(),
+            language: "eng".to_string(),
+        }),
+        force_ocr: true,
+        ..Default::default()
+    };
+    let result = extract_file(&file_path, None, &config).await;
+    assert!(result.is_ok());
+}
+/// Test extraction with chunking config.
+#[cfg(feature = "chunking")]
+#[tokio::test]
+async fn test_extraction_with_chunking_config() {
+    let dir = tempdir().unwrap();
+    let file_path = dir.path().join("chunking_test.txt");
+    let long_content = "content for chunking. ".repeat(100);
+    fs::write(&file_path, &long_content).unwrap();
+    let config = ExtractionConfig {
+        chunking: Some(kreuzberg::ChunkingConfig {
+            max_chars: 100,
+            max_overlap: 20,
+            embedding: None,
+            preset: None,
+        }),
+        ..Default::default()
+    };
+    let result = extract_file(&file_path, None, &config).await;
+    assert!(result.is_ok(), "Extraction with chunking should succeed");
+    let result = result.unwrap();
+    assert!(
+        result.chunks.is_some(),
+        "Chunks should be populated when chunking enabled"
+    );
+    let chunks = result.chunks.unwrap();
+    assert!(chunks.len() > 1, "Should have multiple chunks for long content");
+    assert!(result.metadata.additional.contains_key("chunk_count"));
+    let chunk_count = result.metadata.additional.get("chunk_count").unwrap();
+    assert_eq!(
+        chunks.len(),
+        chunk_count.as_u64().unwrap() as usize,
+        "chunk_count should match chunks length"
+    );
+    for chunk in &chunks {
+        assert!(
+            chunk.content.len() <= 100 + 20,
+            "Chunk length {} exceeds max_chars + overlap",
+            chunk.content.len()
+        );
+    }
+}