RubyGems - kreuzberg - Versions diffs - 4.0.0.pre.rc.29 → 4.0.0.rc1 - Mend

kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (321) hide show

checksums.yaml +4 -4
data/.gitignore +0 -6
data/.rubocop.yaml +534 -1
data/Gemfile +2 -1
data/Gemfile.lock +28 -116
data/README.md +269 -629
data/Rakefile +0 -9
data/Steepfile +4 -8
data/examples/async_patterns.rb +58 -1
data/ext/kreuzberg_rb/extconf.rb +5 -35
data/ext/kreuzberg_rb/native/Cargo.toml +16 -55
data/ext/kreuzberg_rb/native/build.rs +14 -12
data/ext/kreuzberg_rb/native/include/ieeefp.h +1 -1
data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +1 -1
data/ext/kreuzberg_rb/native/include/strings.h +2 -2
data/ext/kreuzberg_rb/native/include/unistd.h +1 -1
data/ext/kreuzberg_rb/native/src/lib.rs +34 -897
data/extconf.rb +6 -38
data/kreuzberg.gemspec +20 -114
data/lib/kreuzberg/api_proxy.rb +18 -2
data/lib/kreuzberg/cache_api.rb +0 -22
data/lib/kreuzberg/cli.rb +10 -2
data/lib/kreuzberg/cli_proxy.rb +10 -0
data/lib/kreuzberg/config.rb +22 -274
data/lib/kreuzberg/errors.rb +7 -73
data/lib/kreuzberg/extraction_api.rb +8 -237
data/lib/kreuzberg/mcp_proxy.rb +11 -2
data/lib/kreuzberg/ocr_backend_protocol.rb +73 -0
data/lib/kreuzberg/post_processor_protocol.rb +71 -0
data/lib/kreuzberg/result.rb +33 -151
data/lib/kreuzberg/setup_lib_path.rb +2 -22
data/lib/kreuzberg/validator_protocol.rb +73 -0
data/lib/kreuzberg/version.rb +1 -1
data/lib/kreuzberg.rb +13 -27
data/pkg/kreuzberg-4.0.0.rc1.gem +0 -0
data/sig/kreuzberg.rbs +12 -105
data/spec/binding/cache_spec.rb +22 -22
data/spec/binding/cli_proxy_spec.rb +4 -2
data/spec/binding/cli_spec.rb +11 -12
data/spec/binding/config_spec.rb +0 -74
data/spec/binding/config_validation_spec.rb +6 -100
data/spec/binding/error_handling_spec.rb +97 -283
data/spec/binding/plugins/ocr_backend_spec.rb +8 -8
data/spec/binding/plugins/postprocessor_spec.rb +11 -11
data/spec/binding/plugins/validator_spec.rb +13 -12
data/spec/examples.txt +104 -0
data/spec/fixtures/config.toml +1 -0
data/spec/fixtures/config.yaml +1 -0
data/spec/fixtures/invalid_config.toml +1 -0
data/spec/smoke/package_spec.rb +3 -2
data/spec/spec_helper.rb +3 -1
data/vendor/kreuzberg/Cargo.toml +67 -192
data/vendor/kreuzberg/README.md +9 -97
data/vendor/kreuzberg/build.rs +194 -516
data/vendor/kreuzberg/src/api/handlers.rs +9 -130
data/vendor/kreuzberg/src/api/mod.rs +3 -18
data/vendor/kreuzberg/src/api/server.rs +71 -236
data/vendor/kreuzberg/src/api/types.rs +7 -43
data/vendor/kreuzberg/src/bin/profile_extract.rs +455 -0
data/vendor/kreuzberg/src/cache/mod.rs +3 -27
data/vendor/kreuzberg/src/chunking/mod.rs +79 -1705
data/vendor/kreuzberg/src/core/batch_mode.rs +0 -60
data/vendor/kreuzberg/src/core/config.rs +23 -905
data/vendor/kreuzberg/src/core/extractor.rs +106 -403
data/vendor/kreuzberg/src/core/io.rs +2 -4
data/vendor/kreuzberg/src/core/mime.rs +12 -2
data/vendor/kreuzberg/src/core/mod.rs +3 -22
data/vendor/kreuzberg/src/core/pipeline.rs +78 -395
data/vendor/kreuzberg/src/embeddings.rs +21 -169
data/vendor/kreuzberg/src/error.rs +2 -2
data/vendor/kreuzberg/src/extraction/archive.rs +31 -36
data/vendor/kreuzberg/src/extraction/docx.rs +1 -365
data/vendor/kreuzberg/src/extraction/email.rs +11 -12
data/vendor/kreuzberg/src/extraction/excel.rs +129 -138
data/vendor/kreuzberg/src/extraction/html.rs +170 -1447
data/vendor/kreuzberg/src/extraction/image.rs +14 -138
data/vendor/kreuzberg/src/extraction/libreoffice.rs +3 -13
data/vendor/kreuzberg/src/extraction/mod.rs +5 -21
data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +0 -2
data/vendor/kreuzberg/src/extraction/pandoc/batch.rs +275 -0
data/vendor/kreuzberg/src/extraction/pandoc/mime_types.rs +178 -0
data/vendor/kreuzberg/src/extraction/pandoc/mod.rs +491 -0
data/vendor/kreuzberg/src/extraction/pandoc/server.rs +496 -0
data/vendor/kreuzberg/src/extraction/pandoc/subprocess.rs +1188 -0
data/vendor/kreuzberg/src/extraction/pandoc/version.rs +162 -0
data/vendor/kreuzberg/src/extraction/pptx.rs +94 -196
data/vendor/kreuzberg/src/extraction/structured.rs +4 -5
data/vendor/kreuzberg/src/extraction/table.rs +1 -2
data/vendor/kreuzberg/src/extraction/text.rs +10 -18
data/vendor/kreuzberg/src/extractors/archive.rs +0 -22
data/vendor/kreuzberg/src/extractors/docx.rs +148 -69
data/vendor/kreuzberg/src/extractors/email.rs +9 -37
data/vendor/kreuzberg/src/extractors/excel.rs +40 -81
data/vendor/kreuzberg/src/extractors/html.rs +173 -182
data/vendor/kreuzberg/src/extractors/image.rs +8 -32
data/vendor/kreuzberg/src/extractors/mod.rs +10 -171
data/vendor/kreuzberg/src/extractors/pandoc.rs +201 -0
data/vendor/kreuzberg/src/extractors/pdf.rs +64 -329
data/vendor/kreuzberg/src/extractors/pptx.rs +34 -79
data/vendor/kreuzberg/src/extractors/structured.rs +0 -16
data/vendor/kreuzberg/src/extractors/text.rs +7 -30
data/vendor/kreuzberg/src/extractors/xml.rs +8 -27
data/vendor/kreuzberg/src/keywords/processor.rs +1 -9
data/vendor/kreuzberg/src/keywords/rake.rs +1 -0
data/vendor/kreuzberg/src/language_detection/mod.rs +51 -94
data/vendor/kreuzberg/src/lib.rs +5 -17
data/vendor/kreuzberg/src/mcp/mod.rs +1 -4
data/vendor/kreuzberg/src/mcp/server.rs +21 -145
data/vendor/kreuzberg/src/ocr/mod.rs +0 -2
data/vendor/kreuzberg/src/ocr/processor.rs +8 -19
data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +0 -2
data/vendor/kreuzberg/src/pdf/error.rs +1 -93
data/vendor/kreuzberg/src/pdf/metadata.rs +100 -263
data/vendor/kreuzberg/src/pdf/mod.rs +2 -33
data/vendor/kreuzberg/src/pdf/rendering.rs +12 -12
data/vendor/kreuzberg/src/pdf/table.rs +64 -61
data/vendor/kreuzberg/src/pdf/text.rs +24 -416
data/vendor/kreuzberg/src/plugins/extractor.rs +8 -40
data/vendor/kreuzberg/src/plugins/mod.rs +0 -3
data/vendor/kreuzberg/src/plugins/ocr.rs +14 -22
data/vendor/kreuzberg/src/plugins/processor.rs +1 -10
data/vendor/kreuzberg/src/plugins/registry.rs +0 -15
data/vendor/kreuzberg/src/plugins/validator.rs +8 -20
data/vendor/kreuzberg/src/stopwords/mod.rs +2 -2
data/vendor/kreuzberg/src/text/mod.rs +0 -8
data/vendor/kreuzberg/src/text/quality.rs +15 -28
data/vendor/kreuzberg/src/text/string_utils.rs +10 -22
data/vendor/kreuzberg/src/text/token_reduction/core.rs +50 -86
data/vendor/kreuzberg/src/text/token_reduction/filters.rs +16 -37
data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +1 -2
data/vendor/kreuzberg/src/types.rs +67 -907
data/vendor/kreuzberg/src/utils/mod.rs +0 -14
data/vendor/kreuzberg/src/utils/quality.rs +3 -12
data/vendor/kreuzberg/tests/api_tests.rs +0 -506
data/vendor/kreuzberg/tests/archive_integration.rs +0 -2
data/vendor/kreuzberg/tests/batch_orchestration.rs +12 -57
data/vendor/kreuzberg/tests/batch_processing.rs +8 -32
data/vendor/kreuzberg/tests/chunking_offset_demo.rs +92 -0
data/vendor/kreuzberg/tests/concurrency_stress.rs +8 -40
data/vendor/kreuzberg/tests/config_features.rs +1 -33
data/vendor/kreuzberg/tests/config_loading_tests.rs +39 -16
data/vendor/kreuzberg/tests/core_integration.rs +9 -35
data/vendor/kreuzberg/tests/csv_integration.rs +81 -71
data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +25 -23
data/vendor/kreuzberg/tests/email_integration.rs +1 -3
data/vendor/kreuzberg/tests/error_handling.rs +34 -43
data/vendor/kreuzberg/tests/format_integration.rs +1 -7
data/vendor/kreuzberg/tests/helpers/mod.rs +0 -60
data/vendor/kreuzberg/tests/image_integration.rs +0 -2
data/vendor/kreuzberg/tests/mime_detection.rs +16 -17
data/vendor/kreuzberg/tests/ocr_configuration.rs +0 -4
data/vendor/kreuzberg/tests/ocr_errors.rs +0 -22
data/vendor/kreuzberg/tests/ocr_quality.rs +0 -2
data/vendor/kreuzberg/tests/pandoc_integration.rs +503 -0
data/vendor/kreuzberg/tests/pdf_integration.rs +0 -2
data/vendor/kreuzberg/tests/pipeline_integration.rs +2 -36
data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +0 -5
data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +1 -17
data/vendor/kreuzberg/tests/plugin_system.rs +0 -6
data/vendor/kreuzberg/tests/registry_integration_tests.rs +22 -2
data/vendor/kreuzberg/tests/security_validation.rs +1 -13
data/vendor/kreuzberg/tests/test_fastembed.rs +23 -45
metadata +25 -171
data/.rubocop.yml +0 -543
data/ext/kreuzberg_rb/native/.cargo/config.toml +0 -23
data/ext/kreuzberg_rb/native/Cargo.lock +0 -7619
data/lib/kreuzberg/error_context.rb +0 -136
data/lib/kreuzberg/types.rb +0 -170
data/lib/libpdfium.so +0 -0
data/spec/binding/async_operations_spec.rb +0 -473
data/spec/binding/batch_operations_spec.rb +0 -595
data/spec/binding/batch_spec.rb +0 -359
data/spec/binding/config_result_spec.rb +0 -377
data/spec/binding/embeddings_spec.rb +0 -816
data/spec/binding/error_recovery_spec.rb +0 -488
data/spec/binding/font_config_spec.rb +0 -220
data/spec/binding/images_spec.rb +0 -738
data/spec/binding/keywords_extraction_spec.rb +0 -600
data/spec/binding/metadata_types_spec.rb +0 -1228
data/spec/binding/pages_extraction_spec.rb +0 -471
data/spec/binding/tables_spec.rb +0 -641
data/spec/unit/config/chunking_config_spec.rb +0 -213
data/spec/unit/config/embedding_config_spec.rb +0 -343
data/spec/unit/config/extraction_config_spec.rb +0 -438
data/spec/unit/config/font_config_spec.rb +0 -285
data/spec/unit/config/hierarchy_config_spec.rb +0 -314
data/spec/unit/config/image_extraction_config_spec.rb +0 -209
data/spec/unit/config/image_preprocessing_config_spec.rb +0 -249
data/spec/unit/config/keyword_config_spec.rb +0 -229
data/spec/unit/config/language_detection_config_spec.rb +0 -258
data/spec/unit/config/ocr_config_spec.rb +0 -171
data/spec/unit/config/page_config_spec.rb +0 -221
data/spec/unit/config/pdf_config_spec.rb +0 -267
data/spec/unit/config/postprocessor_config_spec.rb +0 -290
data/spec/unit/config/tesseract_config_spec.rb +0 -181
data/spec/unit/config/token_reduction_config_spec.rb +0 -251
data/test/metadata_types_test.rb +0 -959
data/vendor/Cargo.toml +0 -61
data/vendor/kreuzberg/examples/bench_fixes.rs +0 -71
data/vendor/kreuzberg/examples/test_pdfium_fork.rs +0 -62
data/vendor/kreuzberg/src/chunking/processor.rs +0 -219
data/vendor/kreuzberg/src/core/batch_optimizations.rs +0 -385
data/vendor/kreuzberg/src/core/config_validation.rs +0 -949
data/vendor/kreuzberg/src/core/formats.rs +0 -235
data/vendor/kreuzberg/src/core/server_config.rs +0 -1220
data/vendor/kreuzberg/src/extraction/capacity.rs +0 -263
data/vendor/kreuzberg/src/extraction/markdown.rs +0 -216
data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +0 -284
data/vendor/kreuzberg/src/extractors/bibtex.rs +0 -470
data/vendor/kreuzberg/src/extractors/docbook.rs +0 -504
data/vendor/kreuzberg/src/extractors/epub.rs +0 -696
data/vendor/kreuzberg/src/extractors/fictionbook.rs +0 -492
data/vendor/kreuzberg/src/extractors/jats.rs +0 -1054
data/vendor/kreuzberg/src/extractors/jupyter.rs +0 -368
data/vendor/kreuzberg/src/extractors/latex.rs +0 -653
data/vendor/kreuzberg/src/extractors/markdown.rs +0 -701
data/vendor/kreuzberg/src/extractors/odt.rs +0 -628
data/vendor/kreuzberg/src/extractors/opml.rs +0 -635
data/vendor/kreuzberg/src/extractors/orgmode.rs +0 -529
data/vendor/kreuzberg/src/extractors/rst.rs +0 -577
data/vendor/kreuzberg/src/extractors/rtf.rs +0 -809
data/vendor/kreuzberg/src/extractors/security.rs +0 -484
data/vendor/kreuzberg/src/extractors/security_tests.rs +0 -367
data/vendor/kreuzberg/src/extractors/typst.rs +0 -651
data/vendor/kreuzberg/src/language_detection/processor.rs +0 -218
data/vendor/kreuzberg/src/ocr/language_registry.rs +0 -520
data/vendor/kreuzberg/src/panic_context.rs +0 -154
data/vendor/kreuzberg/src/pdf/bindings.rs +0 -306
data/vendor/kreuzberg/src/pdf/bundled.rs +0 -408
data/vendor/kreuzberg/src/pdf/fonts.rs +0 -358
data/vendor/kreuzberg/src/pdf/hierarchy.rs +0 -903
data/vendor/kreuzberg/src/text/quality_processor.rs +0 -231
data/vendor/kreuzberg/src/text/utf8_validation.rs +0 -193
data/vendor/kreuzberg/src/utils/pool.rs +0 -503
data/vendor/kreuzberg/src/utils/pool_sizing.rs +0 -364
data/vendor/kreuzberg/src/utils/string_pool.rs +0 -761
data/vendor/kreuzberg/tests/api_embed.rs +0 -360
data/vendor/kreuzberg/tests/api_extract_multipart.rs +0 -52
data/vendor/kreuzberg/tests/api_large_pdf_extraction.rs +0 -471
data/vendor/kreuzberg/tests/api_large_pdf_extraction_diagnostics.rs +0 -289
data/vendor/kreuzberg/tests/batch_pooling_benchmark.rs +0 -154
data/vendor/kreuzberg/tests/bibtex_parity_test.rs +0 -421
data/vendor/kreuzberg/tests/config_integration_test.rs +0 -753
data/vendor/kreuzberg/tests/data/hierarchy_ground_truth.json +0 -294
data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +0 -500
data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +0 -370
data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +0 -275
data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +0 -228
data/vendor/kreuzberg/tests/html_table_test.rs +0 -551
data/vendor/kreuzberg/tests/instrumentation_test.rs +0 -139
data/vendor/kreuzberg/tests/jats_extractor_tests.rs +0 -639
data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +0 -704
data/vendor/kreuzberg/tests/latex_extractor_tests.rs +0 -496
data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +0 -490
data/vendor/kreuzberg/tests/ocr_language_registry.rs +0 -191
data/vendor/kreuzberg/tests/odt_extractor_tests.rs +0 -674
data/vendor/kreuzberg/tests/opml_extractor_tests.rs +0 -616
data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +0 -822
data/vendor/kreuzberg/tests/page_markers.rs +0 -297
data/vendor/kreuzberg/tests/pdf_hierarchy_detection.rs +0 -301
data/vendor/kreuzberg/tests/pdf_hierarchy_quality.rs +0 -589
data/vendor/kreuzberg/tests/pdf_ocr_triggering.rs +0 -301
data/vendor/kreuzberg/tests/pdf_text_merging.rs +0 -475
data/vendor/kreuzberg/tests/pdfium_linking.rs +0 -340
data/vendor/kreuzberg/tests/rst_extractor_tests.rs +0 -694
data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +0 -775
data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +0 -1260
data/vendor/kreuzberg/tests/typst_extractor_tests.rs +0 -648
data/vendor/kreuzberg-ffi/Cargo.toml +0 -67
data/vendor/kreuzberg-ffi/README.md +0 -851
data/vendor/kreuzberg-ffi/benches/result_view_benchmark.rs +0 -227
data/vendor/kreuzberg-ffi/build.rs +0 -168
data/vendor/kreuzberg-ffi/cbindgen.toml +0 -37
data/vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in +0 -12
data/vendor/kreuzberg-ffi/kreuzberg.h +0 -3012
data/vendor/kreuzberg-ffi/src/batch_streaming.rs +0 -588
data/vendor/kreuzberg-ffi/src/config.rs +0 -1341
data/vendor/kreuzberg-ffi/src/error.rs +0 -901
data/vendor/kreuzberg-ffi/src/extraction.rs +0 -555
data/vendor/kreuzberg-ffi/src/helpers.rs +0 -879
data/vendor/kreuzberg-ffi/src/lib.rs +0 -977
data/vendor/kreuzberg-ffi/src/memory.rs +0 -493
data/vendor/kreuzberg-ffi/src/mime.rs +0 -329
data/vendor/kreuzberg-ffi/src/panic_shield.rs +0 -265
data/vendor/kreuzberg-ffi/src/plugins/document_extractor.rs +0 -442
data/vendor/kreuzberg-ffi/src/plugins/mod.rs +0 -14
data/vendor/kreuzberg-ffi/src/plugins/ocr_backend.rs +0 -628
data/vendor/kreuzberg-ffi/src/plugins/post_processor.rs +0 -438
data/vendor/kreuzberg-ffi/src/plugins/validator.rs +0 -329
data/vendor/kreuzberg-ffi/src/result.rs +0 -510
data/vendor/kreuzberg-ffi/src/result_pool.rs +0 -639
data/vendor/kreuzberg-ffi/src/result_view.rs +0 -773
data/vendor/kreuzberg-ffi/src/string_intern.rs +0 -568
data/vendor/kreuzberg-ffi/src/types.rs +0 -363
data/vendor/kreuzberg-ffi/src/util.rs +0 -210
data/vendor/kreuzberg-ffi/src/validation.rs +0 -848
data/vendor/kreuzberg-ffi/tests.disabled/README.md +0 -48
data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs +0 -299
data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs +0 -346
data/vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs +0 -232
data/vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs +0 -470
data/vendor/kreuzberg-tesseract/.commitlintrc.json +0 -13
data/vendor/kreuzberg-tesseract/.crate-ignore +0 -2
data/vendor/kreuzberg-tesseract/Cargo.lock +0 -2933
data/vendor/kreuzberg-tesseract/Cargo.toml +0 -57
data/vendor/kreuzberg-tesseract/LICENSE +0 -22
data/vendor/kreuzberg-tesseract/README.md +0 -399
data/vendor/kreuzberg-tesseract/build.rs +0 -1127
data/vendor/kreuzberg-tesseract/patches/README.md +0 -71
data/vendor/kreuzberg-tesseract/patches/tesseract.diff +0 -199
data/vendor/kreuzberg-tesseract/src/api.rs +0 -1371
data/vendor/kreuzberg-tesseract/src/choice_iterator.rs +0 -77
data/vendor/kreuzberg-tesseract/src/enums.rs +0 -297
data/vendor/kreuzberg-tesseract/src/error.rs +0 -81
data/vendor/kreuzberg-tesseract/src/lib.rs +0 -145
data/vendor/kreuzberg-tesseract/src/monitor.rs +0 -57
data/vendor/kreuzberg-tesseract/src/mutable_iterator.rs +0 -197
data/vendor/kreuzberg-tesseract/src/page_iterator.rs +0 -253
data/vendor/kreuzberg-tesseract/src/result_iterator.rs +0 -286
data/vendor/kreuzberg-tesseract/src/result_renderer.rs +0 -183
data/vendor/kreuzberg-tesseract/tests/integration_test.rs +0 -211

data/vendor/kreuzberg/src/core/io.rs CHANGED Viewed

@@ -4,6 +4,7 @@
 use crate::{KreuzbergError, Result};
 use std::path::Path;
+use tokio::fs;
 /// Read a file asynchronously.
 ///
@@ -18,9 +19,8 @@ use std::path::Path;
 /// # Errors
 ///
 /// Returns `KreuzbergError::Io` for I/O errors (these always bubble up).
-#[cfg(feature = "tokio-runtime")]
 pub async fn read_file_async(path: impl AsRef<Path>) -> Result<Vec<u8>> {
-    tokio::fs::read(path.as_ref()).await.map_err(KreuzbergError::Io)
+    fs::read(path.as_ref()).await.map_err(KreuzbergError::Io)
 }
 /// Read a file synchronously.
@@ -181,7 +181,6 @@ mod tests {
     use std::io::Write;
     use tempfile::tempdir;
-    #[cfg(feature = "tokio-runtime")]
     #[tokio::test]
     async fn test_read_file_async() {
         let dir = tempdir().unwrap();
@@ -312,7 +311,6 @@ mod tests {
         assert!(result.is_err());
     }
-    #[cfg(feature = "tokio-runtime")]
     #[tokio::test]
     async fn test_read_file_async_io_error() {
         let result = read_file_async("/nonexistent/file.txt").await;

data/vendor/kreuzberg/src/core/mime.rs CHANGED Viewed

@@ -152,7 +152,6 @@ static SUPPORTED_MIME_TYPES: Lazy<HashSet<&'static str>> = Lazy::new(|| {
     set.insert("application/x-ipynb+json");
     set.insert("application/x-jats+xml");
     set.insert("application/x-latex");
-    set.insert("application/xml+opml");
     set.insert("application/x-opml+xml");
     set.insert("application/x-research-info-systems");
     set.insert("application/x-typst");
@@ -165,7 +164,6 @@ static SUPPORTED_MIME_TYPES: Lazy<HashSet<&'static str>> = Lazy::new(|| {
     set.insert("text/x-markdown-extra");
     set.insert("text/x-mdoc");
     set.insert("text/x-multimarkdown");
-    set.insert("text/x-opml");
     set.insert("text/x-org");
     set.insert("text/x-pod");
     set.insert("text/x-rst");
@@ -329,35 +327,43 @@ pub fn detect_or_validate(path: Option<&Path>, mime_type: Option<&str>) -> Resul
 ///
 /// Returns `KreuzbergError::UnsupportedFormat` if MIME type cannot be determined.
 pub fn detect_mime_type_from_bytes(content: &[u8]) -> Result<String> {
+    // Use infer crate for magic byte detection
     if let Some(kind) = infer::get(content) {
         let mime_type = kind.mime_type();
+        // Validate that it's a supported type
         if SUPPORTED_MIME_TYPES.contains(mime_type) || mime_type.starts_with("image/") {
             return Ok(mime_type.to_string());
         }
     }
+    // Try to detect text-based formats
     if let Ok(text) = std::str::from_utf8(content) {
         let trimmed = text.trim_start();
+        // Detect JSON
         if (trimmed.starts_with('{') || trimmed.starts_with('['))
             && serde_json::from_str::<serde_json::Value>(text).is_ok()
         {
             return Ok(JSON_MIME_TYPE.to_string());
         }
+        // Detect XML
         if trimmed.starts_with("<?xml") || trimmed.starts_with('<') {
             return Ok(XML_MIME_TYPE.to_string());
         }
+        // Detect HTML
         if trimmed.starts_with("<!DOCTYPE html") || trimmed.starts_with("<html") {
             return Ok(HTML_MIME_TYPE.to_string());
         }
+        // Detect PDF header
         if trimmed.starts_with("%PDF") {
             return Ok(PDF_MIME_TYPE.to_string());
         }
+        // Default to plain text for valid UTF-8
         return Ok(PLAIN_TEXT_MIME_TYPE.to_string());
     }
@@ -392,21 +398,25 @@ pub fn detect_mime_type_from_bytes(content: &[u8]) -> Result<String> {
 pub fn get_extensions_for_mime(mime_type: &str) -> Result<Vec<String>> {
     let mut extensions = Vec::new();
+    // Search through EXT_TO_MIME for matching MIME types
     for (ext, mime) in EXT_TO_MIME.iter() {
         if *mime == mime_type {
             extensions.push(ext.to_string());
         }
     }
+    // If we found extensions, return them
     if !extensions.is_empty() {
         return Ok(extensions);
     }
+    // Try using mime_guess crate for reverse lookup
     let guessed = mime_guess::get_mime_extensions_str(mime_type);
     if let Some(exts) = guessed {
         return Ok(exts.iter().map(|s| s.to_string()).collect());
     }
+    // No extensions found
     Err(KreuzbergError::UnsupportedFormat(format!(
         "No known extensions for MIME type: {}",
         mime_type

data/vendor/kreuzberg/src/core/mod.rs CHANGED Viewed

@@ -28,34 +28,15 @@
 //! # }
 //! ```
-#[cfg(feature = "tokio-runtime")]
 pub(crate) mod batch_mode;
-#[cfg(feature = "tokio-runtime")]
-pub mod batch_optimizations;
 pub mod config;
-pub mod config_validation;
 pub mod extractor;
-pub mod formats;
 pub mod io;
 pub mod mime;
 pub mod pipeline;
-pub mod server_config;
 pub use config::{
-    ChunkingConfig, ExtractionConfig, ImageExtractionConfig, LanguageDetectionConfig, OcrConfig, TokenReductionConfig,
+    ChunkingConfig, ExtractionConfig, ImageExtractionConfig, LanguageDetectionConfig, OcrConfig, PdfConfig,
+    TokenReductionConfig,
 };
-pub use config_validation::{
-    validate_binarization_method, validate_chunking_params, validate_confidence, validate_dpi, validate_language_code,
-    validate_ocr_backend, validate_output_format, validate_tesseract_oem, validate_tesseract_psm,
-    validate_token_reduction_level,
-};
-pub use formats::{KNOWN_FORMATS, is_valid_format_field};
-pub use server_config::ServerConfig;
-#[cfg(feature = "tokio-runtime")]
-pub use batch_optimizations::{BatchProcessor, BatchProcessorConfig};
-#[cfg(feature = "pdf")]
-pub use config::PdfConfig;
-#[cfg(feature = "tokio-runtime")]
-pub use extractor::{batch_extract_bytes, batch_extract_file};
-pub use extractor::{extract_bytes, extract_file};
+pub use extractor::{batch_extract_bytes, batch_extract_file, extract_bytes, extract_file};