kreuzberg 4.0.0.pre.rc.6 → 4.0.0.pre.rc.7

data/vendor/kreuzberg/tests/docbook_extractor_tests.rs

@@ -1,5 +1,7 @@
 //! Comprehensive tests for DocBook extractor supporting both 4.x and 5.x versions.
 
+#![cfg(feature = "xml")]
+
 use kreuzberg::core::config::ExtractionConfig;
 use kreuzberg::plugins::{DocumentExtractor, Plugin};
 use std::path::PathBuf;

data/vendor/kreuzberg/tests/email_integration.rs

@@ -3,6 +3,8 @@
 //! Tests for .eml (RFC822) email extraction.
 //! Validates metadata extraction, content extraction, HTML/plain text handling, and attachments.
 
+#![cfg(feature = "email")]
+
 use kreuzberg::core::config::ExtractionConfig;
 use kreuzberg::core::extractor::extract_bytes;
 

data/vendor/kreuzberg/tests/error_handling.rs

@@ -12,6 +12,7 @@ mod helpers;
 
 /// Test truncated PDF - incomplete PDF file.
 #[tokio::test]
+#[cfg(feature = "pdf")]
 async fn test_truncated_pdf() {
     let config = ExtractionConfig::default();
 
@@ -31,6 +32,7 @@ async fn test_truncated_pdf() {
 
 /// Test corrupted ZIP - malformed archive.
 #[tokio::test]
+#[cfg(feature = "archives")]
 async fn test_corrupted_zip() {
     let config = ExtractionConfig::default();
 
@@ -50,6 +52,7 @@ async fn test_corrupted_zip() {
 
 /// Test invalid XML - bad XML syntax.
 #[tokio::test]
+#[cfg(feature = "xml")]
 async fn test_invalid_xml() {
     let config = ExtractionConfig::default();
 
@@ -80,6 +83,7 @@ async fn test_invalid_xml() {
 
 /// Test corrupted image - invalid image data.
 #[tokio::test]
+#[cfg(feature = "ocr")]
 async fn test_corrupted_image() {
     let config = ExtractionConfig::default();
 
@@ -112,27 +116,28 @@ async fn test_empty_file() {
 
     let empty_data = b"";
 
-    let result_pdf = extract_bytes(empty_data, "application/pdf", &config).await;
     let result_text = extract_bytes(empty_data, "text/plain", &config).await;
-    let result_xml = extract_bytes(empty_data, "application/xml", &config).await;
-
-    match result_pdf {
-        Ok(extraction) => {
-            assert!(
-                extraction.content.is_empty(),
-                "Empty PDF should have empty content if it succeeds"
-            );
-            assert!(extraction.chunks.is_none(), "Chunks should be None");
-        }
-        Err(error) => {
-            assert!(
-                matches!(
-                    error,
-                    kreuzberg::KreuzbergError::Parsing { .. } | kreuzberg::KreuzbergError::Validation { .. }
-                ),
-                "Empty PDF should produce Parsing or Validation error, got: {:?}",
-                error
-            );
+    #[cfg(feature = "pdf")]
+    {
+        let result_pdf = extract_bytes(empty_data, "application/pdf", &config).await;
+        match result_pdf {
+            Ok(extraction) => {
+                assert!(
+                    extraction.content.is_empty(),
+                    "Empty PDF should have empty content if it succeeds"
+                );
+                assert!(extraction.chunks.is_none(), "Chunks should be None");
+            }
+            Err(error) => {
+                assert!(
+                    matches!(
+                        error,
+                        kreuzberg::KreuzbergError::Parsing { .. } | kreuzberg::KreuzbergError::Validation { .. }
+                    ),
+                    "Empty PDF should produce Parsing or Validation error, got: {:?}",
+                    error
+                );
+            }
         }
     }
 
@@ -149,20 +154,24 @@ async fn test_empty_file() {
         }
     }
 
-    match result_xml {
-        Ok(extraction) => {
-            assert!(
-                extraction.content.is_empty(),
-                "Empty XML should have empty content if it succeeds"
-            );
-            assert!(extraction.chunks.is_none(), "Chunks should be None");
-        }
-        Err(error) => {
-            assert!(
-                matches!(error, kreuzberg::KreuzbergError::Parsing { .. }),
-                "Empty XML error should be Parsing type, got: {:?}",
-                error
-            );
+    #[cfg(feature = "xml")]
+    {
+        let result_xml = extract_bytes(empty_data, "application/xml", &config).await;
+        match result_xml {
+            Ok(extraction) => {
+                assert!(
+                    extraction.content.is_empty(),
+                    "Empty XML should have empty content if it succeeds"
+                );
+                assert!(extraction.chunks.is_none(), "Chunks should be None");
+            }
+            Err(error) => {
+                assert!(
+                    matches!(error, kreuzberg::KreuzbergError::Parsing { .. }),
+                    "Empty XML error should be Parsing type, got: {:?}",
+                    error
+                );
+            }
         }
     }
 }

data/vendor/kreuzberg/tests/format_integration.rs

@@ -5,6 +5,8 @@
 //! asynchronous APIs or to graceful handling when optional system
 //! dependencies are missing.
 
+#![cfg(any(feature = "pdf", feature = "office", feature = "ocr"))]
+
 mod helpers;
 
 use helpers::{assert_mime_type, assert_non_empty_content, get_test_file_path, test_documents_available};

data/vendor/kreuzberg/tests/image_integration.rs

@@ -11,6 +11,8 @@
 //! - Test OCR with various languages and layouts
 //! - Verify graceful handling of images without text
 
+#![cfg(feature = "ocr")]
+
 mod helpers;
 
 use helpers::*;

data/vendor/kreuzberg/tests/mime_detection.rs

@@ -276,22 +276,23 @@ async fn test_no_extension() {
 
     let detected = detect_mime_type(&temp_path, true);
 
-    if detected.is_err() {
-        let error = detected.unwrap_err();
-        assert!(
-            matches!(
-                error,
-                kreuzberg::KreuzbergError::Validation { .. } | kreuzberg::KreuzbergError::UnsupportedFormat(_)
-            ),
-            "Should return appropriate error for file without extension"
-        );
-    } else {
-        let mime = detected.unwrap();
-        assert!(
-            mime.contains('/'),
-            "Detected MIME type should be valid format: {}",
-            mime
-        );
+    match detected {
+        Err(error) => {
+            assert!(
+                matches!(
+                    error,
+                    kreuzberg::KreuzbergError::Validation { .. } | kreuzberg::KreuzbergError::UnsupportedFormat(_)
+                ),
+                "Should return appropriate error for file without extension"
+            );
+        }
+        Ok(mime) => {
+            assert!(
+                mime.contains('/'),
+                "Detected MIME type should be valid format: {}",
+                mime
+            );
+        }
     }
 
     let _ = std::fs::remove_file(&temp_path);

data/vendor/kreuzberg/tests/ocr_configuration.rs

@@ -11,6 +11,8 @@
 //! - Verify configuration changes actually affect output
 //! - Test table detection with various settings
 
+#![cfg(feature = "ocr")]
+
 mod helpers;
 
 use helpers::*;
@@ -204,6 +206,7 @@ fn test_ocr_psm_single_line() {
 }
 
 #[test]
+#[cfg(feature = "pdf")]
 fn test_force_ocr_on_text_pdf() {
     if skip_if_missing("pdfs/fake_memo.pdf") {
         return;
@@ -233,6 +236,7 @@ fn test_force_ocr_on_text_pdf() {
 }
 
 #[test]
+#[cfg(feature = "pdf")]
 fn test_force_ocr_disabled() {
     if skip_if_missing("pdfs/fake_memo.pdf") {
         return;

data/vendor/kreuzberg/tests/ocr_errors.rs

@@ -13,6 +13,8 @@
 //! - Test recovery from transient failures
 //! - Validate resource limits and constraints
 
+#![cfg(feature = "ocr")]
+
 mod helpers;
 
 use helpers::*;
@@ -451,6 +453,9 @@ fn test_ocr_cache_disabled_then_enabled() {
     };
 
     let result1 = extract_file_sync(&file_path, None, &config_no_cache);
+    if matches!(result1, Err(KreuzbergError::MissingDependency(_))) {
+        return;
+    }
     assert!(result1.is_ok(), "First extraction should succeed");
 
     let config_with_cache = ExtractionConfig {
@@ -468,6 +473,9 @@ fn test_ocr_cache_disabled_then_enabled() {
     };
 
     let result2 = extract_file_sync(&file_path, None, &config_with_cache);
+    if matches!(result2, Err(KreuzbergError::MissingDependency(_))) {
+        return;
+    }
     assert!(result2.is_ok(), "Second extraction should succeed");
 
     assert_non_empty_content(&result1.unwrap());
@@ -495,6 +503,13 @@ fn test_ocr_concurrent_same_file() {
         ..Default::default()
     });
 
+    if matches!(
+        extract_file_sync(&*file_path, None, &config),
+        Err(KreuzbergError::MissingDependency(_))
+    ) {
+        return;
+    }
+
     let mut handles = vec![];
     for i in 0..5 {
         let file_path_clone = Arc::clone(&file_path);
@@ -554,6 +569,13 @@ fn test_ocr_concurrent_different_files() {
         ..Default::default()
     });
 
+    if matches!(
+        extract_file_sync(&files[0], None, &config),
+        Err(KreuzbergError::MissingDependency(_))
+    ) {
+        return;
+    }
+
     let mut handles = vec![];
     for (i, file_path) in files.iter().enumerate() {
         let file_path_clone = file_path.clone();

data/vendor/kreuzberg/tests/ocr_quality.rs

@@ -14,6 +14,8 @@
 //! - Verify layout preservation (line counts, structure)
 //! - Assert minimum quality thresholds
 
+#![cfg(all(feature = "ocr", feature = "pdf"))]
+
 mod helpers;
 
 use helpers::*;

data/vendor/kreuzberg/tests/odt_extractor_tests.rs

@@ -69,16 +69,13 @@ async fn test_odt_metadata_extraction() {
         "Should contain document title in content"
     );
 
-    // Verify metadata extraction
     let metadata = &result.metadata.additional;
     println!("Extracted metadata: {:?}", metadata);
 
-    // Check title
     if let Some(title) = metadata.get("title") {
         assert_eq!(title.as_str(), Some("Test Metadata Document"), "Title should match");
     }
 
-    // Check subject
     if let Some(subject) = metadata.get("subject") {
         assert_eq!(
             subject.as_str(),
@@ -87,28 +84,23 @@ async fn test_odt_metadata_extraction() {
         );
     }
 
-    // Check creator/author
     if let Some(created_by) = metadata.get("created_by") {
         assert_eq!(created_by.as_str(), Some("John Doe"), "Creator should match");
     }
 
-    // Check authors array
     if let Some(authors) = metadata.get("authors") {
         let authors_array = authors.as_array().expect("Authors should be an array");
         assert_eq!(authors_array.len(), 1, "Should have one author");
         assert_eq!(authors_array[0].as_str(), Some("John Doe"), "Author name should match");
     }
 
-    // Check creation date (should exist)
     assert!(metadata.get("created_at").is_some(), "Creation date should be present");
 
-    // Check modification date (should exist)
     assert!(
         metadata.get("modified_at").is_some(),
         "Modification date should be present"
     );
 
-    // Check generator
     if let Some(generator) = metadata.get("generator") {
         let gen_str = generator.as_str().expect("Generator should be a string");
         assert!(gen_str.contains("Pandoc"), "Generator should be Pandoc");
@@ -604,18 +596,11 @@ async fn test_odt_table_no_duplicate_content() {
 
     assert!(!result.content.is_empty(), "Content should not be empty");
 
-    // Count how many times we see "Content" in the output
-    // In a properly fixed version, it should appear only once in the markdown table
-    // or possibly twice if headers appear with the same name, but not multiple times
-    // for the same cell
     let content_count = result.content.matches("Content").count();
 
-    // "Content" appears twice in the header "More content" in a simple table
-    // It should not appear more than 3 times (once in header, once in data cell, once in a different word like "More content")
     println!("   'Content' appears {} times in output", content_count);
     println!("   Content preview:\n{}", result.content);
 
-    // This verifies that we're not getting duplicate cell content extracted
     assert!(
         content_count <= 3,
         "Content should not appear excessively, indicating no duplicate table cell extraction"
@@ -628,7 +613,6 @@ async fn test_odt_table_no_duplicate_content() {
 /// Uses the extraction_test document created with pandoc to ensure complete content
 #[tokio::test]
 async fn test_odt_comprehensive_table_extraction() {
-    // This test uses the pandoc-generated test document
     let test_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
         .parent()
         .unwrap()
@@ -648,7 +632,6 @@ async fn test_odt_comprehensive_table_extraction() {
 
     assert!(!result.content.is_empty(), "Content should not be empty");
 
-    // Verify all sections are present
     assert!(result.content.contains("Comprehensive"), "Should contain heading");
     assert!(
         result.content.contains("First Section") || result.content.contains("First"),
@@ -663,14 +646,12 @@ async fn test_odt_comprehensive_table_extraction() {
         "Should contain third section"
     );
 
-    // Verify tables are present and formatted correctly (as markdown)
     assert!(
         result.content.contains("|"),
         "Should contain pipe characters for markdown tables"
     );
     assert!(result.content.contains("---"), "Should contain table separator");
 
-    // Verify table content is extracted
     assert!(
         result.content.contains("Header 1") || result.content.contains("Cell 1A"),
         "Should contain table data"
@@ -680,8 +661,6 @@ async fn test_odt_comprehensive_table_extraction() {
         "Should contain second table data"
     );
 
-    // Verify no excessive duplication of cells (a simple heuristic check)
-    // Count "Cell 1A" - should appear once or twice at most
     let cell_count = result.content.matches("Cell 1A").count();
     assert!(
         cell_count <= 2,

data/vendor/kreuzberg/tests/pdf_integration.rs

@@ -4,6 +4,8 @@
 //! multi-language E2E generator. This module keeps only the cases that
 //! exercise Rust-specific failure handling or error propagation.
 
+#![cfg(feature = "pdf")]
+
 mod helpers;
 
 use helpers::*;

data/vendor/kreuzberg/tests/pipeline_integration.rs

@@ -138,6 +138,7 @@ async fn test_pipeline_empty_no_processors() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -182,6 +183,7 @@ async fn test_pipeline_single_processor_per_stage() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -226,6 +228,7 @@ async fn test_pipeline_multiple_processors_per_stage() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -261,6 +264,7 @@ async fn test_pipeline_all_stages_enabled() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -294,6 +298,7 @@ async fn test_pipeline_postprocessing_disabled() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
@@ -340,6 +345,7 @@ async fn test_pipeline_early_stage_runs_first() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -379,6 +385,7 @@ async fn test_pipeline_middle_stage_runs_second() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -414,6 +421,7 @@ async fn test_pipeline_late_stage_runs_last() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -449,6 +457,7 @@ async fn test_pipeline_within_stage_priority_order() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -513,6 +522,7 @@ async fn test_pipeline_cross_stage_data_flow() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -569,6 +579,7 @@ async fn test_pipeline_early_stage_error_recorded() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -610,6 +621,7 @@ async fn test_pipeline_middle_stage_error_propagation() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -681,6 +693,7 @@ async fn test_pipeline_late_stage_error_doesnt_affect_earlier_stages() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -768,6 +781,7 @@ async fn test_pipeline_processor_error_doesnt_stop_other_processors() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -845,6 +859,7 @@ async fn test_pipeline_multiple_processor_errors() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -886,6 +901,7 @@ async fn test_pipeline_error_context_preservation() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -957,6 +973,7 @@ async fn test_pipeline_metadata_added_in_early_visible_in_middle() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1027,6 +1044,7 @@ async fn test_pipeline_content_modified_in_middle_visible_in_late() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1095,6 +1113,7 @@ async fn test_pipeline_multiple_processors_modifying_same_metadata() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1182,6 +1201,7 @@ async fn test_pipeline_processors_reading_previous_output() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1236,6 +1256,7 @@ async fn test_pipeline_large_content_modification() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1271,6 +1292,7 @@ async fn test_pipeline_enabled_processors_whitelist() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
@@ -1315,6 +1337,7 @@ async fn test_pipeline_disabled_processors_blacklist() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
@@ -1359,6 +1382,7 @@ async fn test_pipeline_no_filtering_runs_all() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1396,6 +1420,7 @@ async fn test_pipeline_empty_whitelist_runs_none() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {

data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs

@@ -3,6 +3,8 @@
 //! Tests custom OCR backend registration, execution, parameter passing,
 //! error handling, and backend switching with real image extraction.
 
+#![cfg(feature = "ocr")]
+
 use async_trait::async_trait;
 use kreuzberg::core::config::{ExtractionConfig, OcrConfig};
 use kreuzberg::plugins::registry::get_ocr_backend_registry;
@@ -60,6 +62,7 @@ impl OcrBackend for MockOcrBackend {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         })
     }
 
@@ -155,6 +158,7 @@ impl OcrBackend for ValidatingOcrBackend {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         })
     }
 
@@ -211,6 +215,7 @@ impl OcrBackend for MetadataOcrBackend {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         })
     }
 

data/vendor/kreuzberg/tests/plugin_system.rs

@@ -58,6 +58,7 @@ impl DocumentExtractor for FailingExtractor {
                 detected_languages: None,
                 chunks: None,
                 images: None,
+                pages: None,
             })
         }
     }
@@ -302,6 +303,7 @@ fn test_extractor_priority_ordering_complex() {
                 detected_languages: None,
                 chunks: None,
                 images: None,
+                pages: None,
             })
         }
         fn supported_mime_types(&self) -> &[&str] {
@@ -461,6 +463,7 @@ async fn test_processor_execution_order_within_stage() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
 
     let config = ExtractionConfig::default();
@@ -492,6 +495,7 @@ async fn test_processor_error_propagation() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
 
     let config = ExtractionConfig::default();
@@ -663,6 +667,7 @@ async fn test_validator_content_validation() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
 
     let validation = validators[0].validate(&short_result, &config).await;
@@ -676,6 +681,7 @@ async fn test_validator_content_validation() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
 
     let validation = validators[0].validate(&long_result, &config).await;

data/vendor/kreuzberg/tests/registry_integration_tests.rs

@@ -125,6 +125,7 @@ impl DocumentExtractor for MockExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         })
     }