kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/tests/pipeline_integration.rs

@@ -7,7 +7,7 @@
 
 use async_trait::async_trait;
 use kreuzberg::core::config::{ExtractionConfig, PostProcessorConfig};
-use kreuzberg::core::pipeline::{clear_processor_cache, run_pipeline};
+use kreuzberg::core::pipeline::run_pipeline;
 use kreuzberg::plugins::registry::get_post_processor_registry;
 use kreuzberg::plugins::{Plugin, PostProcessor, ProcessingStage};
 use kreuzberg::types::{ExtractionResult, Metadata};
@@ -123,8 +123,6 @@ fn clear_processor_registry() {
         .write()
         .expect("Failed to acquire write lock on registry in test");
     let _ = reg.shutdown_all();
-    drop(reg);
-    let _ = clear_processor_cache();
 }
 
 #[tokio::test]
@@ -140,7 +138,6 @@ async fn test_pipeline_empty_no_processors() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -185,7 +182,6 @@ async fn test_pipeline_single_processor_per_stage() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -230,7 +226,6 @@ async fn test_pipeline_multiple_processors_per_stage() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -266,7 +261,6 @@ async fn test_pipeline_all_stages_enabled() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -300,15 +294,12 @@ async fn test_pipeline_postprocessing_disabled() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
             enabled: false,
             enabled_processors: None,
             disabled_processors: None,
-            enabled_set: None,
-            disabled_set: None,
         }),
         ..Default::default()
     };
@@ -349,7 +340,6 @@ async fn test_pipeline_early_stage_runs_first() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -389,7 +379,6 @@ async fn test_pipeline_middle_stage_runs_second() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -425,7 +414,6 @@ async fn test_pipeline_late_stage_runs_last() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -461,7 +449,6 @@ async fn test_pipeline_within_stage_priority_order() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -526,7 +513,6 @@ async fn test_pipeline_cross_stage_data_flow() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -583,7 +569,6 @@ async fn test_pipeline_early_stage_error_recorded() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -625,7 +610,6 @@ async fn test_pipeline_middle_stage_error_propagation() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -697,7 +681,6 @@ async fn test_pipeline_late_stage_error_doesnt_affect_earlier_stages() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -785,7 +768,6 @@ async fn test_pipeline_processor_error_doesnt_stop_other_processors() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -863,12 +845,12 @@ async fn test_pipeline_multiple_processor_errors() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
     let result = run_pipeline(result, &config).await;
     assert!(result.is_err(), "Expected pipeline to return error");
+    // First failing processor (fail1 in Early stage) will cause pipeline to fail
     match result {
         Err(KreuzbergError::Plugin { message, plugin_name }) => {
             assert_eq!(message, "fail1 error");
@@ -905,7 +887,6 @@ async fn test_pipeline_error_context_preservation() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -977,7 +958,6 @@ async fn test_pipeline_metadata_added_in_early_visible_in_middle() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1048,7 +1028,6 @@ async fn test_pipeline_content_modified_in_middle_visible_in_late() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1117,7 +1096,6 @@ async fn test_pipeline_multiple_processors_modifying_same_metadata() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1205,7 +1183,6 @@ async fn test_pipeline_processors_reading_previous_output() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1260,7 +1237,6 @@ async fn test_pipeline_large_content_modification() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1296,15 +1272,12 @@ async fn test_pipeline_enabled_processors_whitelist() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
             enabled: true,
             enabled_processors: Some(vec!["proc1".to_string(), "proc3".to_string()]),
             disabled_processors: None,
-            enabled_set: None,
-            disabled_set: None,
         }),
         ..Default::default()
     };
@@ -1343,15 +1316,12 @@ async fn test_pipeline_disabled_processors_blacklist() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
             enabled: true,
             enabled_processors: None,
             disabled_processors: Some(vec!["proc2".to_string()]),
-            enabled_set: None,
-            disabled_set: None,
         }),
         ..Default::default()
     };
@@ -1390,7 +1360,6 @@ async fn test_pipeline_no_filtering_runs_all() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1428,15 +1397,12 @@ async fn test_pipeline_empty_whitelist_runs_none() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
             enabled: true,
             enabled_processors: Some(vec![]),
             disabled_processors: None,
-            enabled_set: None,
-            disabled_set: None,
         }),
         ..Default::default()
     };

data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs

@@ -3,8 +3,6 @@
 //! Tests custom OCR backend registration, execution, parameter passing,
 //! error handling, and backend switching with real image extraction.
 
-#![cfg(feature = "ocr")]
-
 use async_trait::async_trait;
 use kreuzberg::core::config::{ExtractionConfig, OcrConfig};
 use kreuzberg::plugins::registry::get_ocr_backend_registry;
@@ -62,7 +60,6 @@ impl OcrBackend for MockOcrBackend {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         })
     }
 
@@ -158,7 +155,6 @@ impl OcrBackend for ValidatingOcrBackend {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         })
     }
 
@@ -215,7 +211,6 @@ impl OcrBackend for MetadataOcrBackend {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         })
     }
 

data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs

@@ -5,7 +5,6 @@
 
 use async_trait::async_trait;
 use kreuzberg::core::config::ExtractionConfig;
-use kreuzberg::core::pipeline::clear_processor_cache;
 use kreuzberg::plugins::registry::get_post_processor_registry;
 use kreuzberg::plugins::{Plugin, PostProcessor, ProcessingStage};
 use kreuzberg::types::ExtractionResult;
@@ -165,16 +164,6 @@ impl PostProcessor for FailingProcessor {
     }
 }
 
-fn clear_processor_registry_and_cache() {
-    let registry = get_post_processor_registry();
-    let mut reg = registry
-        .write()
-        .expect("Failed to acquire write lock on registry in test");
-    let _ = reg.shutdown_all();
-    drop(reg);
-    let _ = clear_processor_cache();
-}
-
 #[serial]
 #[test]
 fn test_register_custom_postprocessor() {
@@ -213,7 +202,6 @@ fn test_register_custom_postprocessor() {
 #[serial]
 #[test]
 fn test_postprocessor_called_during_extraction() {
-    clear_processor_registry_and_cache();
     let test_file = "../../test_documents/text/fake_text.txt";
     let registry = get_post_processor_registry();
 
@@ -256,7 +244,6 @@ fn test_postprocessor_called_during_extraction() {
 #[serial]
 #[test]
 fn test_postprocessor_modifies_content() {
-    clear_processor_registry_and_cache();
     let test_file = "../../test_documents/text/fake_text.txt";
     let registry = get_post_processor_registry();
 
@@ -288,7 +275,6 @@ fn test_postprocessor_modifies_content() {
 #[serial]
 #[test]
 fn test_postprocessor_adds_metadata() {
-    clear_processor_registry_and_cache();
     let test_file = "../../test_documents/text/fake_text.txt";
     let registry = get_post_processor_registry();
 
@@ -431,7 +417,6 @@ fn test_clear_all_postprocessors() {
 #[serial]
 #[test]
 fn test_postprocessor_error_handling() {
-    clear_processor_registry_and_cache();
     let test_file = "../../test_documents/text/fake_text.txt";
     let registry = get_post_processor_registry();
 
@@ -448,6 +433,7 @@ fn test_postprocessor_error_handling() {
     let result = extract_file_sync(test_file, None, &config);
 
     // NOTE: Plugin errors now bubble up and fail the extraction (design change)
+    // Other error types (non-IO, non-Plugin) are caught and recorded in metadata
     assert!(
         result.is_err(),
         "Extraction should fail when postprocessor returns Plugin error"
@@ -500,7 +486,6 @@ fn test_postprocessor_invalid_name() {
 #[serial]
 #[test]
 fn test_multiple_postprocessors_execution_order() {
-    clear_processor_registry_and_cache();
     let test_file = "../../test_documents/text/fake_text.txt";
     let registry = get_post_processor_registry();
 
@@ -546,7 +531,6 @@ fn test_multiple_postprocessors_execution_order() {
 #[serial]
 #[test]
 fn test_postprocessor_preserves_mime_type() {
-    clear_processor_registry_and_cache();
     let test_file = "../../test_documents/text/fake_text.txt";
     let registry = get_post_processor_registry();
 

data/vendor/kreuzberg/tests/plugin_system.rs

@@ -58,7 +58,6 @@ impl DocumentExtractor for FailingExtractor {
                 detected_languages: None,
                 chunks: None,
                 images: None,
-                pages: None,
             })
         }
     }
@@ -303,7 +302,6 @@ fn test_extractor_priority_ordering_complex() {
                 detected_languages: None,
                 chunks: None,
                 images: None,
-                pages: None,
             })
         }
         fn supported_mime_types(&self) -> &[&str] {
@@ -463,7 +461,6 @@ async fn test_processor_execution_order_within_stage() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
 
     let config = ExtractionConfig::default();
@@ -495,7 +492,6 @@ async fn test_processor_error_propagation() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
 
     let config = ExtractionConfig::default();
@@ -667,7 +663,6 @@ async fn test_validator_content_validation() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
 
     let validation = validators[0].validate(&short_result, &config).await;
@@ -681,7 +676,6 @@ async fn test_validator_content_validation() {
         detected_languages: None,
         chunks: None,
         images: None,
-        pages: None,
     };
 
     let validation = validators[0].validate(&long_result, &config).await;

data/vendor/kreuzberg/tests/registry_integration_tests.rs

@@ -15,6 +15,8 @@ use kreuzberg::{KreuzbergError, Result};
 use std::path::Path;
 use std::sync::Arc;
 
+// ===== Mock Validators =====
+
 struct MockValidator {
     name: String,
     should_fail: bool,
@@ -85,6 +87,8 @@ impl Validator for FailingInitValidator {
     }
 }
 
+// ===== Mock Extractors =====
+
 struct MockExtractor {
     name: String,
     mime_types: Vec<&'static str>,
@@ -125,7 +129,6 @@ impl DocumentExtractor for MockExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
-            pages: None,
         })
     }
 
@@ -143,6 +146,8 @@ impl DocumentExtractor for MockExtractor {
     }
 }
 
+// ===== Validator Registry Tests =====
+
 /// Test validator registration and listing.
 #[test]
 fn test_validator_registration_succeeds() {
@@ -275,10 +280,13 @@ fn test_validator_registration_with_failed_init_fails() {
     assert!(result.is_err(), "Registration with failed init should fail");
 
     match result {
-        Err(KreuzbergError::Plugin { .. }) => {}
+        Err(KreuzbergError::Plugin { .. }) => {
+            // Expected error type
+        }
         _ => panic!("Expected Plugin error"),
     }
 
+    // Validator should not be in the list
     assert_eq!(registry.list().len(), 0, "Failed validator should not be registered");
 }
 
@@ -287,6 +295,7 @@ fn test_validator_registration_with_failed_init_fails() {
 fn test_clear_validators_succeeds() {
     let mut registry = ValidatorRegistry::new();
 
+    // Register multiple validators
     let v1 = Arc::new(MockValidator {
         name: "validator-1".to_string(),
         should_fail: false,
@@ -300,6 +309,7 @@ fn test_clear_validators_succeeds() {
     registry.register(v2).unwrap();
     assert_eq!(registry.list().len(), 2);
 
+    // Clear all
     let result = registry.shutdown_all();
     assert!(result.is_ok(), "Clear should succeed");
     assert_eq!(registry.list().len(), 0, "Registry should be empty after clear");
@@ -360,11 +370,14 @@ fn test_get_all_validators_respects_priority() {
     let all = registry.get_all();
     assert_eq!(all.len(), 3, "Should have three validators");
 
+    // Should be in descending priority order
     assert_eq!(all[0].name(), "high-priority");
     assert_eq!(all[1].name(), "medium-priority");
     assert_eq!(all[2].name(), "low-priority");
 }
 
+// ===== Extractor Registry Tests =====
+
 /// Test extractor registration and retrieval.
 #[test]
 fn test_extractor_registration_succeeds() {
@@ -438,6 +451,7 @@ fn test_extractor_priority_selection() {
     registry.register(low_priority).unwrap();
     registry.register(high_priority).unwrap();
 
+    // Should get the high priority extractor
     let result = registry.get("text/plain").unwrap();
     assert_eq!(
         result.name(),
@@ -459,14 +473,17 @@ fn test_extractor_wildcard_mime_matching() {
 
     registry.register(extractor).unwrap();
 
+    // Should match text/plain
     let result = registry.get("text/plain");
     assert!(result.is_ok(), "Should match text/plain with text/*");
     assert_eq!(result.unwrap().name(), "text-extractor");
 
+    // Should match text/html
     let result = registry.get("text/html");
     assert!(result.is_ok(), "Should match text/html with text/*");
     assert_eq!(result.unwrap().name(), "text-extractor");
 
+    // Should not match application/pdf
     let result = registry.get("application/pdf");
     assert!(result.is_err(), "Should not match application/pdf with text/*");
 }
@@ -489,6 +506,7 @@ fn test_extractor_unregistration_succeeds() {
     assert!(result.is_ok(), "Unregistration should succeed");
     assert_eq!(registry.list().len(), 0, "Registry should be empty after removal");
 
+    // Should no longer find extractor for MIME type
     let lookup_result = registry.get("text/plain");
     assert!(lookup_result.is_err(), "Should not find extractor after removal");
 }
@@ -506,10 +524,12 @@ fn test_extractor_multiple_mime_types() {
 
     registry.register(extractor).unwrap();
 
+    // Should find for all MIME types
     assert!(registry.get("application/pdf").is_ok());
     assert!(registry.get("application/vnd.ms-excel").is_ok());
     assert!(registry.get("text/csv").is_ok());
 
+    // All should return the same extractor
     assert_eq!(
         registry.get("application/pdf").unwrap().name(),
         "multi-format-extractor"

data/vendor/kreuzberg/tests/security_validation.rs

@@ -12,17 +12,6 @@ use kreuzberg::core::extractor::{extract_bytes_sync, extract_file_sync};
 use std::io::Write;
 use tempfile::NamedTempFile;
 
-fn trim_trailing_newlines(value: &str) -> &str {
-    value.trim_end_matches(['\n', '\r'])
-}
-
-fn assert_text_content(actual: &str, expected: &str) {
-    assert_eq!(
-        trim_trailing_newlines(actual),
-        expected,
-        "Content mismatch after trimming trailing newlines"
-    );
-}
 #[test]
 fn test_archive_zip_bomb_detection() {
     let mut cursor = std::io::Cursor::new(Vec::new());
@@ -140,7 +129,6 @@ fn test_archive_deeply_nested_directories() {
 }
 
 #[test]
-#[cfg(feature = "archives")]
 fn test_archive_many_small_files() {
     let mut cursor = std::io::Cursor::new(Vec::new());
     {
@@ -278,7 +266,7 @@ fn test_resource_single_byte_file() {
 
     assert!(result.is_ok());
     if let Ok(extracted) = result {
-        assert_text_content(&extracted.content, "a");
+        assert_eq!(extracted.content, "a");
     }
 }