kreuzberg 4.0.0.pre.rc.6 → 4.0.0.pre.rc.8

data/vendor/kreuzberg/tests/pdfium_linking.rs

@@ -0,0 +1,374 @@
+//! PDFium linking integration tests.
+//!
+//! This module tests PDF extraction functionality across all PDFium linking strategies
+//! (default, static, bundled, system). Tests verify that:
+//!
+//! 1. PDF extraction works regardless of linking strategy
+//! 2. Extracted content is valid and non-empty
+//! 3. Bundled PDFium extraction works when feature is enabled
+//! 4. Metadata extraction functions correctly
+//! 5. Error handling works for invalid PDFs
+//!
+//! These tests are strategy-agnostic - they verify functionality works with ANY
+//! linking strategy, not implementation details of specific strategies.
+//!
+//! Test philosophy:
+//! - Use small test PDFs from test_documents/
+//! - Assert on extracted content and behavior, not linking internals
+//! - Test both successful extraction and error cases
+//! - Skip tests gracefully if test files are missing
+
+#![cfg(feature = "pdf")]
+
+mod helpers;
+
+use helpers::*;
+use kreuzberg::core::config::ExtractionConfig;
+use kreuzberg::extract_file_sync;
+
+/// Test basic PDF extraction works regardless of linking strategy.
+///
+/// This is the primary integration test - it verifies that PDF extraction
+/// functions correctly with any linking strategy. We use a small test PDF
+/// and verify that:
+/// - Extraction completes successfully
+/// - Content is non-empty and contains expected text
+/// - MIME type is correctly detected
+#[test]
+fn test_pdf_extraction_basic() {
+    if skip_if_missing("pdfs_with_tables/tiny.pdf") {
+        return;
+    }
+
+    let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
+    let config = ExtractionConfig::default();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF successfully");
+
+    // Verify MIME type detection
+    assert_mime_type(&result, "application/pdf");
+
+    // Verify content was extracted
+    assert_non_empty_content(&result);
+
+    // Verify reasonable content length for a small PDF
+    assert_min_content_length(&result, 10);
+}
+
+/// Test PDF extraction with a medium-sized test PDF.
+///
+/// Verifies that extraction works with slightly larger PDFs and produces
+/// substantive content. This test helps ensure linking strategy works
+/// with real-world document sizes.
+#[test]
+fn test_pdf_extraction_medium() {
+    if skip_if_missing("pdfs_with_tables/medium.pdf") {
+        return;
+    }
+
+    let file_path = get_test_file_path("pdfs_with_tables/medium.pdf");
+    let config = ExtractionConfig::default();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract medium PDF successfully");
+
+    assert_mime_type(&result, "application/pdf");
+    assert_non_empty_content(&result);
+
+    // Medium PDF should have more content than tiny
+    assert!(result.content.len() >= 50, "Medium PDF should have substantial content");
+}
+
+/// Test PDF extraction with rotated pages.
+///
+/// Some PDFs have rotated pages. This test verifies that the linking
+/// strategy handles page rotation correctly (PDFium should handle this
+/// transparently). Note: This specific PDF may have OCR-only content,
+/// so we just verify extraction completes without error.
+#[test]
+fn test_pdf_extraction_rotated_pages() {
+    if skip_if_missing("pdfs/ocr_test_rotated_90.pdf") {
+        return;
+    }
+
+    let file_path = get_test_file_path("pdfs/ocr_test_rotated_90.pdf");
+    let config = ExtractionConfig::default();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract rotated PDF successfully");
+
+    assert_mime_type(&result, "application/pdf");
+    // Rotated PDFs may have OCR-only content, just verify extraction completes
+}
+
+/// Test PDF text extraction with code and formula content.
+///
+/// Verifies extraction works with technical PDFs containing code blocks,
+/// mathematical formulas, and special formatting.
+#[test]
+fn test_pdf_extraction_code_and_formulas() {
+    if skip_if_missing("pdfs/code_and_formula.pdf") {
+        return;
+    }
+
+    let file_path = get_test_file_path("pdfs/code_and_formula.pdf");
+    let config = ExtractionConfig::default();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with code and formulas");
+
+    assert_mime_type(&result, "application/pdf");
+    assert_non_empty_content(&result);
+}
+
+/// Test PDF extraction preserves right-to-left text direction.
+///
+/// PDFs with RTL text (Arabic, Hebrew, etc.) require special handling.
+/// This verifies extraction works with RTL content.
+#[test]
+fn test_pdf_extraction_right_to_left() {
+    if skip_if_missing("pdfs/right_to_left_01.pdf") {
+        return;
+    }
+
+    let file_path = get_test_file_path("pdfs/right_to_left_01.pdf");
+    let config = ExtractionConfig::default();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract RTL PDF successfully");
+
+    assert_mime_type(&result, "application/pdf");
+    // RTL PDFs might have minimal or no extracted text depending on PDF structure
+    // Just verify extraction completes without error
+}
+
+/// Test PDF metadata extraction.
+///
+/// Verifies that PDF metadata (title, author, creation date, etc.) can be
+/// extracted correctly, which is independent of linking strategy.
+#[test]
+fn test_pdf_metadata_extraction() {
+    if skip_if_missing("pdfs_with_tables/tiny.pdf") {
+        return;
+    }
+
+    let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
+    let config = ExtractionConfig::default();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF metadata");
+
+    // Metadata field is always present (may have empty fields for some PDFs)
+    // Just verify the structure exists
+    let _metadata = &result.metadata;
+    assert!(!result.mime_type.is_empty(), "MIME type should be set");
+}
+
+/// Test extraction of byte array from PDF.
+///
+/// Verifies that extract_bytes_sync works for PDF content, which is
+/// important for in-memory processing.
+#[test]
+fn test_pdf_extraction_from_bytes() {
+    if skip_if_missing("pdfs_with_tables/tiny.pdf") {
+        return;
+    }
+
+    let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
+    let pdf_bytes = std::fs::read(&file_path).expect("Should read PDF file");
+
+    let config = ExtractionConfig::default();
+    let result = kreuzberg::extract_bytes_sync(&pdf_bytes, "application/pdf", &config)
+        .expect("Should extract PDF from bytes successfully");
+
+    assert_mime_type(&result, "application/pdf");
+    assert_non_empty_content(&result);
+}
+
+/// Test bundled PDFium extraction when pdf-bundled feature is enabled.
+///
+/// When `pdf-bundled` feature is enabled, PDFium is embedded in the binary.
+/// This test verifies the bundled extraction mechanism works correctly.
+#[test]
+#[cfg(feature = "pdf-bundled")]
+fn test_bundled_pdfium_extraction() {
+    use kreuzberg::pdf::extract_bundled_pdfium;
+
+    // Extract bundled PDFium library to temporary directory
+    let lib_path = extract_bundled_pdfium().expect("Should extract bundled PDFium library");
+
+    // Verify extracted library exists and is a file
+    assert!(
+        lib_path.exists(),
+        "Extracted PDFium library should exist at {}",
+        lib_path.display()
+    );
+    assert!(
+        lib_path.is_file(),
+        "Extracted PDFium library should be a file, not directory"
+    );
+
+    // Verify library has a reasonable file size (at least a few MB)
+    let metadata = std::fs::metadata(&lib_path).expect("Should read library metadata");
+    assert!(
+        metadata.len() > 1_000_000,
+        "Bundled PDFium library should be at least 1MB (got {} bytes)",
+        metadata.len()
+    );
+}
+
+/// Test bundled PDFium extracts consistently.
+///
+/// Verify that repeated calls to extract_bundled_pdfium return the same path
+/// and don't create duplicate extracted libraries.
+#[test]
+#[cfg(feature = "pdf-bundled")]
+fn test_bundled_pdfium_caching() {
+    use kreuzberg::pdf::extract_bundled_pdfium;
+
+    let path1 = extract_bundled_pdfium().expect("First extraction should succeed");
+    let path2 = extract_bundled_pdfium().expect("Second extraction should succeed");
+
+    // Both calls should return the same path (caching in place)
+    assert_eq!(path1, path2, "Multiple extractions should return consistent path");
+
+    // Path should still exist
+    assert!(path1.exists(), "Extracted library should still exist");
+}
+
+/// Test bundled PDFium works with PDF extraction.
+///
+/// This integration test verifies that PDFium extracted via bundled mechanism
+/// actually works for PDF content extraction.
+#[test]
+#[cfg(feature = "pdf-bundled")]
+fn test_bundled_pdfium_with_pdf_extraction() {
+    if skip_if_missing("pdfs_with_tables/tiny.pdf") {
+        return;
+    }
+
+    // Ensure bundled PDFium is extracted and available
+    let _lib_path = kreuzberg::pdf::extract_bundled_pdfium().expect("Should extract bundled PDFium library");
+
+    // Now test that PDF extraction works with the bundled library
+    let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
+    let config = ExtractionConfig::default();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with bundled PDFium");
+
+    assert_mime_type(&result, "application/pdf");
+    assert_non_empty_content(&result);
+}
+
+/// Test extraction with custom PDF configuration.
+///
+/// Verifies that custom PDF extraction settings work correctly.
+#[test]
+fn test_pdf_extraction_with_config() {
+    if skip_if_missing("pdfs_with_tables/tiny.pdf") {
+        return;
+    }
+
+    let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
+
+    // Create custom extraction config
+    let config = ExtractionConfig::default();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with custom config");
+
+    assert_mime_type(&result, "application/pdf");
+    assert_non_empty_content(&result);
+
+    // Verify metadata is present
+    assert!(!result.mime_type.is_empty(), "MIME type should be set in result");
+}
+
+/// Test handling of empty or minimal PDFs.
+///
+/// Some edge case PDFs might be empty or contain no text. Verify extraction
+/// handles these gracefully.
+#[test]
+fn test_pdf_extraction_edge_cases() {
+    // Test extraction of a minimal PDF
+    if skip_if_missing("pdfs_with_tables/tiny.pdf") {
+        return;
+    }
+
+    let file_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
+    let config = ExtractionConfig::default();
+
+    // Should complete without panic, regardless of content
+    let result = extract_file_sync(&file_path, None, &config);
+    assert!(result.is_ok(), "PDF extraction should succeed");
+}
+
+/// Test batch extraction of multiple PDFs.
+///
+/// Verifies that linking strategy handles batch processing of multiple
+/// PDF files correctly.
+#[test]
+#[cfg(feature = "tokio-runtime")]
+fn test_pdf_batch_extraction() {
+    if skip_if_missing("pdfs_with_tables/tiny.pdf") || skip_if_missing("pdfs_with_tables/medium.pdf") {
+        return;
+    }
+
+    let paths = vec![
+        get_test_file_path("pdfs_with_tables/tiny.pdf"),
+        get_test_file_path("pdfs_with_tables/medium.pdf"),
+    ];
+
+    let config = ExtractionConfig::default();
+
+    // Use synchronous batch extraction
+    let results = kreuzberg::batch_extract_file_sync(paths, &config).expect("Should extract PDFs in batch");
+
+    // Verify all PDFs were extracted
+    assert_eq!(results.len(), 2, "Should extract both PDFs");
+
+    // Verify all results are valid
+    for result in &results {
+        assert_mime_type(result, "application/pdf");
+        assert_non_empty_content(result);
+    }
+}
+
+/// Test PDF with Unicode content extraction.
+///
+/// Verifies that PDFium linking strategy handles PDFs with international
+/// characters correctly.
+#[test]
+fn test_pdf_unicode_content() {
+    // Right-to-left PDFs often contain Unicode content
+    if skip_if_missing("pdfs/right_to_left_01.pdf") {
+        return;
+    }
+
+    let file_path = get_test_file_path("pdfs/right_to_left_01.pdf");
+    let config = ExtractionConfig::default();
+
+    let result = extract_file_sync(&file_path, None, &config).expect("Should extract PDF with Unicode content");
+
+    // Just verify extraction completes - Unicode content handling
+    // varies by PDF structure
+    assert_mime_type(&result, "application/pdf");
+}
+
+/// Verify PDF module compiles with all feature combinations.
+///
+/// This is a compile-time test that ensures the pdf module and bundled
+/// submodule (when enabled) compile correctly. It's implicitly tested
+/// by the fact that these test functions compile.
+#[test]
+#[cfg(feature = "pdf")]
+fn test_pdf_module_availability() {
+    // This test verifies that:
+    // 1. The pdf module is available (feature = "pdf")
+    // 2. Key functions are accessible
+    // 3. pdf-bundled module is available when feature enabled
+
+    // Verify we can access the text extraction function
+    let _ = kreuzberg::pdf::extract_text_from_pdf;
+
+    // Verify we can access the metadata function
+    let _ = kreuzberg::pdf::extract_metadata;
+
+    // Verify bundled module is available when feature enabled
+    #[cfg(feature = "pdf-bundled")]
+    let _ = kreuzberg::pdf::extract_bundled_pdfium;
+}

data/vendor/kreuzberg/tests/pipeline_integration.rs

@@ -138,6 +138,7 @@ async fn test_pipeline_empty_no_processors() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -182,6 +183,7 @@ async fn test_pipeline_single_processor_per_stage() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -226,6 +228,7 @@ async fn test_pipeline_multiple_processors_per_stage() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -261,6 +264,7 @@ async fn test_pipeline_all_stages_enabled() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -294,6 +298,7 @@ async fn test_pipeline_postprocessing_disabled() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
@@ -340,6 +345,7 @@ async fn test_pipeline_early_stage_runs_first() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -379,6 +385,7 @@ async fn test_pipeline_middle_stage_runs_second() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -414,6 +421,7 @@ async fn test_pipeline_late_stage_runs_last() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -449,6 +457,7 @@ async fn test_pipeline_within_stage_priority_order() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -513,6 +522,7 @@ async fn test_pipeline_cross_stage_data_flow() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -569,6 +579,7 @@ async fn test_pipeline_early_stage_error_recorded() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -610,6 +621,7 @@ async fn test_pipeline_middle_stage_error_propagation() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -681,6 +693,7 @@ async fn test_pipeline_late_stage_error_doesnt_affect_earlier_stages() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -768,6 +781,7 @@ async fn test_pipeline_processor_error_doesnt_stop_other_processors() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -845,6 +859,7 @@ async fn test_pipeline_multiple_processor_errors() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -886,6 +901,7 @@ async fn test_pipeline_error_context_preservation() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -957,6 +973,7 @@ async fn test_pipeline_metadata_added_in_early_visible_in_middle() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1027,6 +1044,7 @@ async fn test_pipeline_content_modified_in_middle_visible_in_late() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1095,6 +1113,7 @@ async fn test_pipeline_multiple_processors_modifying_same_metadata() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1182,6 +1201,7 @@ async fn test_pipeline_processors_reading_previous_output() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1236,6 +1256,7 @@ async fn test_pipeline_large_content_modification() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1271,6 +1292,7 @@ async fn test_pipeline_enabled_processors_whitelist() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
@@ -1315,6 +1337,7 @@ async fn test_pipeline_disabled_processors_blacklist() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {
@@ -1359,6 +1382,7 @@ async fn test_pipeline_no_filtering_runs_all() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig::default();
 
@@ -1396,6 +1420,7 @@ async fn test_pipeline_empty_whitelist_runs_none() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
     let config = ExtractionConfig {
         postprocessor: Some(PostProcessorConfig {

data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs

@@ -3,6 +3,8 @@
 //! Tests custom OCR backend registration, execution, parameter passing,
 //! error handling, and backend switching with real image extraction.
 
+#![cfg(feature = "ocr")]
+
 use async_trait::async_trait;
 use kreuzberg::core::config::{ExtractionConfig, OcrConfig};
 use kreuzberg::plugins::registry::get_ocr_backend_registry;
@@ -60,6 +62,7 @@ impl OcrBackend for MockOcrBackend {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         })
     }
 
@@ -155,6 +158,7 @@ impl OcrBackend for ValidatingOcrBackend {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         })
     }
 
@@ -211,6 +215,7 @@ impl OcrBackend for MetadataOcrBackend {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         })
     }
 

data/vendor/kreuzberg/tests/plugin_system.rs

@@ -58,6 +58,7 @@ impl DocumentExtractor for FailingExtractor {
                 detected_languages: None,
                 chunks: None,
                 images: None,
+                pages: None,
             })
         }
     }
@@ -302,6 +303,7 @@ fn test_extractor_priority_ordering_complex() {
                 detected_languages: None,
                 chunks: None,
                 images: None,
+                pages: None,
             })
         }
         fn supported_mime_types(&self) -> &[&str] {
@@ -461,6 +463,7 @@ async fn test_processor_execution_order_within_stage() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
 
     let config = ExtractionConfig::default();
@@ -492,6 +495,7 @@ async fn test_processor_error_propagation() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
 
     let config = ExtractionConfig::default();
@@ -663,6 +667,7 @@ async fn test_validator_content_validation() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
 
     let validation = validators[0].validate(&short_result, &config).await;
@@ -676,6 +681,7 @@ async fn test_validator_content_validation() {
         detected_languages: None,
         chunks: None,
         images: None,
+        pages: None,
     };
 
     let validation = validators[0].validate(&long_result, &config).await;

data/vendor/kreuzberg/tests/registry_integration_tests.rs

@@ -125,6 +125,7 @@ impl DocumentExtractor for MockExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            pages: None,
         })
     }
 

data/vendor/kreuzberg/tests/rst_extractor_tests.rs

@@ -8,6 +8,8 @@
 //! - Section structure and heading levels
 //! - Table extraction (simple and grid tables)
 //! - Reference links and images
+
+#![cfg(feature = "office")]
 //! - Comments and special blocks
 //! - Content quality validation
 

data/vendor/kreuzberg/tests/rtf_extractor_tests.rs

@@ -731,7 +731,6 @@ async fn test_rtf_comprehensive_extraction_alignment() {
         "Should include third section heading"
     );
 
-    // Table/text alignment with DOCX/ODT variants
     for expected in ["Header 1", "Cell 1A", "Product", "Apple"] {
         assert!(
             rtf_result.content.contains(expected),

data/vendor/kreuzberg/tests/security_validation.rs

@@ -140,6 +140,7 @@ fn test_archive_deeply_nested_directories() {
 }
 
 #[test]
+#[cfg(feature = "archives")]
 fn test_archive_many_small_files() {
     let mut cursor = std::io::Cursor::new(Vec::new());
     {