kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/tests/batch_orchestration.rs

@@ -19,18 +19,6 @@ use kreuzberg::core::extractor::extract_file_sync;
 
 mod helpers;
 
-fn trim_trailing_newlines(value: &str) -> &str {
-    value.trim_end_matches(['\n', '\r'])
-}
-
-fn assert_text_content(actual: &str, expected: &str) {
-    assert_eq!(
-        trim_trailing_newlines(actual),
-        expected,
-        "Content mismatch after trimming trailing newlines"
-    );
-}
-
 /// Test that batch extraction processes documents in parallel.
 ///
 /// Validates:
@@ -307,13 +295,9 @@ async fn test_batch_bytes_parallel_processing() {
         .collect();
 
     let contents_ref: Vec<(&[u8], &str)> = contents.iter().map(|(bytes, mime)| (bytes.as_slice(), *mime)).collect();
-    let owned_contents: Vec<(Vec<u8>, String)> = contents_ref
-        .into_iter()
-        .map(|(bytes, mime)| (bytes.to_vec(), mime.to_string()))
-        .collect();
 
     let start = Instant::now();
-    let results = batch_extract_bytes(owned_contents, &config).await;
+    let results = batch_extract_bytes(contents_ref, &config).await;
     let duration = start.elapsed();
 
     assert!(results.is_ok());
@@ -321,8 +305,7 @@ async fn test_batch_bytes_parallel_processing() {
     assert_eq!(results.len(), 30);
 
     for (i, result) in results.iter().enumerate() {
-        let expected = format!("Test content number {}", i);
-        assert_text_content(&result.content, &expected);
+        assert_eq!(result.content, format!("Test content number {}", i));
     }
 
     println!("Batch processed 30 byte arrays in {:?}", duration);
@@ -341,20 +324,15 @@ async fn test_batch_bytes_mixed_valid_invalid() {
         (b"valid content 3".as_slice(), "text/plain"),
     ];
 
-    let owned_contents: Vec<(Vec<u8>, String)> = contents
-        .into_iter()
-        .map(|(bytes, mime)| (bytes.to_vec(), mime.to_string()))
-        .collect();
-
-    let results = batch_extract_bytes(owned_contents, &config).await;
+    let results = batch_extract_bytes(contents, &config).await;
 
     assert!(results.is_ok());
     let results = results.unwrap();
     assert_eq!(results.len(), 5);
 
-    assert_text_content(&results[0].content, "valid content 1");
-    assert_text_content(&results[2].content, "valid content 2");
-    assert_text_content(&results[4].content, "valid content 3");
+    assert_eq!(results[0].content, "valid content 1");
+    assert_eq!(results[2].content, "valid content 2");
+    assert_eq!(results[4].content, "valid content 3");
 
     assert!(results[1].metadata.error.is_some());
     assert!(results[3].metadata.error.is_some());
@@ -383,13 +361,9 @@ async fn test_batch_utilizes_multiple_cores() {
     }
 
     let contents_ref: Vec<(&[u8], &str)> = contents.iter().map(|(bytes, mime)| (bytes.as_slice(), *mime)).collect();
-    let owned_contents: Vec<(Vec<u8>, String)> = contents_ref
-        .into_iter()
-        .map(|(bytes, mime)| (bytes.to_vec(), mime.to_string()))
-        .collect();
 
     let start = Instant::now();
-    let results = batch_extract_bytes(owned_contents, &config).await;
+    let results = batch_extract_bytes(contents_ref, &config).await;
     let duration = start.elapsed();
 
     assert!(results.is_ok());
@@ -426,13 +400,9 @@ async fn test_batch_memory_pressure_handling() {
     }
 
     let contents_ref: Vec<(&[u8], &str)> = contents.iter().map(|(bytes, mime)| (bytes.as_slice(), *mime)).collect();
-    let owned_contents: Vec<(Vec<u8>, String)> = contents_ref
-        .into_iter()
-        .map(|(bytes, mime)| (bytes.to_vec(), mime.to_string()))
-        .collect();
 
     let start = Instant::now();
-    let results = batch_extract_bytes(owned_contents, &config).await;
+    let results = batch_extract_bytes(contents_ref, &config).await;
     let duration = start.elapsed();
 
     assert!(results.is_ok());
@@ -462,13 +432,8 @@ async fn test_batch_scales_with_cpu_count() {
 
     let contents_ref: Vec<(&[u8], &str)> = contents.iter().map(|(bytes, mime)| (bytes.as_slice(), *mime)).collect();
 
-    let owned_contents_1: Vec<(Vec<u8>, String)> = contents_ref
-        .iter()
-        .map(|(bytes, mime)| (bytes.to_vec(), mime.to_string()))
-        .collect();
-
     let start = Instant::now();
-    let _ = batch_extract_bytes(owned_contents_1, &config_1).await.unwrap();
+    let _ = batch_extract_bytes(contents_ref.clone(), &config_1).await.unwrap();
     let duration_1 = start.elapsed();
 
     let config_full = ExtractionConfig {
@@ -476,13 +441,8 @@ async fn test_batch_scales_with_cpu_count() {
         ..Default::default()
     };
 
-    let owned_contents_full: Vec<(Vec<u8>, String)> = contents_ref
-        .into_iter()
-        .map(|(bytes, mime)| (bytes.to_vec(), mime.to_string()))
-        .collect();
-
     let start = Instant::now();
-    let _ = batch_extract_bytes(owned_contents_full, &config_full).await.unwrap();
+    let _ = batch_extract_bytes(contents_ref, &config_full).await.unwrap();
     let duration_full = start.elapsed();
 
     println!(
@@ -566,20 +526,15 @@ async fn test_batch_accuracy_under_load() {
     }
 
     let contents_ref: Vec<(&[u8], &str)> = contents.iter().map(|(bytes, mime)| (bytes.as_slice(), *mime)).collect();
-    let owned_contents: Vec<(Vec<u8>, String)> = contents_ref
-        .into_iter()
-        .map(|(bytes, mime)| (bytes.to_vec(), mime.to_string()))
-        .collect();
 
-    let results = batch_extract_bytes(owned_contents, &config).await.unwrap();
+    let results = batch_extract_bytes(contents_ref, &config).await.unwrap();
 
     assert_eq!(results.len(), 100);
 
     for (i, result) in results.iter().enumerate() {
         let expected = format!("Document number {} with unique content", i);
         assert_eq!(
-            trim_trailing_newlines(&result.content),
-            expected,
+            result.content, expected,
             "Document {} content mismatch - possible cross-contamination",
             i
         );

data/vendor/kreuzberg/tests/batch_processing.rs

@@ -4,29 +4,16 @@
 //! Validates concurrent processing, error handling, and performance.
 
 use kreuzberg::core::config::ExtractionConfig;
-#[cfg(feature = "pdf")]
-use kreuzberg::core::extractor::batch_extract_file_sync;
-use kreuzberg::core::extractor::{batch_extract_bytes, batch_extract_bytes_sync, batch_extract_file};
+use kreuzberg::core::extractor::{
+    batch_extract_bytes, batch_extract_bytes_sync, batch_extract_file, batch_extract_file_sync,
+};
 use std::path::PathBuf;
 
 mod helpers;
 use helpers::{get_test_documents_dir, get_test_file_path, skip_if_missing, test_documents_available};
 
-fn trim_trailing_newlines(value: &str) -> &str {
-    value.trim_end_matches(['\n', '\r'])
-}
-
-fn assert_text_content(actual: &str, expected: &str) {
-    assert_eq!(
-        trim_trailing_newlines(actual),
-        expected,
-        "Content mismatch after trimming trailing newlines"
-    );
-}
-
 /// Test batch extraction with multiple file formats (PDF, DOCX, TXT).
 #[tokio::test]
-#[cfg(all(feature = "pdf", feature = "office", feature = "tokio-runtime"))]
 async fn test_batch_extract_file_multiple_formats() {
     if !test_documents_available() {
         println!("Skipping test: test_documents/ directory not found");
@@ -74,7 +61,6 @@ async fn test_batch_extract_file_multiple_formats() {
 
 /// Test synchronous batch extraction variant.
 #[test]
-#[cfg(feature = "pdf")]
 fn test_batch_extract_file_sync_variant() {
     if !test_documents_available() {
         println!("Skipping test: test_documents/ directory not found");
@@ -129,19 +115,14 @@ async fn test_batch_extract_bytes_multiple() {
         (json_bytes.as_slice(), "application/json"),
     ];
 
-    let owned_contents: Vec<(Vec<u8>, String)> = contents
-        .into_iter()
-        .map(|(bytes, mime)| (bytes.to_vec(), mime.to_string()))
-        .collect();
-
-    let results = batch_extract_bytes(owned_contents, &config).await;
+    let results = batch_extract_bytes(contents, &config).await;
 
     assert!(results.is_ok(), "Batch bytes extraction should succeed");
     let results = results.unwrap();
 
     assert_eq!(results.len(), 3);
 
-    assert_text_content(&results[0].content, "This is plain text content");
+    assert_eq!(results[0].content, "This is plain text content");
     assert_eq!(results[0].mime_type, "text/plain");
 
     assert!(results[1].content.contains("Markdown Header"));
@@ -311,18 +292,13 @@ fn test_batch_extract_bytes_sync_variant() {
         (b"# content 3".as_slice(), "text/markdown"),
     ];
 
-    let owned_contents: Vec<(Vec<u8>, String)> = contents
-        .into_iter()
-        .map(|(bytes, mime)| (bytes.to_vec(), mime.to_string()))
-        .collect();
-
-    let results = batch_extract_bytes_sync(owned_contents, &config);
+    let results = batch_extract_bytes_sync(contents, &config);
 
     assert!(results.is_ok(), "Sync batch bytes extraction should succeed");
     let results = results.unwrap();
 
     assert_eq!(results.len(), 3);
-    assert_text_content(&results[0].content, "content 1");
-    assert_text_content(&results[1].content, "content 2");
+    assert_eq!(results[0].content, "content 1");
+    assert_eq!(results[1].content, "content 2");
     assert!(results[2].content.contains("content 3"));
 }

data/vendor/kreuzberg/tests/chunking_offset_demo.rs

@@ -0,0 +1,92 @@
+#[cfg(feature = "chunking")]
+#[test]
+fn demonstrate_correct_offset_calculation() {
+    use kreuzberg::chunking::{ChunkerType, ChunkingConfig, chunk_text};
+
+    println!("\n=== Demonstrating Correct Chunking Offset Calculation ===\n");
+
+    let config_with_overlap = ChunkingConfig {
+        max_characters: 20,
+        overlap: 5,
+        trim: false,
+        chunker_type: ChunkerType::Text,
+    };
+
+    let text = "AAAAA BBBBB CCCCC DDDDD EEEEE FFFFF";
+    println!("Text: \"{}\"", text);
+    println!(
+        "Max characters: {}, Overlap: {}\n",
+        config_with_overlap.max_characters, config_with_overlap.overlap
+    );
+
+    let result = chunk_text(text, &config_with_overlap).unwrap();
+
+    println!("WITH OVERLAP (5 chars):");
+    for (i, chunk) in result.chunks.iter().enumerate() {
+        println!(
+            "  Chunk {}: [{:3} - {:3}] = \"{}\"",
+            i,
+            chunk.metadata.char_start,
+            chunk.metadata.char_end,
+            chunk.content.replace('\n', "\\n")
+        );
+    }
+
+    println!("\nOverlap verification:");
+    for i in 0..result.chunks.len() - 1 {
+        let current = &result.chunks[i];
+        let next = &result.chunks[i + 1];
+        let overlap_size = current.metadata.char_end - next.metadata.char_start;
+        println!(
+            "  Chunks {} and {}: overlap = {} chars (next starts at {} while current ends at {})",
+            i,
+            i + 1,
+            overlap_size,
+            next.metadata.char_start,
+            current.metadata.char_end
+        );
+        assert!(
+            overlap_size > 0 && overlap_size <= config_with_overlap.overlap + 10,
+            "Overlap should exist and be reasonable"
+        );
+    }
+
+    println!("\n\n=== Without Overlap ===\n");
+    let config_no_overlap = ChunkingConfig {
+        max_characters: 20,
+        overlap: 0,
+        trim: false,
+        chunker_type: ChunkerType::Text,
+    };
+
+    let result_no_overlap = chunk_text(text, &config_no_overlap).unwrap();
+
+    println!("WITHOUT OVERLAP:");
+    for (i, chunk) in result_no_overlap.chunks.iter().enumerate() {
+        println!(
+            "  Chunk {}: [{:3} - {:3}] = \"{}\"",
+            i,
+            chunk.metadata.char_start,
+            chunk.metadata.char_end,
+            chunk.content.replace('\n', "\\n")
+        );
+    }
+
+    println!("\nAdjacency verification:");
+    for i in 0..result_no_overlap.chunks.len() - 1 {
+        let current = &result_no_overlap.chunks[i];
+        let next = &result_no_overlap.chunks[i + 1];
+        let gap = next.metadata.char_start as i32 - current.metadata.char_end as i32;
+        println!(
+            "  Chunks {} and {}: gap = {} (next starts at {}, current ends at {})",
+            i,
+            i + 1,
+            gap,
+            next.metadata.char_start,
+            current.metadata.char_end
+        );
+        assert!(gap >= 0, "Should have no overlap (gap >= 0)");
+    }
+
+    println!("\n✓ All offset calculations are correct!");
+}

data/vendor/kreuzberg/tests/concurrency_stress.rs

@@ -18,6 +18,7 @@ use kreuzberg::plugins::registry::{get_document_extractor_registry, get_post_pro
 use kreuzberg::plugins::{Plugin, PostProcessor, ProcessingStage};
 use kreuzberg::types::{ExtractionResult, Metadata};
 use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
 
 #[cfg(feature = "ocr")]
 use kreuzberg::core::config::OcrConfig;
@@ -29,18 +30,6 @@ use tokio::time::timeout;
 
 mod helpers;
 
-fn trim_trailing_newlines(value: &str) -> &str {
-    value.trim_end_matches(['\n', '\r'])
-}
-
-fn assert_text_content(actual: &str, expected: &str) {
-    assert_eq!(
-        trim_trailing_newlines(actual),
-        expected,
-        "Content mismatch after trimming trailing newlines"
-    );
-}
-
 /// Test many concurrent extractions of different MIME types.
 ///
 /// Validates that:
@@ -51,16 +40,13 @@ fn assert_text_content(actual: &str, expected: &str) {
 async fn test_concurrent_extractions_mixed_formats() {
     let config = ExtractionConfig::default();
 
-    #[allow(unused_mut)]
-    let mut test_cases = vec![
+    let test_cases = vec![
         (b"Plain text content" as &[u8], "text/plain"),
         (b"{\"key\": \"value\"}", "application/json"),
+        (b"<root><item>XML content</item></root>", "application/xml"),
         (b"# Markdown\n\nContent here", "text/markdown"),
     ];
 
-    #[cfg(feature = "xml")]
-    test_cases.push((b"<root><item>XML content</item></root>" as &[u8], "application/xml"));
-
     let mut handles = vec![];
     for _ in 0..10 {
         for (data, mime_type) in &test_cases {
@@ -109,11 +95,7 @@ async fn test_concurrent_batch_extractions() {
 
         handles.push(tokio::spawn(async move {
             let data: Vec<(&[u8], &str)> = contents_clone.iter().map(|c| (c.as_slice(), "text/plain")).collect();
-            let owned_data: Vec<(Vec<u8>, String)> = data
-                .into_iter()
-                .map(|(bytes, mime)| (bytes.to_vec(), mime.to_string()))
-                .collect();
-            batch_extract_bytes(owned_data, &config).await
+            batch_extract_bytes(data, &config).await
         }));
     }
 
@@ -139,8 +121,6 @@ async fn test_concurrent_extractions_with_cache() {
             enabled: false,
             enabled_processors: None,
             disabled_processors: None,
-            enabled_set: None,
-            disabled_set: None,
         }),
         ..Default::default()
     };
@@ -164,7 +144,7 @@ async fn test_concurrent_extractions_with_cache() {
         let result = handle.await.expect("Task should not panic");
         assert!(result.is_ok(), "Cache read should succeed");
         let extraction = result.unwrap();
-        assert_text_content(&extraction.content, expected_content);
+        assert_eq!(extraction.content, expected_content);
     }
 }
 
@@ -179,10 +159,6 @@ async fn test_concurrent_extractions_with_cache() {
 async fn test_concurrent_ocr_processing() {
     use helpers::{get_test_file_path, skip_if_missing};
 
-    if cfg!(windows) {
-        return;
-    }
-
     if skip_if_missing("images/ocr_image.jpg") {
         tracing::debug!("Skipping concurrent OCR test: test file not available");
         return;
@@ -250,7 +226,6 @@ async fn test_concurrent_ocr_processing() {
 #[test]
 fn test_concurrent_ocr_cache_stress() {
     use helpers::{get_test_file_path, skip_if_missing};
-    use std::sync::atomic::{AtomicUsize, Ordering};
 
     if skip_if_missing("images/ocr_image.jpg") {
         tracing::debug!("Skipping OCR cache stress test: test file not available");
@@ -358,8 +333,6 @@ async fn test_concurrent_pipeline_processing() {
             enabled: true,
             enabled_processors: Some(vec!["concurrent-test".to_string()]),
             disabled_processors: None,
-            enabled_set: None,
-            disabled_set: None,
         }),
         ..Default::default()
     };
@@ -377,7 +350,6 @@ async fn test_concurrent_pipeline_processing() {
                 detected_languages: None,
                 chunks: None,
                 images: None,
-                pages: None,
             };
 
             run_pipeline(result, &config).await
@@ -498,16 +470,13 @@ async fn test_high_concurrency_stress() {
         ..Default::default()
     };
 
-    #[allow(unused_mut)]
-    let mut formats = vec![
+    let formats = vec![
         (b"Text content" as &[u8], "text/plain"),
         (b"{\"json\": true}", "application/json"),
+        (b"<xml><item>content</item></xml>", "application/xml"),
         (b"# Markdown\n\nContent", "text/markdown"),
     ];
 
-    #[cfg(feature = "xml")]
-    formats.push((b"<xml><item>content</item></xml>" as &[u8], "application/xml"));
-
     let mut handles = vec![];
     for _ in 0..100 {
         for (data, mime_type) in &formats {
@@ -531,10 +500,9 @@ async fn test_high_concurrency_stress() {
     .await
     .expect("High-load stress test should complete within 60s");
 
-    let expected_successes = 100 * formats.len();
     let success_count = results.iter().filter(|r| r.is_ok()).count();
     assert_eq!(
-        success_count, expected_successes,
+        success_count, 400,
         "All extractions should succeed under stress, got {} successes",
         success_count
     );

data/vendor/kreuzberg/tests/config_features.rs

@@ -3,19 +3,13 @@
 //! Tests for chunking, language detection, caching, token reduction, and quality processing.
 //! Validates that configuration options work correctly end-to-end.
 
-#[cfg(feature = "chunking")]
-use kreuzberg::core::config::ChunkingConfig;
-use kreuzberg::core::config::ExtractionConfig;
-#[cfg(feature = "language-detection")]
-use kreuzberg::core::config::LanguageDetectionConfig;
-use kreuzberg::core::config::TokenReductionConfig;
+use kreuzberg::core::config::{ChunkingConfig, ExtractionConfig, LanguageDetectionConfig, TokenReductionConfig};
 use kreuzberg::core::extractor::extract_bytes;
 
 mod helpers;
 
 /// Test chunking enabled - text split into chunks.
 #[tokio::test]
-#[cfg(feature = "chunking")]
 async fn test_chunking_enabled() {
     let config = ExtractionConfig {
         chunking: Some(ChunkingConfig {
@@ -58,7 +52,6 @@ async fn test_chunking_enabled() {
 
 /// Test chunking with overlap - overlap preserved between chunks.
 #[tokio::test]
-#[cfg(feature = "chunking")]
 async fn test_chunking_with_overlap() {
     let config = ExtractionConfig {
         chunking: Some(ChunkingConfig {
@@ -98,7 +91,6 @@ async fn test_chunking_with_overlap() {
 
 /// Test chunking with custom sizes - custom chunk size and overlap.
 #[tokio::test]
-#[cfg(feature = "chunking")]
 async fn test_chunking_custom_sizes() {
     let config = ExtractionConfig {
         chunking: Some(ChunkingConfig {
@@ -159,7 +151,6 @@ async fn test_chunking_disabled() {
 
 /// Test language detection for single language document.
 #[tokio::test]
-#[cfg(feature = "language-detection")]
 async fn test_language_detection_single() {
     let config = ExtractionConfig {
         language_detection: Some(LanguageDetectionConfig {
@@ -186,7 +177,6 @@ async fn test_language_detection_single() {
 /// Test language detection for multi-language document.
 #[cfg_attr(coverage, ignore = "coverage instrumentation affects multi-language heuristics")]
 #[tokio::test]
-#[cfg(feature = "language-detection")]
 async fn test_language_detection_multiple() {
     let config = ExtractionConfig {
         language_detection: Some(LanguageDetectionConfig {
@@ -211,7 +201,6 @@ async fn test_language_detection_multiple() {
 
 /// Test language detection with confidence threshold.
 #[tokio::test]
-#[cfg(feature = "language-detection")]
 async fn test_language_detection_confidence() {
     let config = ExtractionConfig {
         language_detection: Some(LanguageDetectionConfig {
@@ -236,7 +225,6 @@ async fn test_language_detection_confidence() {
 
 /// Test language detection disabled.
 #[tokio::test]
-#[cfg(feature = "language-detection")]
 async fn test_language_detection_disabled() {
     let config = ExtractionConfig {
         language_detection: Some(LanguageDetectionConfig {
@@ -409,7 +397,6 @@ async fn test_token_reduction_disabled() {
 
 /// Test quality processing enabled - quality scoring applied.
 #[tokio::test]
-#[cfg(feature = "quality")]
 async fn test_quality_processing_enabled() {
     let config = ExtractionConfig {
         enable_quality_processing: true,
@@ -433,7 +420,6 @@ async fn test_quality_processing_enabled() {
 
 /// Test quality processing calculates score for different text quality.
 #[tokio::test]
-#[cfg(feature = "quality")]
 async fn test_quality_threshold_filtering() {
     let config = ExtractionConfig {
         enable_quality_processing: true,
@@ -498,15 +484,8 @@ async fn test_quality_processing_disabled() {
 }
 
 /// Test chunking with embeddings using balanced preset.
-///
-/// This test requires ONNX Runtime to be installed as a system dependency.
-/// On macOS with Homebrew: `brew install onnxruntime`
-/// On Linux: Install via your package manager or download from https://github.com/microsoft/onnxruntime/releases
-/// On Windows: Download from https://github.com/microsoft/onnxruntime/releases
 #[tokio::test]
 #[cfg(feature = "embeddings")]
-#[cfg_attr(target_os = "macos", ignore = "ONNX models not cached on macOS")]
-#[cfg_attr(target_os = "windows", ignore = "ONNX models not cached on Windows")]
 async fn test_chunking_with_embeddings() {
     use kreuzberg::core::config::EmbeddingConfig;
 
@@ -564,15 +543,8 @@ async fn test_chunking_with_embeddings() {
 }
 
 /// Test chunking with fast embedding preset.
-///
-/// This test requires ONNX Runtime to be installed as a system dependency.
-/// On macOS with Homebrew: `brew install onnxruntime`
-/// On Linux: Install via your package manager or download from https://github.com/microsoft/onnxruntime/releases
-/// On Windows: Download from https://github.com/microsoft/onnxruntime/releases
 #[tokio::test]
 #[cfg(feature = "embeddings")]
-#[cfg_attr(target_os = "macos", ignore = "ONNX models not cached on macOS")]
-#[cfg_attr(target_os = "windows", ignore = "ONNX models not cached on Windows")]
 async fn test_chunking_with_fast_embeddings() {
     use kreuzberg::core::config::{EmbeddingConfig, EmbeddingModelType};
 
@@ -601,10 +573,6 @@ async fn test_chunking_with_fast_embeddings() {
     let chunks = result.chunks.expect("Should have chunks");
     assert!(!chunks.is_empty(), "Should have at least one chunk");
 
-    if let Some(error) = result.metadata.additional.get("embedding_error") {
-        panic!("Embedding generation failed: {}", error);
-    }
-
     for chunk in &chunks {
         let embedding = chunk.embedding.as_ref().expect("Should have embedding");
         assert_eq!(embedding.len(), 384, "Fast preset should produce 384-dim embeddings");