kreuzberg 4.0.0.rc1

data/vendor/kreuzberg/tests/batch_processing.rs

@@ -0,0 +1,304 @@
+//! Batch processing integration tests.
+//!
+//! Tests for `batch_extract_file` and `batch_extract_bytes` functions.
+//! Validates concurrent processing, error handling, and performance.
+
+use kreuzberg::core::config::ExtractionConfig;
+use kreuzberg::core::extractor::{
+    batch_extract_bytes, batch_extract_bytes_sync, batch_extract_file, batch_extract_file_sync,
+};
+use std::path::PathBuf;
+
+mod helpers;
+use helpers::{get_test_documents_dir, get_test_file_path, skip_if_missing, test_documents_available};
+
+/// Test batch extraction with multiple file formats (PDF, DOCX, TXT).
+#[tokio::test]
+async fn test_batch_extract_file_multiple_formats() {
+    if !test_documents_available() {
+        println!("Skipping test: test_documents/ directory not found");
+        return;
+    }
+
+    if skip_if_missing("pdfs/fake_memo.pdf")
+        || skip_if_missing("documents/fake.docx")
+        || skip_if_missing("text/fake_text.txt")
+    {
+        return;
+    }
+
+    let config = ExtractionConfig::default();
+
+    let paths = vec![
+        get_test_file_path("pdfs/fake_memo.pdf"),
+        get_test_file_path("documents/fake.docx"),
+        get_test_file_path("text/fake_text.txt"),
+    ];
+
+    let results = batch_extract_file(paths, &config).await;
+
+    assert!(results.is_ok(), "Batch extraction should succeed");
+    let results = results.unwrap();
+
+    assert_eq!(results.len(), 3);
+
+    assert!(!results[0].content.is_empty(), "PDF content should not be empty");
+    assert_eq!(results[0].mime_type, "application/pdf");
+
+    assert!(!results[1].content.is_empty(), "DOCX content should not be empty");
+    assert_eq!(
+        results[1].mime_type,
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+    );
+
+    assert!(!results[2].content.is_empty(), "TXT content should not be empty");
+    assert_eq!(results[2].mime_type, "text/plain");
+
+    assert!(results[0].metadata.error.is_none());
+    assert!(results[1].metadata.error.is_none());
+    assert!(results[2].metadata.error.is_none());
+}
+
+/// Test synchronous batch extraction variant.
+#[test]
+fn test_batch_extract_file_sync_variant() {
+    if !test_documents_available() {
+        println!("Skipping test: test_documents/ directory not found");
+        return;
+    }
+
+    if skip_if_missing("pdfs/fake_memo.pdf") || skip_if_missing("text/fake_text.txt") {
+        return;
+    }
+
+    let config = ExtractionConfig::default();
+
+    let paths = vec![
+        get_test_file_path("pdfs/fake_memo.pdf"),
+        get_test_file_path("text/fake_text.txt"),
+    ];
+
+    let results = batch_extract_file_sync(paths, &config);
+
+    assert!(results.is_ok(), "Sync batch extraction should succeed");
+    let results = results.unwrap();
+
+    assert_eq!(results.len(), 2);
+
+    assert!(!results[0].content.is_empty(), "PDF content should not be empty");
+    assert_eq!(
+        results[0].mime_type, "application/pdf",
+        "PDF MIME type should be correct"
+    );
+    assert!(results[0].metadata.error.is_none(), "PDF should extract without errors");
+
+    assert!(!results[1].content.is_empty(), "Text content should not be empty");
+    assert_eq!(results[1].mime_type, "text/plain", "Text MIME type should be correct");
+    assert!(
+        results[1].metadata.error.is_none(),
+        "Text should extract without errors"
+    );
+}
+
+/// Test batch extraction from bytes.
+#[tokio::test]
+async fn test_batch_extract_bytes_multiple() {
+    let config = ExtractionConfig::default();
+
+    let text_bytes = b"This is plain text content";
+    let markdown_bytes = b"# Markdown Header\n\nThis is markdown content";
+    let json_bytes = b"{\"key\": \"value\", \"number\": 42}";
+
+    let contents = vec![
+        (text_bytes.as_slice(), "text/plain"),
+        (markdown_bytes.as_slice(), "text/markdown"),
+        (json_bytes.as_slice(), "application/json"),
+    ];
+
+    let results = batch_extract_bytes(contents, &config).await;
+
+    assert!(results.is_ok(), "Batch bytes extraction should succeed");
+    let results = results.unwrap();
+
+    assert_eq!(results.len(), 3);
+
+    assert_eq!(results[0].content, "This is plain text content");
+    assert_eq!(results[0].mime_type, "text/plain");
+
+    assert!(results[1].content.contains("Markdown Header"));
+    assert_eq!(results[1].mime_type, "text/markdown");
+
+    assert!(results[2].content.contains("key"));
+    assert!(results[2].content.contains("value"));
+    assert_eq!(results[2].mime_type, "application/json");
+}
+
+/// Test batch extraction with empty file list.
+#[tokio::test]
+async fn test_batch_extract_empty_list() {
+    let config = ExtractionConfig::default();
+
+    let paths: Vec<PathBuf> = vec![];
+    let results = batch_extract_file(paths, &config).await;
+
+    assert!(results.is_ok(), "Empty batch should succeed");
+    assert_eq!(results.unwrap().len(), 0, "Should return empty vector");
+}
+
+/// Test batch extraction when one file fails (others should succeed).
+#[tokio::test]
+async fn test_batch_extract_one_file_fails() {
+    if !test_documents_available() {
+        println!("Skipping test: test_documents/ directory not found");
+        return;
+    }
+
+    if skip_if_missing("text/fake_text.txt") {
+        return;
+    }
+
+    let config = ExtractionConfig::default();
+
+    let paths = vec![
+        get_test_file_path("text/fake_text.txt"),
+        get_test_documents_dir().join("nonexistent_file.txt"),
+        get_test_file_path("text/contract.txt"),
+    ];
+
+    let results = batch_extract_file(paths, &config).await;
+
+    assert!(results.is_ok(), "Batch should succeed even with one failure");
+    let results = results.unwrap();
+
+    assert_eq!(results.len(), 3);
+
+    assert!(!results[0].content.is_empty());
+    assert!(results[0].metadata.error.is_none());
+
+    assert!(results[1].metadata.error.is_some());
+    assert!(results[1].content.contains("Error:"));
+
+    assert!(!results[2].content.is_empty());
+    assert!(results[2].metadata.error.is_none());
+}
+
+/// Test batch extraction when all files fail.
+#[tokio::test]
+async fn test_batch_extract_all_fail() {
+    let config = ExtractionConfig::default();
+
+    let test_dir = get_test_documents_dir();
+    let paths = vec![
+        test_dir.join("nonexistent1.txt"),
+        test_dir.join("nonexistent2.pdf"),
+        test_dir.join("nonexistent3.docx"),
+    ];
+
+    let results = batch_extract_file(paths, &config).await;
+
+    assert!(results.is_ok(), "Batch should succeed (errors in metadata)");
+    let results = results.unwrap();
+
+    assert_eq!(results.len(), 3);
+
+    assert!(results[0].metadata.error.is_some());
+    assert!(results[1].metadata.error.is_some());
+    assert!(results[2].metadata.error.is_some());
+
+    assert!(results[0].content.contains("Error:"));
+    assert!(results[1].content.contains("Error:"));
+    assert!(results[2].content.contains("Error:"));
+}
+
+/// Test concurrent batch processing (verify parallelism).
+#[tokio::test]
+async fn test_batch_extract_concurrent() {
+    if !test_documents_available() {
+        println!("Skipping test: test_documents/ directory not found");
+        return;
+    }
+
+    if skip_if_missing("text/fake_text.txt") {
+        return;
+    }
+
+    let config = ExtractionConfig::default();
+
+    let base_path = get_test_file_path("text/fake_text.txt");
+    let paths: Vec<PathBuf> = (0..20).map(|_| base_path.clone()).collect();
+
+    let start = std::time::Instant::now();
+    let results = batch_extract_file(paths, &config).await;
+    let duration = start.elapsed();
+
+    assert!(results.is_ok(), "Concurrent batch should succeed");
+    let results = results.unwrap();
+
+    assert_eq!(results.len(), 20);
+
+    for result in &results {
+        assert!(result.metadata.error.is_none(), "Result should not have errors");
+        assert!(!result.content.is_empty(), "Result content should not be empty");
+        assert_eq!(result.mime_type, "text/plain", "MIME type should be text/plain");
+    }
+
+    assert!(
+        !results[0].content.is_empty(),
+        "Should have extracted actual text content"
+    );
+
+    assert!(duration.as_secs() < 5, "Batch processing took too long: {:?}", duration);
+}
+
+/// Test large batch (50+ files).
+#[tokio::test]
+async fn test_batch_extract_large_batch() {
+    if !test_documents_available() {
+        println!("Skipping test: test_documents/ directory not found");
+        return;
+    }
+
+    if skip_if_missing("text/fake_text.txt") {
+        return;
+    }
+
+    let config = ExtractionConfig::default();
+
+    let base_path = get_test_file_path("text/fake_text.txt");
+    let paths: Vec<PathBuf> = (0..50).map(|_| base_path.clone()).collect();
+
+    let results = batch_extract_file(paths, &config).await;
+
+    assert!(results.is_ok(), "Large batch should succeed");
+    let results = results.unwrap();
+
+    assert_eq!(results.len(), 50);
+
+    for result in &results {
+        assert!(result.metadata.error.is_none());
+        assert!(!result.content.is_empty());
+        assert_eq!(result.mime_type, "text/plain");
+    }
+}
+
+/// Test sync variant with bytes.
+#[test]
+fn test_batch_extract_bytes_sync_variant() {
+    let config = ExtractionConfig::default();
+
+    let contents = vec![
+        (b"content 1".as_slice(), "text/plain"),
+        (b"content 2".as_slice(), "text/plain"),
+        (b"# content 3".as_slice(), "text/markdown"),
+    ];
+
+    let results = batch_extract_bytes_sync(contents, &config);
+
+    assert!(results.is_ok(), "Sync batch bytes extraction should succeed");
+    let results = results.unwrap();
+
+    assert_eq!(results.len(), 3);
+    assert_eq!(results[0].content, "content 1");
+    assert_eq!(results[1].content, "content 2");
+    assert!(results[2].content.contains("content 3"));
+}

data/vendor/kreuzberg/tests/chunking_offset_demo.rs

@@ -0,0 +1,92 @@
+#[cfg(feature = "chunking")]
+#[test]
+fn demonstrate_correct_offset_calculation() {
+    use kreuzberg::chunking::{ChunkerType, ChunkingConfig, chunk_text};
+
+    println!("\n=== Demonstrating Correct Chunking Offset Calculation ===\n");
+
+    let config_with_overlap = ChunkingConfig {
+        max_characters: 20,
+        overlap: 5,
+        trim: false,
+        chunker_type: ChunkerType::Text,
+    };
+
+    let text = "AAAAA BBBBB CCCCC DDDDD EEEEE FFFFF";
+    println!("Text: \"{}\"", text);
+    println!(
+        "Max characters: {}, Overlap: {}\n",
+        config_with_overlap.max_characters, config_with_overlap.overlap
+    );
+
+    let result = chunk_text(text, &config_with_overlap).unwrap();
+
+    println!("WITH OVERLAP (5 chars):");
+    for (i, chunk) in result.chunks.iter().enumerate() {
+        println!(
+            "  Chunk {}: [{:3} - {:3}] = \"{}\"",
+            i,
+            chunk.metadata.char_start,
+            chunk.metadata.char_end,
+            chunk.content.replace('\n', "\\n")
+        );
+    }
+
+    println!("\nOverlap verification:");
+    for i in 0..result.chunks.len() - 1 {
+        let current = &result.chunks[i];
+        let next = &result.chunks[i + 1];
+        let overlap_size = current.metadata.char_end - next.metadata.char_start;
+        println!(
+            "  Chunks {} and {}: overlap = {} chars (next starts at {} while current ends at {})",
+            i,
+            i + 1,
+            overlap_size,
+            next.metadata.char_start,
+            current.metadata.char_end
+        );
+        assert!(
+            overlap_size > 0 && overlap_size <= config_with_overlap.overlap + 10,
+            "Overlap should exist and be reasonable"
+        );
+    }
+
+    println!("\n\n=== Without Overlap ===\n");
+    let config_no_overlap = ChunkingConfig {
+        max_characters: 20,
+        overlap: 0,
+        trim: false,
+        chunker_type: ChunkerType::Text,
+    };
+
+    let result_no_overlap = chunk_text(text, &config_no_overlap).unwrap();
+
+    println!("WITHOUT OVERLAP:");
+    for (i, chunk) in result_no_overlap.chunks.iter().enumerate() {
+        println!(
+            "  Chunk {}: [{:3} - {:3}] = \"{}\"",
+            i,
+            chunk.metadata.char_start,
+            chunk.metadata.char_end,
+            chunk.content.replace('\n', "\\n")
+        );
+    }
+
+    println!("\nAdjacency verification:");
+    for i in 0..result_no_overlap.chunks.len() - 1 {
+        let current = &result_no_overlap.chunks[i];
+        let next = &result_no_overlap.chunks[i + 1];
+        let gap = next.metadata.char_start as i32 - current.metadata.char_end as i32;
+        println!(
+            "  Chunks {} and {}: gap = {} (next starts at {}, current ends at {})",
+            i,
+            i + 1,
+            gap,
+            next.metadata.char_start,
+            current.metadata.char_end
+        );
+        assert!(gap >= 0, "Should have no overlap (gap >= 0)");
+    }
+
+    println!("\n✓ All offset calculations are correct!");
+}