kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs

@@ -1,370 +0,0 @@
-//! Detailed comparison test between Kreuzberg and Pandoc DOCX extraction
-
-#![cfg(feature = "office")]
-
-use kreuzberg::core::config::ExtractionConfig;
-use kreuzberg::extractors::DocxExtractor;
-use kreuzberg::plugins::DocumentExtractor;
-
-#[tokio::test]
-async fn test_docx_kreuzberg_vs_pandoc_comparison() {
-    let docx_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
-        .parent()
-        .unwrap()
-        .parent()
-        .unwrap()
-        .join("test_documents/documents/word_sample.docx");
-
-    if !docx_path.exists() {
-        println!("Skipping test: Test file not found at {:?}", docx_path);
-        return;
-    }
-
-    let content = std::fs::read(&docx_path).expect("Failed to read DOCX");
-
-    let extractor = DocxExtractor::new();
-    let config = ExtractionConfig::default();
-
-    let kreuzberg_result = extractor
-        .extract_bytes(
-            &content,
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-            &config,
-        )
-        .await
-        .expect("Kreuzberg extraction failed");
-
-    println!("\n");
-    println!("╔════════════════════════════════════════════════════════════════╗");
-    println!("║         KREUZBERG vs PANDOC - DOCX EXTRACTION COMPARISON       ║");
-    println!("╚════════════════════════════════════════════════════════════════╝");
-    println!();
-
-    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
-    println!("DOCUMENT INFORMATION");
-    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
-    println!("File: word_sample.docx");
-    println!("Format: Microsoft Word 2007+ (.docx)");
-    println!("Size: 102 KB");
-    println!("Content Type: application/vnd.openxmlformats-officedocument.wordprocessingml.document");
-    println!();
-
-    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
-    println!("KREUZBERG EXTRACTION RESULTS");
-    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
-
-    let kreuzberg_lines = kreuzberg_result.content.lines().count();
-    let kreuzberg_words = kreuzberg_result.content.split_whitespace().count();
-    let kreuzberg_chars = kreuzberg_result.content.len();
-
-    println!("Text Metrics:");
-    println!("  Lines: {}", kreuzberg_lines);
-    println!("  Words: {}", kreuzberg_words);
-    println!("  Characters: {}", kreuzberg_chars);
-    println!();
-
-    println!("Content Preview (first 1500 characters):");
-    println!("─────────────────────────────────────────────────────────────────");
-    let preview = if kreuzberg_result.content.len() > 1500 {
-        &kreuzberg_result.content[..1500]
-    } else {
-        &kreuzberg_result.content
-    };
-    println!("{}", preview);
-    println!("─────────────────────────────────────────────────────────────────");
-    println!();
-
-    println!(
-        "Metadata Fields Extracted: {}",
-        kreuzberg_result.metadata.additional.len()
-    );
-    println!(
-        "  - created_by: {}",
-        kreuzberg_result
-            .metadata
-            .additional
-            .get("created_by")
-            .map(|v| v.to_string())
-            .unwrap_or_default()
-    );
-    println!(
-        "  - modified_by: {}",
-        kreuzberg_result
-            .metadata
-            .additional
-            .get("modified_by")
-            .map(|v| v.to_string())
-            .unwrap_or_default()
-    );
-    println!(
-        "  - created_at: {}",
-        kreuzberg_result
-            .metadata
-            .additional
-            .get("created_at")
-            .map(|v| v.to_string())
-            .unwrap_or_default()
-    );
-    println!(
-        "  - modified_at: {}",
-        kreuzberg_result
-            .metadata
-            .additional
-            .get("modified_at")
-            .map(|v| v.to_string())
-            .unwrap_or_default()
-    );
-    println!(
-        "  - page_count: {}",
-        kreuzberg_result
-            .metadata
-            .additional
-            .get("page_count")
-            .map(|v| v.to_string())
-            .unwrap_or_default()
-    );
-    println!(
-        "  - word_count: {}",
-        kreuzberg_result
-            .metadata
-            .additional
-            .get("word_count")
-            .map(|v| v.to_string())
-            .unwrap_or_default()
-    );
-    println!(
-        "  - character_count: {}",
-        kreuzberg_result
-            .metadata
-            .additional
-            .get("character_count")
-            .map(|v| v.to_string())
-            .unwrap_or_default()
-    );
-    println!(
-        "  - line_count: {}",
-        kreuzberg_result
-            .metadata
-            .additional
-            .get("line_count")
-            .map(|v| v.to_string())
-            .unwrap_or_default()
-    );
-    println!(
-        "  - paragraph_count: {}",
-        kreuzberg_result
-            .metadata
-            .additional
-            .get("paragraph_count")
-            .map(|v| v.to_string())
-            .unwrap_or_default()
-    );
-    println!(
-        "  - application: {}",
-        kreuzberg_result
-            .metadata
-            .additional
-            .get("application")
-            .map(|v| v.to_string())
-            .unwrap_or_default()
-    );
-    println!();
-
-    println!("Tables:");
-    println!("  Count: {}", kreuzberg_result.tables.len());
-    for (idx, table) in kreuzberg_result.tables.iter().enumerate() {
-        println!("  Table {} (Page {}):", idx + 1, table.page_number);
-        println!("    Rows: {}", table.cells.len());
-        if !table.cells.is_empty() {
-            println!("    Columns: {}", table.cells[0].len());
-        }
-        println!("    Markdown:");
-        for line in table.markdown.lines() {
-            println!("      {}", line);
-        }
-    }
-    println!();
-
-    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
-    println!("PANDOC EXTRACTION RESULTS (for comparison)");
-    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
-
-    println!("Pandoc Text Output Metrics:");
-    println!("  Lines: 52");
-    println!("  Words: 135");
-    println!("  Characters: 1152");
-    println!();
-
-    println!("Pandoc Content Preview (first 1500 characters):");
-    println!("─────────────────────────────────────────────────────────────────");
-    let pandoc_preview = "[A cartoon duck holding a paper Description automatically generated]
-
-Let's swim!
-
-To get started with swimming, first lay down in a water and try not to
-drown:
-
-- You can relax and look around
-
-- Paddle about
-
-- Enjoy summer warmth
-
-  Also, don't forget:
-
-1.  Wear sunglasses
-
-2.  Don't forget to drink water
-
-3.  Use sun cream
-
-    Hmm, what else…
-
-Let's eat
-
-After we had a good day of swimming in the lake, it's important to eat
-something nice
-
-  I like to eat leaves
-
-Here are some interesting things a respectful duck could eat:
-
-  -------";
-    println!("{}", pandoc_preview);
-    println!("─────────────────────────────────────────────────────────────────");
-    println!();
-
-    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
-    println!("COMPARATIVE ANALYSIS");
-    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
-    println!();
-
-    println!("1. CONTENT EXTRACTION");
-    println!(
-        "   ├─ Kreuzberg extracts: {} lines, {} words, {} chars",
-        kreuzberg_lines, kreuzberg_words, kreuzberg_chars
-    );
-    println!("   ├─ Pandoc extracts:   52 lines, 135 words, 1152 chars");
-    println!("   └─ Assessment: Kreuzberg extracts MORE content (includes image alt text, structure)");
-    println!();
-
-    println!("2. METADATA HANDLING");
-    println!(
-        "   ├─ Kreuzberg: {} metadata fields",
-        kreuzberg_result.metadata.additional.len()
-    );
-    println!("   │  - Extracts core properties (creator, dates, revision)");
-    println!("   │  - Extracts app properties (page count, word count, character count)");
-    println!("   │  - Includes document statistics");
-    println!("   ├─ Pandoc: Extracts minimal metadata");
-    println!("   │  - Does not extract structured metadata");
-    println!("   │  - Returns empty meta object in JSON");
-    println!("   └─ Assessment: SUPERIOR - Kreuzberg is significantly better at metadata");
-    println!();
-
-    println!("3. TABLE HANDLING");
-    println!(
-        "   ├─ Kreuzberg: {} tables with markdown representation",
-        kreuzberg_result.tables.len()
-    );
-    println!("   │  - Tables converted to markdown format");
-    println!("   │  - Structured cell data preserved");
-    println!("   ├─ Pandoc: Converts tables to plain text or ASCII format");
-    println!("   │  - Less structured table representation");
-    println!("   └─ Assessment: SUPERIOR - Kreuzberg provides better structured data");
-    println!();
-
-    println!("4. FORMATTING PRESERVATION");
-    println!("   ├─ Kreuzberg: ");
-    println!("   │  - Preserves list structure through text");
-    println!("   │  - Maintains paragraph boundaries");
-    println!("   │  - Extracts image descriptions (alt text)");
-    println!("   ├─ Pandoc:");
-    println!("   │  - Converts lists to plain text with symbols");
-    println!("   │  - Includes image descriptions as text");
-    println!("   └─ Assessment: COMPARABLE - Both handle formatting reasonably");
-    println!();
-
-    println!("5. PERFORMANCE");
-    println!("   ├─ Kreuzberg: ~160 MB/s (streaming XML parsing)");
-    println!("   │  - No subprocess overhead");
-    println!("   │  - Direct binary parsing");
-    println!("   ├─ Pandoc: Subprocess-based");
-    println!("   │  - Higher overhead per document");
-    println!("   │  - Process spawn cost");
-    println!("   └─ Assessment: SUPERIOR - Kreuzberg ~400x faster");
-    println!();
-
-    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
-    println!("VERDICT");
-    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
-    println!();
-    println!("Kreuzberg vs Pandoc: ✅ SUPERIOR");
-    println!();
-    println!("Reasoning:");
-    println!("  1. Extracts significantly more comprehensive metadata (17 fields vs 0)");
-    println!("  2. Provides structured table data with markdown representation");
-    println!("  3. Preserves document statistics (word count, line count, paragraph count)");
-    println!("  4. Approximately 400x faster (no subprocess overhead)");
-    println!("  5. Extracts image descriptions and alt text");
-    println!("  6. Better integration as a library vs subprocess");
-    println!();
-    println!("Use Case Recommendations:");
-    println!("  • Use Kreuzberg for: Document intelligence, metadata extraction, structured data");
-    println!("  • Use Pandoc for: Format conversion, very specific format output (e.g., HTML, LaTeX)");
-    println!();
-}
-
-#[tokio::test]
-async fn test_docx_lorem_ipsum_comparison() {
-    let docx_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
-        .parent()
-        .unwrap()
-        .parent()
-        .unwrap()
-        .join("test_documents/documents/lorem_ipsum.docx");
-
-    if !docx_path.exists() {
-        println!("Skipping test: Test file not found at {:?}", docx_path);
-        return;
-    }
-
-    let content = std::fs::read(&docx_path).expect("Failed to read DOCX");
-
-    let extractor = DocxExtractor::new();
-    let config = ExtractionConfig::default();
-
-    let kreuzberg_result = extractor
-        .extract_bytes(
-            &content,
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-            &config,
-        )
-        .await
-        .expect("Kreuzberg extraction failed");
-
-    println!("\n╔════════════════════════════════════════════════════════════════╗");
-    println!("║    LOREM IPSUM TEST - Minimal Metadata Document               ║");
-    println!("╚════════════════════════════════════════════════════════════════╝");
-    println!();
-
-    println!("Document: lorem_ipsum.docx (14 KB)");
-    println!();
-
-    println!("KREUZBERG METRICS:");
-    println!("  Lines: {}", kreuzberg_result.content.lines().count());
-    println!("  Words: {}", kreuzberg_result.content.split_whitespace().count());
-    println!("  Characters: {}", kreuzberg_result.content.len());
-    println!();
-
-    println!("METADATA EXTRACTED: {}", kreuzberg_result.metadata.additional.len());
-    for (key, value) in &kreuzberg_result.metadata.additional {
-        println!("  {}: {}", key, value);
-    }
-    println!();
-
-    println!("COMPARISON NOTES:");
-    println!("  • Pandoc plain text: 55 lines, ~520 words");
-    println!("  • Kreuzberg: Full content with pagination");
-    println!("  • Metadata: Both extract similar metadata for minimal documents");
-}

data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs

@@ -1,275 +0,0 @@
-//! Integration tests for the native EPUB extractor
-//!
-//! These tests validate the native Rust EPUB extractor (EpubExtractor)
-//! which uses zip + roxmltree + html-to-markdown-rs (permissive licenses).
-//!
-//! This test suite verifies the fix for the two-pass OPF parsing bug that
-//! caused 99.84% content loss due to single-pass manifest/spine resolution.
-
-#![cfg(feature = "office")]
-
-use kreuzberg::core::config::ExtractionConfig;
-use kreuzberg::extractors::EpubExtractor;
-use kreuzberg::plugins::DocumentExtractor;
-use std::path::PathBuf;
-
-/// Helper to resolve workspace root and construct test file paths
-fn get_test_epub_path(filename: &str) -> PathBuf {
-    let workspace_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
-        .parent()
-        .unwrap()
-        .parent()
-        .unwrap();
-    workspace_root.join(format!("test_documents/epub/{}", filename))
-}
-
-/// Test 1: Basic EPUB extraction - wasteland.epub
-///
-/// Validates:
-/// - Two-pass OPF parsing works correctly
-/// - Manifest is fully populated before spine resolution
-/// - Content is extracted successfully (>2000 bytes expected)
-/// - Metadata is extracted correctly
-#[tokio::test]
-async fn test_native_epub_wasteland_extraction() {
-    let test_file = get_test_epub_path("wasteland.epub");
-    if !test_file.exists() {
-        println!("Skipping test: Test file not found at {:?}", test_file);
-        return;
-    }
-
-    let bytes = std::fs::read(&test_file).expect("Failed to read wasteland.epub");
-    let extractor = EpubExtractor::new();
-    let config = ExtractionConfig::default();
-
-    let result = extractor
-        .extract_bytes(&bytes, "application/epub+zip", &config)
-        .await
-        .expect("Should extract wasteland.epub successfully");
-
-    assert!(
-        result.content.len() > 2000,
-        "Should extract substantial content from Wasteland, got {} bytes",
-        result.content.len()
-    );
-
-    assert!(
-        result.metadata.additional.contains_key("title"),
-        "Should extract title metadata"
-    );
-    assert_eq!(
-        result.metadata.additional.get("title").and_then(|v| v.as_str()),
-        Some("The Waste Land"),
-        "Should have correct title"
-    );
-
-    assert!(
-        result.metadata.additional.contains_key("creator"),
-        "Should extract creator metadata"
-    );
-
-    assert!(
-        result.content.contains("April") || result.content.contains("cruellest"),
-        "Should contain key phrases from The Waste Land"
-    );
-
-    println!("✅ Wasteland extraction test passed ({} bytes)", result.content.len());
-}
-
-/// Test 2: EPUB with images - img.epub
-///
-/// Validates:
-/// - EPUB with embedded images extracts successfully
-/// - Text content is extracted (images are in manifest but not in content)
-/// - Metadata is extracted
-#[tokio::test]
-async fn test_native_epub_images_extraction() {
-    let test_file = get_test_epub_path("img.epub");
-    if !test_file.exists() {
-        println!("Skipping test: Test file not found at {:?}", test_file);
-        return;
-    }
-
-    let bytes = std::fs::read(&test_file).expect("Failed to read img.epub");
-    let extractor = EpubExtractor::new();
-    let config = ExtractionConfig::default();
-
-    let result = extractor
-        .extract_bytes(&bytes, "application/epub+zip", &config)
-        .await
-        .expect("Should extract img.epub successfully");
-
-    assert!(
-        result.content.len() > 50,
-        "Should extract text content from EPUB with images, got {} bytes",
-        result.content.len()
-    );
-
-    assert!(
-        result.metadata.additional.contains_key("title"),
-        "Should extract title metadata"
-    );
-
-    println!("✅ Images EPUB extraction test passed ({} bytes)", result.content.len());
-}
-
-/// Test 3: Features EPUB - features.epub
-///
-/// Validates:
-/// - Complex EPUB3 features document extracts successfully
-/// - Multiple chapters/sections are extracted (not just first)
-/// - Substantial content is present (>1000 bytes)
-#[tokio::test]
-async fn test_native_epub_features_extraction() {
-    let test_file = get_test_epub_path("features.epub");
-    if !test_file.exists() {
-        println!("Skipping test: Test file not found at {:?}", test_file);
-        return;
-    }
-
-    let bytes = std::fs::read(&test_file).expect("Failed to read features.epub");
-    let extractor = EpubExtractor::new();
-    let config = ExtractionConfig::default();
-
-    let result = extractor
-        .extract_bytes(&bytes, "application/epub+zip", &config)
-        .await
-        .expect("Should extract features.epub successfully");
-
-    assert!(
-        result.content.len() > 1000,
-        "CRITICAL: Should extract from ALL chapters, got only {} bytes. \
-         This indicates the two-pass bug is not fixed!",
-        result.content.len()
-    );
-
-    println!(
-        "✅ Features EPUB extraction test passed ({} bytes)",
-        result.content.len()
-    );
-}
-
-/// Test 4: EPUB2 with cover - epub2_cover.epub
-///
-/// Validates:
-/// - EPUB2 format is supported
-/// - Cover handling works correctly
-/// - Content and metadata extracted
-#[tokio::test]
-async fn test_native_epub2_cover_extraction() {
-    let test_file = get_test_epub_path("epub2_cover.epub");
-    if !test_file.exists() {
-        println!("Skipping test: Test file not found at {:?}", test_file);
-        return;
-    }
-
-    let bytes = std::fs::read(&test_file).expect("Failed to read epub2_cover.epub");
-    let extractor = EpubExtractor::new();
-    let config = ExtractionConfig::default();
-
-    let result = extractor
-        .extract_bytes(&bytes, "application/epub+zip", &config)
-        .await
-        .expect("Should extract epub2_cover.epub successfully");
-
-    assert!(
-        result.content.len() > 50,
-        "Should extract content from EPUB2 with cover, got {} bytes",
-        result.content.len()
-    );
-
-    assert_eq!(
-        result.metadata.additional.get("title").and_then(|v| v.as_str()),
-        Some("Pandoc EPUB Test"),
-        "Should have correct title"
-    );
-
-    println!("✅ EPUB2 cover extraction test passed ({} bytes)", result.content.len());
-}
-
-/// Test 5: Deterministic extraction
-///
-/// Validates:
-/// - Same input produces same output (no randomness)
-/// - Extraction is stable and reproducible
-#[tokio::test]
-async fn test_native_epub_deterministic_extraction() {
-    let test_file = get_test_epub_path("features.epub");
-    if !test_file.exists() {
-        println!("Skipping test: Test file not found at {:?}", test_file);
-        return;
-    }
-
-    let bytes = std::fs::read(&test_file).expect("Failed to read features.epub");
-    let extractor = EpubExtractor::new();
-    let config = ExtractionConfig::default();
-
-    let result1 = extractor
-        .extract_bytes(&bytes, "application/epub+zip", &config)
-        .await
-        .expect("First extraction should succeed");
-
-    let result2 = extractor
-        .extract_bytes(&bytes, "application/epub+zip", &config)
-        .await
-        .expect("Second extraction should succeed");
-
-    assert_eq!(
-        result1.content, result2.content,
-        "Extraction should be deterministic - same input should produce same output"
-    );
-
-    assert_eq!(
-        result1.metadata.additional, result2.metadata.additional,
-        "Metadata extraction should be deterministic"
-    );
-
-    println!("✅ Deterministic extraction test passed");
-}
-
-/// Test 6: No content loss across multiple EPUBs
-///
-/// Validates:
-/// - All test EPUB files extract successfully
-/// - No file has empty or nearly-empty content
-/// - Bug causing 99.84% content loss is fixed
-#[tokio::test]
-async fn test_native_epub_no_content_loss() {
-    let epub_files = vec![
-        ("epub2_cover.epub", 50),
-        ("epub2_no_cover.epub", 50),
-        ("img.epub", 50),
-        ("features.epub", 1000),
-        ("wasteland.epub", 2000),
-    ];
-
-    let extractor = EpubExtractor::new();
-    let config = ExtractionConfig::default();
-
-    for (epub_file, min_bytes) in epub_files {
-        let test_file = get_test_epub_path(epub_file);
-        if !test_file.exists() {
-            println!("⚠ Skipping {}: not found", epub_file);
-            continue;
-        }
-
-        let bytes = std::fs::read(&test_file).unwrap_or_else(|_| panic!("Failed to read {}", epub_file));
-
-        let result = extractor
-            .extract_bytes(&bytes, "application/epub+zip", &config)
-            .await
-            .unwrap_or_else(|_| panic!("Should extract {}", epub_file));
-
-        assert!(
-            result.content.len() >= min_bytes,
-            "CRITICAL: {} extracted only {} bytes (expected >= {}). Content loss bug?",
-            epub_file,
-            result.content.len(),
-            min_bytes
-        );
-
-        println!("✓ {} - {} bytes extracted", epub_file, result.content.len());
-    }
-
-    println!("✅ All EPUBs extracted successfully - no content loss!");
-}