kreuzberg 4.0.0.rc1 → 4.0.0.rc2

data/vendor/kreuzberg/tests/typst_extractor_tests.rs

@@ -0,0 +1,647 @@
+//! Comprehensive TDD test suite for Typst document extraction.
+//!
+//! This test suite validates Typst document extraction against expected outputs.
+//! The tests verify:
+//! - Document metadata extraction (title, author, date, keywords)
+//! - Heading hierarchy parsing (=, ==, ===, etc.)
+//! - Inline formatting (bold, italic, code)
+//! - Table extraction and parsing
+//! - List handling (ordered and unordered)
+//! - Link extraction
+//! - Mathematical notation preservation
+//!
+//! Each test document is extracted and validated for correct content extraction.
+
+use kreuzberg::core::config::ExtractionConfig;
+use kreuzberg::core::extractor::extract_bytes;
+use std::{fs, path::PathBuf};
+
+fn typst_fixture(name: &str) -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("../../test_documents/typst")
+        .join(name)
+}
+
+/// Test simple.typ - Basic Typst document with fundamental formatting
+///
+/// Document contains:
+/// - Document metadata: title, author, date
+/// - Level 1 heading: "Introduction"
+/// - Level 2 headings: "Subsection", "Features", "Lists", "Code", "Tables", "Links", "Conclusion"
+/// - Inline formatting: *bold*, _italic_, `inline code`
+/// - Unordered list with 3 items
+/// - Code snippet
+/// - 2x2 table with headers
+/// - Link to Typst website
+///
+/// Expected: Document should extract text, preserve headings, metadata, and formatting markers
+#[tokio::test]
+async fn test_simple_typst_document_extraction() {
+    let config = ExtractionConfig::default();
+
+    let doc_path = typst_fixture("simple.typ");
+    let content = match fs::read(doc_path) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("Warning: Could not read simple.typ: {}. Skipping test.", e);
+            return;
+        }
+    };
+
+    let result = extract_bytes(&content, "text/x-typst", &config).await;
+    if result.is_err() {
+        println!("Skipping test: Typst extractor may not be available");
+        return;
+    }
+
+    let extraction = result.unwrap();
+
+    assert_eq!(extraction.mime_type, "text/x-typst", "MIME type should be preserved");
+
+    assert!(!extraction.content.is_empty(), "Extracted content should not be empty");
+
+    assert!(
+        extraction.metadata.additional.contains_key("title"),
+        "Document title should be extracted from #set document()"
+    );
+
+    assert!(
+        extraction.metadata.additional.contains_key("author"),
+        "Document author should be extracted"
+    );
+
+    assert!(
+        extraction.content.contains("Introduction"),
+        "Should extract 'Introduction' heading"
+    );
+    assert!(
+        extraction.content.contains("Features"),
+        "Should extract 'Features' heading"
+    );
+    assert!(
+        extraction.content.contains("Conclusion"),
+        "Should extract 'Conclusion' heading"
+    );
+
+    let intro_count = extraction.content.matches("= Introduction").count();
+    let subsection_count = extraction.content.matches("== Subsection").count();
+    let features_count = extraction.content.matches("= Features").count();
+    let lists_count = extraction.content.matches("== Lists").count();
+    let code_count = extraction.content.matches("== Code").count();
+    let tables_count = extraction.content.matches("== Tables").count();
+    let links_count = extraction.content.matches("== Links").count();
+    let conclusion_count = extraction.content.matches("= Conclusion").count();
+
+    assert_eq!(intro_count, 1, "Should extract 'Introduction' (level 1)");
+    assert_eq!(subsection_count, 1, "Should extract 'Subsection' (level 2)");
+    assert_eq!(features_count, 1, "Should extract 'Features' (level 1)");
+    assert_eq!(lists_count, 1, "Should extract 'Lists' (level 2)");
+    assert_eq!(code_count, 1, "Should extract 'Code' (level 2)");
+    assert_eq!(tables_count, 1, "Should extract 'Tables' (level 2)");
+    assert_eq!(links_count, 1, "Should extract 'Links' (level 2)");
+    assert_eq!(conclusion_count, 1, "Should extract 'Conclusion' (level 1)");
+
+    assert!(
+        extraction.content.contains("*") || extraction.content.contains("bold"),
+        "Should preserve bold formatting or text"
+    );
+
+    assert!(
+        extraction.content.contains("-") || extraction.content.contains("First") || extraction.content.contains("item"),
+        "Should extract list content"
+    );
+
+    println!(
+        "✓ simple.typ: Successfully extracted {} characters with all 8 headings",
+        extraction.content.len()
+    );
+}
+
+/// Test minimal.typ - Minimal Typst document
+///
+/// Document contains:
+/// - Single level 1 heading: "Hello World"
+/// - Simple text content
+///
+/// Expected: Basic heading and content extraction
+#[tokio::test]
+async fn test_minimal_typst_document_extraction() {
+    let config = ExtractionConfig::default();
+
+    let doc_path = typst_fixture("minimal.typ");
+    let content = match fs::read(doc_path) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("Warning: Could not read minimal.typ: {}. Skipping test.", e);
+            return;
+        }
+    };
+
+    let result = extract_bytes(&content, "application/x-typst", &config).await;
+    if result.is_err() {
+        println!("Skipping test: Typst extractor may not be available");
+        return;
+    }
+
+    let extraction = result.unwrap();
+
+    assert!(
+        !extraction.content.is_empty(),
+        "Minimal document should extract content"
+    );
+
+    assert!(
+        extraction.content.contains("Hello") || extraction.content.contains("World"),
+        "Should extract heading content"
+    );
+
+    println!(
+        "✓ minimal.typ: Successfully extracted {} characters",
+        extraction.content.len()
+    );
+}
+
+/// Test headings.typ - Document focusing on heading hierarchy
+///
+/// Document contains:
+/// - 6 heading levels (=, ==, ===, ====, =====, ======)
+/// - Content under each heading level
+///
+/// Expected: Heading structure should be preserved with level information
+#[tokio::test]
+async fn test_heading_hierarchy_extraction() {
+    let config = ExtractionConfig::default();
+
+    let doc_path = typst_fixture("headings.typ");
+    let content = match fs::read(doc_path) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("Warning: Could not read headings.typ: {}. Skipping test.", e);
+            return;
+        }
+    };
+
+    let result = extract_bytes(&content, "text/x-typst", &config).await;
+    if result.is_err() {
+        println!("Skipping test: Typst extractor may not be available");
+        return;
+    }
+
+    let extraction = result.unwrap();
+
+    assert!(!extraction.content.is_empty(), "Document should extract content");
+
+    assert!(
+        extraction.content.contains("= Level 1") || extraction.content.contains("Level 1 Heading"),
+        "Should extract level 1 heading"
+    );
+
+    assert!(
+        extraction.content.contains("== Level 2") || extraction.content.contains("Level 2 Heading"),
+        "Should extract level 2 heading"
+    );
+
+    assert!(
+        extraction.content.contains("=== Level 3") || extraction.content.contains("Level 3 Heading"),
+        "Should extract level 3 heading"
+    );
+
+    assert!(
+        extraction.content.contains("==== Level 4") || extraction.content.contains("Level 4 Heading"),
+        "Should extract level 4 heading"
+    );
+
+    assert!(
+        extraction.content.contains("===== Level 5") || extraction.content.contains("Level 5 Heading"),
+        "Should extract level 5 heading"
+    );
+
+    assert!(
+        extraction.content.contains("====== Level 6") || extraction.content.contains("Level 6 Heading"),
+        "Should extract level 6 heading"
+    );
+
+    let level_1_count = extraction.content.matches("= Level 1").count();
+    let level_2_count = extraction.content.matches("== Level 2").count();
+    let level_3_count = extraction.content.matches("=== Level 3").count();
+    let level_4_count = extraction.content.matches("==== Level 4").count();
+    let level_5_count = extraction.content.matches("===== Level 5").count();
+    let level_6_count = extraction.content.matches("====== Level 6").count();
+
+    assert_eq!(level_1_count, 1, "Should extract exactly one level 1 heading");
+    assert_eq!(level_2_count, 1, "Should extract exactly one level 2 heading");
+    assert_eq!(level_3_count, 1, "Should extract exactly one level 3 heading");
+    assert_eq!(level_4_count, 1, "Should extract exactly one level 4 heading");
+    assert_eq!(level_5_count, 1, "Should extract exactly one level 5 heading");
+    assert_eq!(level_6_count, 1, "Should extract exactly one level 6 heading");
+
+    println!(
+        "✓ headings.typ: Successfully extracted {} characters with heading structure",
+        extraction.content.len()
+    );
+}
+
+/// Test metadata.typ - Document with comprehensive metadata
+///
+/// Document contains:
+/// - #set document() with: title, author, subject, keywords
+/// - Content sections
+///
+/// Expected: All metadata fields should be extracted correctly
+#[tokio::test]
+async fn test_metadata_extraction() {
+    let config = ExtractionConfig::default();
+
+    let doc_path = typst_fixture("metadata.typ");
+    let content = match fs::read(doc_path) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("Warning: Could not read metadata.typ: {}. Skipping test.", e);
+            return;
+        }
+    };
+
+    let result = extract_bytes(&content, "application/x-typst", &config).await;
+    if result.is_err() {
+        println!("Skipping test: Typst extractor may not be available");
+        return;
+    }
+
+    let extraction = result.unwrap();
+
+    if let Some(title) = extraction.metadata.additional.get("title") {
+        assert!(
+            title.to_string().contains("Metadata") || title.to_string().contains("Example"),
+            "Title should contain expected text"
+        );
+    }
+
+    if let Some(author) = extraction.metadata.additional.get("author") {
+        assert!(
+            author.to_string().contains("John") || author.to_string().contains("Doe"),
+            "Author should contain expected text"
+        );
+    }
+
+    if let Some(keywords) = extraction.metadata.additional.get("keywords") {
+        assert!(!keywords.to_string().is_empty(), "Keywords should be present");
+    }
+
+    assert!(!extraction.content.is_empty(), "Document should extract content");
+
+    println!(
+        "✓ metadata.typ: Successfully extracted metadata and {} characters of content",
+        extraction.content.len()
+    );
+}
+
+/// Test advanced.typ - Complex Typst document with multiple features
+///
+/// Document contains:
+/// - Metadata: title, author, keywords, date
+/// - Heading numbering configuration
+/// - Mathematical notation (inline and display)
+/// - Nested heading levels (level 1, 2, 3, 4)
+/// - Code blocks (Python example)
+/// - Complex tables with 3 columns and 4 rows
+/// - Multiple paragraph sections
+/// - Links with text
+/// - Multiple formatting combinations
+///
+/// Expected: Comprehensive extraction of all document elements
+#[tokio::test]
+async fn test_advanced_typst_document_extraction() {
+    let config = ExtractionConfig::default();
+
+    let doc_path = typst_fixture("advanced.typ");
+    let content = match fs::read(doc_path) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("Warning: Could not read advanced.typ: {}. Skipping test.", e);
+            return;
+        }
+    };
+
+    let result = extract_bytes(&content, "text/x-typst", &config).await;
+    if result.is_err() {
+        println!("Skipping test: Typst extractor may not be available");
+        return;
+    }
+
+    let extraction = result.unwrap();
+
+    assert!(
+        extraction.metadata.additional.contains_key("title"),
+        "Title should be extracted"
+    );
+
+    assert!(
+        !extraction.content.is_empty(),
+        "Advanced document should extract content"
+    );
+
+    assert!(
+        extraction.content.contains("$")
+            || extraction.content.contains("equation")
+            || extraction.content.contains("math"),
+        "Should extract or preserve mathematical notation"
+    );
+
+    assert!(
+        extraction.content.contains("Mathematical")
+            || extraction.content.contains("Formatting")
+            || extraction.content.contains("Features"),
+        "Should extract section headings"
+    );
+
+    assert!(
+        extraction.content.contains("python")
+            || extraction.content.contains("def")
+            || extraction.content.contains("fibonacci")
+            || extraction.content.contains("```"),
+        "Should extract code block content"
+    );
+
+    let level_count = extraction.content.matches("=").count();
+    assert!(level_count >= 3, "Should preserve nested heading hierarchy");
+
+    assert!(
+        extraction.content.contains("Name")
+            || extraction.content.contains("Alice")
+            || extraction.content.contains("Table"),
+        "Should extract table content"
+    );
+
+    assert!(
+        extraction.content.contains("example")
+            || extraction.content.contains("link")
+            || extraction.content.contains("http"),
+        "Should extract link content"
+    );
+
+    println!(
+        "✓ advanced.typ: Successfully extracted {} characters with complex formatting",
+        extraction.content.len()
+    );
+}
+
+/// Test typst-reader.typ - Pandoc test file
+///
+/// Document from Pandoc test suite demonstrating Typst reader functionality
+///
+/// Expected: Proper extraction of Typst-specific syntax
+#[tokio::test]
+async fn test_typst_reader_extraction() {
+    let config = ExtractionConfig::default();
+
+    let doc_path = typst_fixture("typst-reader.typ");
+    let content = match fs::read(doc_path) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("Warning: Could not read typst-reader.typ: {}. Skipping test.", e);
+            return;
+        }
+    };
+
+    let result = extract_bytes(&content, "application/x-typst", &config).await;
+    if result.is_err() {
+        println!("Skipping test: Typst extractor may not be available");
+        return;
+    }
+
+    let extraction = result.unwrap();
+
+    assert!(
+        !extraction.content.is_empty(),
+        "Should extract content from Pandoc test file"
+    );
+
+    assert!(
+        extraction.content.contains("=") || extraction.content.contains("Fibonacci"),
+        "Should extract heading or content from test file"
+    );
+
+    println!(
+        "✓ typst-reader.typ: Successfully extracted {} characters",
+        extraction.content.len()
+    );
+}
+
+/// Test undergradmath.typ - Pandoc test file with complex math
+///
+/// Document from Pandoc test suite with extensive mathematical notation
+/// and complex formatting
+///
+/// Expected: Handling of complex Typst syntax with metadata and content
+#[tokio::test]
+async fn test_undergradmath_extraction() {
+    let config = ExtractionConfig::default();
+
+    let doc_path = typst_fixture("undergradmath.typ");
+    let content = match fs::read(doc_path) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("Warning: Could not read undergradmath.typ: {}. Skipping test.", e);
+            return;
+        }
+    };
+
+    let result = extract_bytes(&content, "text/x-typst", &config).await;
+    if result.is_err() {
+        println!("Skipping test: Typst extractor may not be available");
+        return;
+    }
+
+    let extraction = result.unwrap();
+
+    assert!(
+        !extraction.content.is_empty(),
+        "Should extract content from complex math document"
+    );
+
+    if let Some(title) = extraction.metadata.additional.get("title") {
+        assert!(!title.to_string().is_empty(), "Title should be extracted");
+    }
+
+    assert!(
+        extraction.content.contains("=") || extraction.content.contains("Typst") || extraction.content.len() > 100,
+        "Should extract document structure or content"
+    );
+
+    println!(
+        "✓ undergradmath.typ: Successfully extracted {} characters from math document",
+        extraction.content.len()
+    );
+}
+
+/// Test MIME type detection and fallback
+///
+/// Verifies that Typst documents can be extracted with different MIME type specifications
+#[tokio::test]
+async fn test_typst_mime_type_variants() {
+    let config = ExtractionConfig::default();
+
+    let doc_path = typst_fixture("simple.typ");
+    let content = match fs::read(doc_path) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("Warning: Could not read simple.typ: {}. Skipping test.", e);
+            return;
+        }
+    };
+
+    let mime_types = vec!["application/x-typst", "text/x-typst", "text/plain"];
+
+    for mime_type in mime_types {
+        let result = extract_bytes(&content, mime_type, &config).await;
+
+        if result.is_ok() {
+            let extraction = result.unwrap();
+            assert!(
+                !extraction.content.is_empty(),
+                "Should extract content with MIME type: {}",
+                mime_type
+            );
+            println!(
+                "✓ MIME type '{}': Successfully extracted {} characters",
+                mime_type,
+                extraction.content.len()
+            );
+        }
+    }
+}
+
+/// Test formatting preservation
+///
+/// Validates that inline formatting markers are preserved in extracted content
+#[tokio::test]
+async fn test_formatting_preservation() {
+    let config = ExtractionConfig::default();
+
+    let doc_path = typst_fixture("simple.typ");
+    let content = match fs::read(doc_path) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("Warning: Could not read simple.typ: {}. Skipping test.", e);
+            return;
+        }
+    };
+
+    let result = extract_bytes(&content, "text/x-typst", &config).await;
+    if result.is_err() {
+        println!("Skipping test: Typst extractor may not be available");
+        return;
+    }
+
+    let extraction = result.unwrap();
+
+    assert!(
+        extraction.content.contains("*") || extraction.content.contains("bold"),
+        "Should preserve bold formatting or text"
+    );
+
+    assert!(
+        extraction.content.contains("_") || extraction.content.contains("italic"),
+        "Should preserve italic formatting or text"
+    );
+
+    assert!(
+        extraction.content.contains("`") || extraction.content.contains("code"),
+        "Should preserve code formatting or text"
+    );
+
+    println!("✓ Formatting preservation: All markers/content found in extracted text");
+}
+
+/// Test large document handling
+///
+/// Validates extraction of the large undergradmath document
+#[tokio::test]
+async fn test_large_document_extraction() {
+    let config = ExtractionConfig::default();
+
+    let doc_path = typst_fixture("undergradmath.typ");
+    let content = match fs::read(doc_path) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("Warning: Could not read undergradmath.typ: {}. Skipping test.", e);
+            return;
+        }
+    };
+
+    let result = extract_bytes(&content, "text/x-typst", &config).await;
+    if result.is_err() {
+        println!("Skipping test: Typst extractor may not be available");
+        return;
+    }
+
+    let extraction = result.unwrap();
+
+    assert!(
+        !extraction.content.is_empty(),
+        "Should extract content from large document"
+    );
+
+    println!(
+        "✓ Large document: Extracted {} bytes of content from source file",
+        extraction.content.len()
+    );
+}
+
+/// Test empty/whitespace handling
+///
+/// Validates graceful handling of edge cases
+#[tokio::test]
+async fn test_empty_content_handling() {
+    let config = ExtractionConfig::default();
+
+    let empty_content = b"";
+    let result = extract_bytes(empty_content, "text/x-typst", &config).await;
+
+    match result {
+        Ok(extraction) => {
+            println!(
+                "✓ Empty content: Handled gracefully, extracted {} bytes",
+                extraction.content.len()
+            );
+        }
+        Err(e) => {
+            println!("✓ Empty content: Resulted in expected error: {}", e);
+        }
+    }
+}
+
+/// Test MIME type priority
+///
+/// Validates that Typst extractor has correct priority (50)
+#[tokio::test]
+async fn test_typst_extractor_priority() {
+    use kreuzberg::extractors::TypstExtractor;
+    use kreuzberg::plugins::DocumentExtractor;
+
+    let extractor = TypstExtractor::new();
+    let priority = extractor.priority();
+
+    assert_eq!(priority, 50, "Typst extractor should have priority 50");
+    println!("✓ Typst extractor priority: {}", priority);
+}
+
+/// Test supported MIME types
+///
+/// Validates that extractor claims to support Typst MIME types
+#[tokio::test]
+async fn test_supported_mime_types() {
+    use kreuzberg::extractors::TypstExtractor;
+    use kreuzberg::plugins::DocumentExtractor;
+
+    let extractor = TypstExtractor::new();
+    let mime_types = extractor.supported_mime_types();
+
+    assert!(
+        mime_types.contains(&"application/x-typst"),
+        "Should support application/x-typst"
+    );
+    assert!(mime_types.contains(&"text/x-typst"), "Should support text/x-typst");
+
+    println!("✓ Supported MIME types: {:?}", mime_types);
+}