kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg/tests/page_markers.rs

@@ -1,297 +0,0 @@
-//! Page marker insertion tests.
-//!
-//! Tests the page marker feature that inserts markers before each page in extracted content.
-//! This is critical for downstream applications that need to know where page boundaries are
-//! in the text stream.
-
-#![cfg(feature = "pdf")]
-
-mod helpers;
-
-use helpers::*;
-use kreuzberg::core::config::{ExtractionConfig, PageConfig};
-use kreuzberg::extract_file_sync;
-
-/// Test that page markers are inserted when enabled.
-#[test]
-fn test_page_markers_inserted_when_enabled() {
-    if skip_if_missing("pdfs/sample.pdf") {
-        return;
-    }
-
-    let file_path = get_test_file_path("pdfs/sample.pdf");
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            insert_page_markers: true,
-            ..Default::default()
-        }),
-        ..Default::default()
-    };
-
-    let result = extract_file_sync(&file_path, None, &config).expect("Failed to extract PDF with page markers");
-
-    // Default marker format is "\n\n<!-- PAGE {page_num} -->\n\n"
-    assert!(
-        result.content.contains("<!-- PAGE"),
-        "Content should contain page markers when insert_page_markers is true. Content: {}",
-        &result.content[..result.content.len().min(500)]
-    );
-}
-
-/// Test that page 1 gets a marker (regression test for the bug where page 1 was skipped).
-#[test]
-fn test_page_1_gets_marker() {
-    if skip_if_missing("pdfs/sample.pdf") {
-        return;
-    }
-
-    let file_path = get_test_file_path("pdfs/sample.pdf");
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            insert_page_markers: true,
-            ..Default::default()
-        }),
-        ..Default::default()
-    };
-
-    let result = extract_file_sync(&file_path, None, &config).expect("Failed to extract PDF with page markers");
-
-    // Page 1 should have a marker at the start
-    assert!(
-        result.content.contains("<!-- PAGE 1 -->"),
-        "Content should contain marker for page 1. Content start: {}",
-        &result.content[..result.content.len().min(200)]
-    );
-}
-
-/// Test that custom marker format works correctly.
-#[test]
-fn test_custom_marker_format() {
-    if skip_if_missing("pdfs/sample.pdf") {
-        return;
-    }
-
-    let file_path = get_test_file_path("pdfs/sample.pdf");
-    let custom_format = "=== Page {page_num} ===";
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            insert_page_markers: true,
-            marker_format: custom_format.to_string(),
-            ..Default::default()
-        }),
-        ..Default::default()
-    };
-
-    let result = extract_file_sync(&file_path, None, &config).expect("Failed to extract PDF with custom markers");
-
-    assert!(
-        result.content.contains("=== Page 1 ==="),
-        "Content should contain custom marker for page 1"
-    );
-}
-
-/// Test that {page_num} placeholder is replaced with actual page numbers.
-#[test]
-fn test_page_num_placeholder_replacement() {
-    if skip_if_missing("pdfs/sample.pdf") {
-        return;
-    }
-
-    let file_path = get_test_file_path("pdfs/sample.pdf");
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            insert_page_markers: true,
-            marker_format: "[PAGE {page_num}]".to_string(),
-            ..Default::default()
-        }),
-        ..Default::default()
-    };
-
-    let result = extract_file_sync(&file_path, None, &config).expect("Failed to extract PDF with custom markers");
-
-    // Should NOT contain the placeholder itself
-    assert!(
-        !result.content.contains("{page_num}"),
-        "Placeholder should be replaced, not appear in output"
-    );
-
-    // Should contain actual page number
-    assert!(
-        result.content.contains("[PAGE 1]"),
-        "Should contain marker with actual page number"
-    );
-}
-
-/// Test that page markers and extract_pages work together.
-#[test]
-fn test_markers_and_extract_pages_together() {
-    if skip_if_missing("pdfs/sample.pdf") {
-        return;
-    }
-
-    let file_path = get_test_file_path("pdfs/sample.pdf");
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            insert_page_markers: true,
-            extract_pages: true,
-            marker_format: "--- PAGE {page_num} ---".to_string(),
-        }),
-        ..Default::default()
-    };
-
-    let result = extract_file_sync(&file_path, None, &config).expect("Failed to extract PDF with both features");
-
-    // Should have both features working
-    assert!(
-        result.pages.is_some(),
-        "Pages array should be present when extract_pages is true"
-    );
-
-    assert!(
-        result.content.contains("--- PAGE 1 ---"),
-        "Content should contain page markers"
-    );
-}
-
-/// Test that when markers are disabled, no markers appear in content.
-#[test]
-fn test_no_markers_when_disabled() {
-    if skip_if_missing("pdfs/sample.pdf") {
-        return;
-    }
-
-    let file_path = get_test_file_path("pdfs/sample.pdf");
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            insert_page_markers: false,
-            ..Default::default()
-        }),
-        ..Default::default()
-    };
-
-    let result = extract_file_sync(&file_path, None, &config).expect("Failed to extract PDF without markers");
-
-    // Should NOT contain default marker pattern
-    assert!(
-        !result.content.contains("<!-- PAGE"),
-        "Content should not contain markers when insert_page_markers is false"
-    );
-}
-
-/// Test that markers appear before page content, not after.
-#[test]
-fn test_marker_appears_before_content() {
-    if skip_if_missing("pdfs/sample.pdf") {
-        return;
-    }
-
-    let file_path = get_test_file_path("pdfs/sample.pdf");
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            insert_page_markers: true,
-            marker_format: "[[PAGE {page_num}]]".to_string(),
-            ..Default::default()
-        }),
-        ..Default::default()
-    };
-
-    let result = extract_file_sync(&file_path, None, &config).expect("Failed to extract PDF with markers");
-
-    // The marker should appear at or near the start
-    let marker_pos = result.content.find("[[PAGE 1]]");
-    assert!(marker_pos.is_some(), "Marker should be present");
-
-    // Marker should be very early in the content (within first 50 chars)
-    let pos = marker_pos.unwrap();
-    assert!(
-        pos < 50,
-        "Marker for page 1 should appear at the start, but found at position {}",
-        pos
-    );
-}
-
-/// Test that multi-page PDFs get markers for all pages.
-#[test]
-fn test_multi_page_markers() {
-    if skip_if_missing("pdfs/sample.pdf") {
-        return;
-    }
-
-    let file_path = get_test_file_path("pdfs/sample.pdf");
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            insert_page_markers: true,
-            extract_pages: true,
-            ..Default::default()
-        }),
-        ..Default::default()
-    };
-
-    let result = extract_file_sync(&file_path, None, &config).expect("Failed to extract PDF");
-
-    if let Some(ref pages) = result.pages {
-        let page_count = pages.len();
-
-        // Check that we have markers for each page
-        for page_num in 1..=page_count.min(3) {
-            let marker = format!("<!-- PAGE {} -->", page_num);
-            assert!(
-                result.content.contains(&marker),
-                "Should contain marker for page {} (total pages: {})",
-                page_num,
-                page_count
-            );
-        }
-    }
-}
-
-/// Test default marker format value.
-#[test]
-fn test_default_marker_format() {
-    let config = PageConfig::default();
-    assert_eq!(
-        config.marker_format, "\n\n<!-- PAGE {page_num} -->\n\n",
-        "Default marker format should match expected value"
-    );
-}
-
-/// Test that empty page still gets a marker.
-#[test]
-fn test_empty_page_gets_marker() {
-    // This would require a specific test PDF with an empty page
-    // For now, we just verify the logic doesn't skip pages based on content length
-    let config = PageConfig {
-        insert_page_markers: true,
-        ..Default::default()
-    };
-
-    assert!(
-        config.insert_page_markers,
-        "Config should enable markers regardless of page content"
-    );
-}
-
-/// Test marker format with multiple placeholders (edge case).
-#[test]
-fn test_marker_format_multiple_placeholders() {
-    if skip_if_missing("pdfs/sample.pdf") {
-        return;
-    }
-
-    let file_path = get_test_file_path("pdfs/sample.pdf");
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            insert_page_markers: true,
-            marker_format: "Page {page_num} of document (page {page_num})".to_string(),
-            ..Default::default()
-        }),
-        ..Default::default()
-    };
-
-    let result = extract_file_sync(&file_path, None, &config).expect("Failed to extract PDF");
-
-    assert!(
-        result.content.contains("Page 1 of document (page 1)"),
-        "Multiple {{page_num}} placeholders should all be replaced"
-    );
-}

data/vendor/kreuzberg/tests/pdf_hierarchy_detection.rs

@@ -1,301 +0,0 @@
-//! Integration tests for PDF text hierarchy detection.
-//!
-//! Tests the extraction and detection of document hierarchy levels (H1-H6)
-//! from PDF text using font size clustering and semantic analysis.
-
-#![cfg(feature = "pdf")]
-
-use kreuzberg::core::config::{ExtractionConfig, HierarchyConfig, PageConfig, PdfConfig};
-use kreuzberg::extract_bytes;
-use std::path::Path;
-
-// Note: All tests must run serially because Pdfium can only be initialized once.
-// Using tokio::test with single_threaded doesn't work well, so we use the serial_test crate.
-// For now, we'll just accept that tests run in parallel but handle the Pdfium initialization error.
-
-/// Test full hierarchy extraction from a real PDF.
-///
-/// Loads a PDF from test data directory, extracts with hierarchy detection enabled,
-/// and verifies that PageContent.hierarchy is properly populated with expected
-/// blocks and hierarchy levels.
-#[tokio::test]
-async fn test_full_hierarchy_extraction() {
-    // Use the embedded_images_tables.pdf which has clear text structure
-    // Path is relative to workspace root, not crate root
-    let pdf_path = "../../test_documents/pdfs/embedded_images_tables.pdf";
-
-    if !Path::new(pdf_path).exists() {
-        eprintln!("Test PDF not found at: {}", pdf_path);
-        // Skip the test if PDF doesn't exist
-        return;
-    }
-
-    let pdf_bytes = std::fs::read(pdf_path).expect("Failed to read PDF file");
-
-    // Create extraction config with hierarchy detection enabled
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            extract_pages: true,
-            insert_page_markers: false,
-            marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
-        }),
-        pdf_options: Some(PdfConfig {
-            extract_images: false,
-            passwords: None,
-            extract_metadata: true,
-            hierarchy: Some(HierarchyConfig {
-                enabled: true,
-                k_clusters: 6,
-                include_bbox: true,
-                ocr_coverage_threshold: None,
-            }),
-        }),
-        ..Default::default()
-    };
-
-    // Extract the PDF
-    let result = extract_bytes(&pdf_bytes, "application/pdf", &config)
-        .await
-        .expect("PDF extraction failed");
-
-    // Verify that pages were extracted
-    assert!(
-        result.pages.is_some(),
-        "Pages should be extracted when extract_pages is enabled"
-    );
-
-    let pages = result.pages.as_ref().unwrap();
-    assert!(!pages.is_empty(), "At least one page should be extracted");
-
-    // Check that the first page has hierarchy information
-    let first_page = &pages[0];
-    assert!(
-        first_page.hierarchy.is_some(),
-        "First page should have hierarchy information when hierarchy extraction is enabled"
-    );
-
-    let hierarchy = first_page.hierarchy.as_ref().unwrap();
-
-    // Verify hierarchy structure
-    assert!(hierarchy.block_count > 0, "Hierarchy should contain at least one block");
-    assert!(!hierarchy.blocks.is_empty(), "Hierarchy blocks should not be empty");
-
-    eprintln!("Extracted {} hierarchy blocks from page 1", hierarchy.block_count);
-
-    // Verify that we have multiple hierarchy levels
-    let levels: std::collections::HashSet<String> = hierarchy.blocks.iter().map(|b| b.level.clone()).collect();
-
-    eprintln!("Found hierarchy levels: {:?}", levels);
-
-    // Should have at least 1 level
-    assert!(!levels.is_empty(), "Should have at least one hierarchy level");
-
-    // Verify block structure
-    for block in &hierarchy.blocks {
-        assert!(!block.text.is_empty(), "Block text should not be empty");
-        assert!(block.font_size > 0.0, "Font size should be positive");
-
-        // Check that level is a valid heading level or body
-        let is_valid_level = matches!(block.level.as_str(), "h1" | "h2" | "h3" | "h4" | "h5" | "h6" | "body");
-        assert!(is_valid_level, "Invalid hierarchy level: {}", block.level);
-
-        // Verify bounding box if present
-        if let Some((left, top, right, bottom)) = block.bbox {
-            assert!(left < right, "Bounding box left should be less than right");
-            assert!(top < bottom, "Bounding box top should be less than bottom");
-            assert!(
-                left >= 0.0 && top >= 0.0,
-                "Bounding box coordinates should be non-negative"
-            );
-            eprintln!(
-                "Block '{}' (level: {}, font_size: {}) bbox: ({}, {}, {}, {})",
-                block.text.chars().take(30).collect::<String>(),
-                block.level,
-                block.font_size,
-                left,
-                top,
-                right,
-                bottom
-            );
-        } else {
-            eprintln!(
-                "Block '{}' (level: {}, font_size: {}) no bbox",
-                block.text.chars().take(30).collect::<String>(),
-                block.level,
-                block.font_size
-            );
-        }
-    }
-
-    eprintln!("Hierarchy extraction test passed!");
-}
-
-/// Test that hierarchy extraction respects the enabled flag.
-/// Note: This test is combined with the full_hierarchy_extraction test due to Pdfium initialization constraints.
-#[tokio::test]
-#[ignore]
-async fn test_hierarchy_disabled() {
-    let pdf_path = "../../test_documents/pdfs/embedded_images_tables.pdf";
-
-    if !Path::new(pdf_path).exists() {
-        eprintln!("Test PDF not found at: {}", pdf_path);
-        return;
-    }
-
-    let pdf_bytes = std::fs::read(pdf_path).expect("Failed to read PDF file");
-
-    // Create extraction config with hierarchy detection disabled
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            extract_pages: true,
-            insert_page_markers: false,
-            marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
-        }),
-        pdf_options: Some(PdfConfig {
-            extract_images: false,
-            passwords: None,
-            extract_metadata: true,
-            hierarchy: Some(HierarchyConfig {
-                enabled: false,
-                k_clusters: 6,
-                include_bbox: true,
-                ocr_coverage_threshold: None,
-            }),
-        }),
-        ..Default::default()
-    };
-
-    let result = extract_bytes(&pdf_bytes, "application/pdf", &config)
-        .await
-        .expect("PDF extraction failed");
-
-    // Verify that pages were extracted
-    assert!(result.pages.is_some(), "Pages should be extracted");
-
-    let pages = result.pages.as_ref().unwrap();
-    assert!(!pages.is_empty(), "At least one page should be extracted");
-
-    // Check that the first page does NOT have hierarchy information when disabled
-    let first_page = &pages[0];
-    assert!(
-        first_page.hierarchy.is_none(),
-        "First page should not have hierarchy when hierarchy extraction is disabled"
-    );
-
-    eprintln!("Hierarchy disabled test passed!");
-}
-
-/// Test different hierarchy configurations
-/// Note: This test is ignored due to Pdfium initialization constraints (can only initialize once).
-#[tokio::test]
-#[ignore]
-async fn test_hierarchy_with_explicit_disabled() {
-    let pdf_path = "../../test_documents/pdfs/embedded_images_tables.pdf";
-
-    if !Path::new(pdf_path).exists() {
-        eprintln!("Test PDF not found at: {}", pdf_path);
-        return;
-    }
-
-    let pdf_bytes = std::fs::read(pdf_path).expect("Failed to read PDF file");
-
-    // Create extraction config with hierarchy extraction explicitly disabled
-    let config = ExtractionConfig {
-        pages: Some(PageConfig {
-            extract_pages: true,
-            insert_page_markers: false,
-            marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
-        }),
-        pdf_options: Some(PdfConfig {
-            extract_images: false,
-            passwords: None,
-            extract_metadata: true,
-            hierarchy: Some(HierarchyConfig {
-                enabled: false,
-                k_clusters: 6,
-                include_bbox: true,
-                ocr_coverage_threshold: None,
-            }),
-        }),
-        ..Default::default()
-    };
-
-    let result = extract_bytes(&pdf_bytes, "application/pdf", &config)
-        .await
-        .expect("PDF extraction failed");
-
-    // Verify that pages were extracted
-    assert!(result.pages.is_some(), "Pages should be extracted");
-
-    let pages = result.pages.as_ref().unwrap();
-    assert!(!pages.is_empty(), "At least one page should be extracted");
-
-    // Check that the first page does NOT have hierarchy information when disabled
-    let first_page = &pages[0];
-    assert!(
-        first_page.hierarchy.is_none(),
-        "First page should not have hierarchy when hierarchy extraction is disabled"
-    );
-
-    eprintln!("Hierarchy with explicit disabled test passed!");
-}
-
-/// Test hierarchy extraction with different cluster configurations.
-/// Note: This test is ignored due to Pdfium initialization constraints (can only initialize once).
-#[tokio::test]
-#[ignore]
-async fn test_hierarchy_different_k_clusters() {
-    let pdf_path = "../../test_documents/pdfs/embedded_images_tables.pdf";
-
-    if !Path::new(pdf_path).exists() {
-        eprintln!("Test PDF not found at: {}", pdf_path);
-        return;
-    }
-
-    let pdf_bytes = std::fs::read(pdf_path).expect("Failed to read PDF file");
-
-    // Test with different k values
-    for k in &[2, 4, 6] {
-        let config = ExtractionConfig {
-            pages: Some(PageConfig {
-                extract_pages: true,
-                insert_page_markers: false,
-                marker_format: "\n\n<!-- PAGE {page_num} -->\n\n".to_string(),
-            }),
-            pdf_options: Some(PdfConfig {
-                extract_images: false,
-                passwords: None,
-                extract_metadata: true,
-                hierarchy: Some(HierarchyConfig {
-                    enabled: true,
-                    k_clusters: *k,
-                    include_bbox: true,
-                    ocr_coverage_threshold: None,
-                }),
-            }),
-            ..Default::default()
-        };
-
-        let result = extract_bytes(&pdf_bytes, "application/pdf", &config)
-            .await
-            .expect("PDF extraction failed");
-
-        assert!(result.pages.is_some(), "Pages should be extracted");
-
-        let pages = result.pages.as_ref().unwrap();
-        assert!(!pages.is_empty(), "At least one page should be extracted");
-
-        let first_page = &pages[0];
-        assert!(
-            first_page.hierarchy.is_some(),
-            "Hierarchy should be present with k={}",
-            k
-        );
-
-        let hierarchy = first_page.hierarchy.as_ref().unwrap();
-        eprintln!("K={}: {} hierarchy blocks extracted", k, hierarchy.block_count);
-        assert!(hierarchy.block_count > 0, "Should have blocks with k={}", k);
-    }
-
-    eprintln!("Different k_clusters test passed!");
-}