kreuzberg 4.0.0.rc2 → 4.0.1

data/vendor/kreuzberg/tests/pdf_text_merging.rs

@@ -0,0 +1,475 @@
+//! Tests for PDF text extraction with font information and bounding box operations.
+//!
+//! This module tests character extraction functionality following TDD principles,
+//! verifying that characters are extracted with positions and font sizes,
+//! and that bounding box operations work correctly.
+
+#![cfg(feature = "pdf")]
+
+mod helpers;
+
+use helpers::get_test_file_path;
+use kreuzberg::pdf::hierarchy::{BoundingBox, extract_chars_with_fonts};
+use pdfium_render::prelude::*;
+
+// ============================================================================
+// Character Extraction Tests (Following TDD)
+// ============================================================================
+
+/// Test basic character extraction with positions and font sizes.
+///
+/// Verifies that:
+/// - Characters are extracted with their positions
+/// - Font sizes are captured
+/// - Position order is preserved
+#[test]
+fn test_extract_chars_basic() {
+    let pdf_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
+
+    // Load PDF
+    let pdfium = Pdfium;
+    let document = pdfium
+        .load_pdf_from_file(pdf_path.to_str().unwrap(), None)
+        .expect("Failed to load test PDF");
+
+    // Get first page
+    let page = document.pages().get(0).expect("Failed to get first page");
+
+    // Extract characters with fonts
+    let chars = extract_chars_with_fonts(&page).expect("Failed to extract characters with fonts");
+
+    // Verify we got some characters
+    assert!(!chars.is_empty(), "Should extract at least one character from test PDF");
+
+    // Verify each character has required fields
+    for char_data in chars.iter() {
+        assert!(!char_data.text.is_empty(), "Character text should not be empty");
+        assert!(char_data.font_size > 0.0, "Font size should be positive");
+        assert!(char_data.x >= 0.0, "X position should be non-negative");
+        assert!(char_data.y >= 0.0, "Y position should be non-negative");
+    }
+}
+
+/// Test that character extraction preserves reading order.
+///
+/// Verifies that:
+/// - Characters appear in left-to-right order
+/// - Y-coordinates generally decrease as we move down the page
+#[test]
+fn test_extract_chars_preserves_order() {
+    let pdf_path = get_test_file_path("pdfs_with_tables/tiny.pdf");
+
+    // Load PDF
+    let pdfium = Pdfium;
+    let document = pdfium
+        .load_pdf_from_file(pdf_path.to_str().unwrap(), None)
+        .expect("Failed to load test PDF");
+
+    // Get first page
+    let page = document.pages().get(0).expect("Failed to get first page");
+
+    // Extract characters with fonts
+    let chars = extract_chars_with_fonts(&page).expect("Failed to extract characters with fonts");
+
+    assert!(!chars.is_empty(), "Should extract at least one character");
+
+    // Within each line (similar y-coordinate), characters should be left-to-right
+    let mut last_y = f32::NEG_INFINITY;
+    let mut last_x = f32::NEG_INFINITY;
+    let y_line_threshold = 5.0; // Consider within 5 units as same line
+
+    for char_data in chars.iter() {
+        // If we're on a new line
+        if (char_data.y - last_y).abs() > y_line_threshold {
+            last_x = f32::NEG_INFINITY;
+            last_y = char_data.y;
+        }
+
+        // On same line, x should generally increase (allowing for small variations)
+        // We use a threshold to allow for measurement precision issues
+        if (char_data.y - last_y).abs() <= y_line_threshold && char_data.x < last_x - 1.0 {
+            // This is acceptable if it's a new line or small variation
+            if last_x != f32::NEG_INFINITY && (char_data.y - last_y).abs() <= y_line_threshold {
+                panic!(
+                    "Characters should be left-to-right on same line: {} < {} at y={}",
+                    char_data.x, last_x, char_data.y
+                );
+            }
+        }
+
+        last_x = char_data.x;
+        last_y = char_data.y;
+    }
+}
+
+// ============================================================================
+// Bounding Box Tests
+// ============================================================================
+
+/// Helper function to create a BoundingBox from x, y, width, height
+fn create_bbox(x: f32, y: f32, width: f32, height: f32) -> BoundingBox {
+    BoundingBox {
+        left: x,
+        top: y,
+        right: x + width,
+        bottom: y + height,
+    }
+}
+
+#[test]
+fn test_iou_calculation() {
+    // Two overlapping boxes
+    let bbox1 = create_bbox(0.0, 0.0, 10.0, 10.0);
+    let bbox2 = create_bbox(5.0, 5.0, 10.0, 10.0);
+
+    // Expected intersection: 5x5 = 25
+    // Expected union: 100 + 100 - 25 = 175
+    // Expected IOU: 25/175 ≈ 0.1429
+    let iou = bbox1.iou(&bbox2);
+    assert!((iou - 0.1429).abs() < 0.001, "IOU calculation failed");
+}
+
+#[test]
+fn test_weighted_distance_calculation() {
+    // Two boxes with different X and Y distances
+    let bbox1 = create_bbox(0.0, 0.0, 10.0, 10.0);
+    let bbox2 = create_bbox(20.0, 5.0, 10.0, 10.0);
+
+    // Distance: X=20, Y=5
+    // Weighted: X*5.0 + Y*1.0 = 20*5.0 + 5*1.0 = 100 + 5 = 105
+    let weighted_dist = bbox1.weighted_distance(&bbox2);
+    assert!(
+        (weighted_dist - 105.0).abs() < 0.001,
+        "Weighted distance calculation failed"
+    );
+
+    // Verify X weight (5.0) > Y weight (1.0) by checking ratio
+    let bbox3 = create_bbox(0.0, 0.0, 10.0, 10.0);
+    let bbox4 = create_bbox(10.0, 0.0, 10.0, 10.0);
+    let only_x_dist = bbox3.weighted_distance(&bbox4);
+
+    let bbox5 = create_bbox(0.0, 0.0, 10.0, 10.0);
+    let bbox6 = create_bbox(0.0, 10.0, 10.0, 10.0);
+    let only_y_dist = bbox5.weighted_distance(&bbox6);
+
+    // X distance of 10 with weight 5.0 = 50
+    // Y distance of 10 with weight 1.0 = 10
+    // X weight should be 5x larger than Y weight
+    assert!(only_x_dist > only_y_dist, "X weight should be greater than Y weight");
+    assert!((only_x_dist - 50.0).abs() < 0.001, "X-only weighted distance failed");
+    assert!((only_y_dist - 10.0).abs() < 0.001, "Y-only weighted distance failed");
+}
+
+#[test]
+fn test_intersection_ratio() {
+    // Two overlapping boxes
+    let bbox1 = create_bbox(0.0, 0.0, 10.0, 10.0);
+    let bbox2 = create_bbox(5.0, 5.0, 10.0, 10.0);
+
+    // Expected intersection: 5x5 = 25
+    // bbox1 area: 100
+    // Expected ratio: 25/100 = 0.25
+    let ratio = bbox1.intersection_ratio(&bbox2);
+    assert!((ratio - 0.25).abs() < 0.001, "Intersection ratio calculation failed");
+}
+
+#[test]
+fn test_edge_case_no_overlap() {
+    // Two non-overlapping boxes
+    let bbox1 = create_bbox(0.0, 0.0, 10.0, 10.0);
+    let bbox2 = create_bbox(20.0, 20.0, 10.0, 10.0);
+
+    // IOU should be 0
+    let iou = bbox1.iou(&bbox2);
+    assert!((iou - 0.0).abs() < 0.001, "Non-overlapping boxes should have IOU of 0");
+
+    // Intersection ratio should be 0
+    let ratio = bbox1.intersection_ratio(&bbox2);
+    assert!(
+        (ratio - 0.0).abs() < 0.001,
+        "Non-overlapping boxes should have intersection ratio of 0"
+    );
+}
+
+#[test]
+fn test_edge_case_fully_contained() {
+    // Smaller box fully contained in larger box
+    let bbox_large = create_bbox(0.0, 0.0, 20.0, 20.0);
+    let bbox_small = create_bbox(5.0, 5.0, 10.0, 10.0);
+
+    // Intersection: 10x10 = 100
+    // Union: 400 + 100 - 100 = 400
+    // IOU: 100/400 = 0.25
+    let iou = bbox_large.iou(&bbox_small);
+    assert!((iou - 0.25).abs() < 0.001, "Fully contained box IOU calculation failed");
+
+    // Intersection ratio: 100/400 = 0.25
+    let ratio = bbox_large.intersection_ratio(&bbox_small);
+    assert!(
+        (ratio - 0.25).abs() < 0.001,
+        "Fully contained box intersection ratio failed"
+    );
+}
+
+// ============================================================================
+// Character Merging Tests (Following TDD)
+// ============================================================================
+
+use kreuzberg::pdf::hierarchy::{CharData, merge_chars_into_blocks};
+
+/// Factory helper to create a CharData with minimal parameters.
+fn create_char(text: &str, x: f32, y: f32, font_size: f32) -> CharData {
+    CharData {
+        text: text.to_string(),
+        font_size,
+        x,
+        y,
+        width: font_size * 0.6,
+        height: font_size,
+    }
+}
+
+/// Test horizontal text merging: characters at (0,10), (10,10), (20,10) should merge into 1 block.
+///
+/// This test validates that characters on the same horizontal line and within the merge
+/// distance threshold are grouped into a single text block.
+#[test]
+fn test_merge_horizontal_text_merging() {
+    let chars = vec![
+        create_char("H", 0.0, 10.0, 12.0),
+        create_char("e", 10.0, 10.0, 12.0),
+        create_char("y", 20.0, 10.0, 12.0),
+    ];
+
+    let blocks = merge_chars_into_blocks(chars);
+
+    assert_eq!(blocks.len(), 1, "Expected 1 block for horizontal text");
+    assert_eq!(blocks[0].text, "Hey", "Expected merged text 'Hey'");
+}
+
+/// Test vertical text separation: characters at (0,10) and (0,50) should be 2 blocks.
+///
+/// This test validates that characters with large vertical separation are kept in separate blocks.
+#[test]
+fn test_merge_vertical_text_separation() {
+    let chars = vec![create_char("A", 0.0, 10.0, 12.0), create_char("B", 0.0, 50.0, 12.0)];
+
+    let blocks = merge_chars_into_blocks(chars);
+
+    assert_eq!(blocks.len(), 2, "Expected 2 blocks for vertically separated text");
+    assert_eq!(blocks[0].text, "A", "Expected first block to contain 'A'");
+    assert_eq!(blocks[1].text, "B", "Expected second block to contain 'B'");
+}
+
+/// Test edge case with negative coordinates.
+///
+/// This test ensures the merging algorithm handles negative coordinates correctly.
+#[test]
+fn test_merge_edge_case_negative_coordinates() {
+    let chars = vec![
+        create_char("X", -10.0, -5.0, 12.0),
+        create_char("Y", 0.0, -5.0, 12.0),
+        create_char("Z", 10.0, -5.0, 12.0),
+    ];
+
+    let blocks = merge_chars_into_blocks(chars);
+
+    assert_eq!(blocks.len(), 1, "Expected 1 block for negative coordinates");
+    assert_eq!(blocks[0].text, "XYZ", "Expected merged text 'XYZ'");
+}
+
+/// Test edge case with overlapping blocks.
+///
+/// This test validates behavior when characters have overlapping or very close bounding boxes.
+#[test]
+fn test_merge_edge_case_overlapping_blocks() {
+    let chars = vec![
+        create_char("O", 0.0, 0.0, 12.0),
+        create_char("V", 1.0, 0.0, 12.0),
+        create_char("E", 2.0, 0.0, 12.0),
+    ];
+
+    let blocks = merge_chars_into_blocks(chars);
+
+    assert_eq!(blocks.len(), 1, "Expected 1 block for overlapping characters");
+    assert_eq!(blocks[0].text, "OVE", "Expected merged text 'OVE'");
+}
+
+/// Test max merge distance threshold.
+///
+/// This test validates that characters beyond the maximum merge distance are kept separate.
+#[test]
+fn test_merge_max_merge_distance_threshold() {
+    let chars = vec![
+        create_char("T", 0.0, 10.0, 12.0),
+        create_char("e", 50.0, 10.0, 12.0),  // Large gap, should be separate
+        create_char("s", 100.0, 10.0, 12.0), // Even larger gap
+    ];
+
+    let blocks = merge_chars_into_blocks(chars);
+
+    // With reasonable merge distance (should be ~2.5x font size for distance),
+    // characters at 50 and 100 units apart should create separate blocks
+    assert!(blocks.len() > 1, "Expected multiple blocks due to large gaps");
+    assert_eq!(
+        blocks.iter().map(|b| b.text.len()).sum::<usize>(),
+        3,
+        "Expected all 3 characters to be preserved"
+    );
+}
+
+/// Test edge case with zero font size.
+///
+/// This test ensures the algorithm handles zero or near-zero font sizes gracefully
+/// without panicking or causing division by zero.
+#[test]
+fn test_merge_zero_font_size() {
+    // Create characters with zero font size (edge case)
+    let chars = vec![
+        CharData {
+            text: "A".to_string(),
+            x: 0.0,
+            y: 10.0,
+            font_size: 0.0,
+            width: 0.0,
+            height: 0.0,
+        },
+        CharData {
+            text: "B".to_string(),
+            x: 1.0,
+            y: 10.0,
+            font_size: 0.0,
+            width: 0.0,
+            height: 0.0,
+        },
+    ];
+
+    // Should not panic
+    let blocks = merge_chars_into_blocks(chars);
+
+    // Should still produce some output
+    assert!(!blocks.is_empty(), "Should produce blocks even with zero font size");
+}
+
+/// Test IOU calculation with zero area boxes.
+///
+/// This test validates that IOU calculations handle degenerate bounding boxes
+/// with zero area without panicking.
+#[test]
+fn test_iou_zero_area_boxes() {
+    // Two boxes with zero area (point-like)
+    let bbox1 = create_bbox(0.0, 0.0, 0.0, 0.0);
+    let bbox2 = create_bbox(5.0, 5.0, 0.0, 0.0);
+
+    // Should not panic and should return 0 or similar
+    let iou = bbox1.iou(&bbox2);
+    assert!(
+        (0.0..=1.0).contains(&iou),
+        "IOU should be in valid range for zero-area boxes"
+    );
+
+    // Intersection ratio should also be safe
+    let ratio = bbox1.intersection_ratio(&bbox2);
+    assert!(
+        (0.0..=1.0).contains(&ratio),
+        "Intersection ratio should be in valid range"
+    );
+}
+
+/// Test IOU calculation with identical boxes.
+///
+/// This test validates that identical boxes have IOU of 1.0 (perfect overlap).
+#[test]
+fn test_iou_identical_boxes() {
+    // Two identical boxes
+    let bbox1 = create_bbox(10.0, 20.0, 30.0, 40.0);
+    let bbox2 = create_bbox(10.0, 20.0, 30.0, 40.0);
+
+    // IOU should be 1.0 for identical boxes
+    let iou = bbox1.iou(&bbox2);
+    assert!(
+        (iou - 1.0).abs() < 0.001,
+        "Identical boxes should have IOU of 1.0, got {}",
+        iou
+    );
+
+    // Intersection ratio should also be 1.0
+    let ratio = bbox1.intersection_ratio(&bbox2);
+    assert!(
+        (ratio - 1.0).abs() < 0.001,
+        "Identical boxes should have intersection ratio of 1.0, got {}",
+        ratio
+    );
+}
+
+/// Test contains() method with various box configurations.
+#[test]
+fn test_contains_method() {
+    let large_box = create_bbox(0.0, 0.0, 100.0, 100.0);
+    let small_box = create_bbox(10.0, 10.0, 50.0, 50.0);
+    let outside_box = create_bbox(110.0, 110.0, 150.0, 150.0);
+
+    // Small box should be contained in large box
+    assert!(large_box.contains(&small_box), "Large box should contain small box");
+
+    // Large box should not be contained in small box
+    assert!(
+        !small_box.contains(&large_box),
+        "Small box should not contain large box"
+    );
+
+    // Outside box should not be contained
+    assert!(
+        !large_box.contains(&outside_box),
+        "Large box should not contain outside box"
+    );
+}
+
+/// Test center() method for bounding box centerpoint calculation.
+#[test]
+fn test_center_method() {
+    let bbox = create_bbox(0.0, 0.0, 100.0, 100.0);
+    let center = bbox.center();
+
+    assert_eq!(center.0, 50.0, "Center X should be 50.0");
+    assert_eq!(center.1, 50.0, "Center Y should be 50.0");
+
+    // Test with offset box
+    let offset_bbox = create_bbox(20.0, 30.0, 80.0, 70.0);
+    let offset_center = offset_bbox.center();
+
+    assert_eq!(offset_center.0, 60.0, "Offset center X should be 60.0");
+    assert_eq!(offset_center.1, 65.0, "Offset center Y should be 65.0");
+}
+
+/// Test merge() method for combining bounding boxes.
+#[test]
+fn test_merge_method() {
+    let bbox1 = create_bbox(0.0, 0.0, 50.0, 50.0);
+    let bbox2 = create_bbox(30.0, 30.0, 100.0, 100.0);
+
+    let merged = bbox1.merge(&bbox2);
+
+    assert_eq!(merged.left, 0.0, "Merged left should be 0.0");
+    assert_eq!(merged.top, 0.0, "Merged top should be 0.0");
+    assert_eq!(merged.right, 130.0, "Merged right should be 130.0");
+    assert_eq!(merged.bottom, 130.0, "Merged bottom should be 130.0");
+}
+
+/// Test relaxed_iou() method with expansion factor.
+#[test]
+fn test_relaxed_iou_method() {
+    let bbox1 = create_bbox(0.0, 0.0, 10.0, 10.0);
+    let bbox2 = create_bbox(15.0, 15.0, 25.0, 25.0);
+
+    // Without relaxation, IOU should be 0
+    let normal_iou = bbox1.iou(&bbox2);
+    assert!(normal_iou < 0.01, "Non-overlapping boxes should have near-zero IOU");
+
+    // With relaxation, IOU should increase
+    let relaxed_iou = bbox1.relaxed_iou(&bbox2, 0.5);
+    assert!(
+        relaxed_iou > normal_iou,
+        "Relaxed IOU should be greater than normal IOU"
+    );
+}