html-to-markdown 2.30.0 → 3.0.1

data/vendor/html-to-markdown-rs/src/hocr/spatial/layout.rs

@@ -1,335 +0,0 @@
-#![allow(clippy::cast_precision_loss, clippy::cast_sign_loss, clippy::unused_self)]
-//! Layout analysis and table reconstruction
-
-use crate::hocr::spatial::coords::HocrWord;
-
-/// Detect column positions from word positions
-///
-/// Groups words by their x-position and returns the median x-position
-/// for each detected column.
-///
-/// Optimized with O(n log n) complexity using sorted insertion.
-#[must_use]
-pub fn detect_columns(words: &[HocrWord], column_threshold: u32) -> Vec<u32> {
-    if words.is_empty() {
-        return Vec::new();
-    }
-
-    let mut x_positions: Vec<u32> = words.iter().map(|w| w.left).collect();
-    x_positions.sort_unstable();
-
-    let mut position_groups: Vec<Vec<u32>> = Vec::new();
-    let mut current_group = vec![x_positions[0]];
-
-    for &x_pos in &x_positions[1..] {
-        let matches_group = current_group.iter().any(|&pos| x_pos.abs_diff(pos) <= column_threshold);
-
-        if matches_group {
-            current_group.push(x_pos);
-        } else {
-            position_groups.push(std::mem::replace(&mut current_group, vec![x_pos]));
-        }
-    }
-
-    if !current_group.is_empty() {
-        position_groups.push(current_group);
-    }
-
-    let mut columns: Vec<u32> = position_groups
-        .iter()
-        .map(|group| {
-            let mid = group.len() / 2;
-            group[mid]
-        })
-        .collect();
-
-    columns.sort_unstable();
-    columns
-}
-
-/// Detect row positions from word positions
-///
-/// Groups words by their vertical center position and returns the median
-/// y-position for each detected row.
-///
-/// Optimized with O(n log n) complexity using sorted insertion.
-#[must_use]
-#[allow(clippy::cast_possible_truncation)]
-pub fn detect_rows(words: &[HocrWord], row_threshold_ratio: f64) -> Vec<u32> {
-    if words.is_empty() {
-        return Vec::new();
-    }
-
-    let mut heights: Vec<u32> = words.iter().map(|w| w.height).collect();
-    heights.sort_unstable();
-    let median_height = heights[heights.len() / 2];
-    let row_threshold = f64::from(median_height) * row_threshold_ratio;
-
-    let mut y_centers: Vec<f64> = words.iter().map(HocrWord::y_center).collect();
-    y_centers.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
-
-    let mut position_groups: Vec<Vec<f64>> = Vec::new();
-    let mut current_group = vec![y_centers[0]];
-
-    for &y_center in &y_centers[1..] {
-        let matches_group = current_group.iter().any(|&pos| (y_center - pos).abs() <= row_threshold);
-
-        if matches_group {
-            current_group.push(y_center);
-        } else {
-            position_groups.push(std::mem::replace(&mut current_group, vec![y_center]));
-        }
-    }
-
-    if !current_group.is_empty() {
-        position_groups.push(current_group);
-    }
-
-    let mut rows: Vec<u32> = position_groups
-        .iter()
-        .map(|group| {
-            let mid = group.len() / 2;
-            group[mid] as u32
-        })
-        .collect();
-
-    rows.sort_unstable();
-    rows
-}
-
-/// Remove empty rows and columns from table
-fn remove_empty_rows_and_columns(table: Vec<Vec<String>>) -> Vec<Vec<String>> {
-    if table.is_empty() {
-        return table;
-    }
-
-    let num_cols = table[0].len();
-    let mut non_empty_cols: Vec<bool> = vec![false; num_cols];
-
-    for row in &table {
-        for (col_idx, cell) in row.iter().enumerate() {
-            if !cell.trim().is_empty() {
-                non_empty_cols[col_idx] = true;
-            }
-        }
-    }
-
-    table
-        .into_iter()
-        .filter(|row| row.iter().any(|cell| !cell.trim().is_empty()))
-        .map(|row| {
-            row.into_iter()
-                .enumerate()
-                .filter(|(idx, _)| non_empty_cols[*idx])
-                .map(|(_, cell)| cell)
-                .collect()
-        })
-        .collect()
-}
-
-/// Find which row a word belongs to based on its y-center
-#[allow(clippy::cast_possible_truncation)]
-fn find_row_index(row_positions: &[u32], word: &HocrWord) -> Option<usize> {
-    let y_center = word.y_center() as u32;
-
-    row_positions
-        .iter()
-        .enumerate()
-        .min_by_key(|&(_, row_y)| row_y.abs_diff(y_center))
-        .map(|(idx, _)| idx)
-}
-
-/// Find which column a word belongs to based on its x-position
-fn find_column_index(col_positions: &[u32], word: &HocrWord) -> Option<usize> {
-    let x_pos = word.left;
-
-    col_positions
-        .iter()
-        .enumerate()
-        .min_by_key(|&(_, col_x)| col_x.abs_diff(x_pos))
-        .map(|(idx, _)| idx)
-}
-
-/// Reconstruct table structure from words
-///
-/// Takes detected words and reconstructs a 2D table by:
-/// 1. Detecting column and row positions
-/// 2. Assigning words to cells based on position
-/// 3. Combining words within the same cell
-#[must_use]
-pub fn reconstruct_table(words: &[HocrWord], column_threshold: u32, row_threshold_ratio: f64) -> Vec<Vec<String>> {
-    if words.is_empty() {
-        return Vec::new();
-    }
-
-    let col_positions = detect_columns(words, column_threshold);
-    let row_positions = detect_rows(words, row_threshold_ratio);
-
-    if col_positions.is_empty() || row_positions.is_empty() {
-        return Vec::new();
-    }
-
-    let num_rows = row_positions.len();
-    let num_cols = col_positions.len();
-    let mut table: Vec<Vec<Vec<String>>> = vec![vec![vec![]; num_cols]; num_rows];
-
-    for word in words {
-        if let (Some(r), Some(c)) = (
-            find_row_index(&row_positions, word),
-            find_column_index(&col_positions, word),
-        ) {
-            if r < num_rows && c < num_cols {
-                table[r][c].push(word.text.clone());
-            }
-        }
-    }
-
-    let result: Vec<Vec<String>> = table
-        .into_iter()
-        .map(|row| {
-            row.into_iter()
-                .map(|cell_words| {
-                    if cell_words.is_empty() {
-                        String::new()
-                    } else {
-                        cell_words.join(" ")
-                    }
-                })
-                .collect()
-        })
-        .collect();
-
-    remove_empty_rows_and_columns(result)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_detect_columns() {
-        let words = vec![
-            HocrWord {
-                text: "A".to_string(),
-                left: 100,
-                top: 50,
-                width: 20,
-                height: 30,
-                confidence: 95.0,
-            },
-            HocrWord {
-                text: "B".to_string(),
-                left: 200,
-                top: 50,
-                width: 20,
-                height: 30,
-                confidence: 95.0,
-            },
-            HocrWord {
-                text: "C".to_string(),
-                left: 105,
-                top: 100,
-                width: 20,
-                height: 30,
-                confidence: 95.0,
-            },
-        ];
-
-        let columns = detect_columns(&words, 50);
-        assert_eq!(columns.len(), 2);
-        assert!(columns.contains(&100) || columns.contains(&105));
-        assert!(columns.contains(&200));
-    }
-
-    #[test]
-    fn test_reconstruct_simple_table() {
-        let words = vec![
-            HocrWord {
-                text: "Name".to_string(),
-                left: 100,
-                top: 50,
-                width: 50,
-                height: 20,
-                confidence: 95.0,
-            },
-            HocrWord {
-                text: "Age".to_string(),
-                left: 200,
-                top: 50,
-                width: 50,
-                height: 20,
-                confidence: 95.0,
-            },
-            HocrWord {
-                text: "Alice".to_string(),
-                left: 100,
-                top: 100,
-                width: 50,
-                height: 20,
-                confidence: 95.0,
-            },
-            HocrWord {
-                text: "30".to_string(),
-                left: 200,
-                top: 100,
-                width: 50,
-                height: 20,
-                confidence: 95.0,
-            },
-        ];
-
-        let table = reconstruct_table(&words, 50, 0.5);
-
-        assert_eq!(table.len(), 2);
-        assert_eq!(table[0].len(), 2);
-        assert_eq!(table[0][0], "Name");
-        assert_eq!(table[0][1], "Age");
-        assert_eq!(table[1][0], "Alice");
-        assert_eq!(table[1][1], "30");
-    }
-
-    #[test]
-    fn test_reconstruct_table_with_multi_word_cells() {
-        let words = vec![
-            HocrWord {
-                text: "First".to_string(),
-                left: 100,
-                top: 50,
-                width: 30,
-                height: 20,
-                confidence: 95.0,
-            },
-            HocrWord {
-                text: "Name".to_string(),
-                left: 135,
-                top: 50,
-                width: 30,
-                height: 20,
-                confidence: 95.0,
-            },
-            HocrWord {
-                text: "Last".to_string(),
-                left: 200,
-                top: 50,
-                width: 30,
-                height: 20,
-                confidence: 95.0,
-            },
-            HocrWord {
-                text: "Name".to_string(),
-                left: 235,
-                top: 50,
-                width: 30,
-                height: 20,
-                confidence: 95.0,
-            },
-        ];
-
-        let table = reconstruct_table(&words, 50, 0.5);
-
-        assert_eq!(table.len(), 1);
-        assert_eq!(table[0].len(), 2);
-        assert_eq!(table[0][0], "First Name");
-        assert_eq!(table[0][1], "Last Name");
-    }
-}

data/vendor/html-to-markdown-rs/src/hocr/spatial/mod.rs

@@ -1,15 +0,0 @@
-#![allow(clippy::cast_precision_loss, clippy::cast_sign_loss, clippy::unused_self)]
-//! Spatial table reconstruction from hOCR bounding box coordinates
-//!
-//! This module provides functions to detect and reconstruct tabular data from OCR'd text
-//! by analyzing the spatial positions of words using their bounding box (bbox) coordinates.
-
-mod coords;
-mod grouping;
-mod layout;
-mod output;
-
-pub use coords::{HocrWord, parse_bbox, parse_confidence};
-pub use grouping::extract_hocr_words;
-pub use layout::reconstruct_table;
-pub use output::table_to_markdown;

data/vendor/html-to-markdown-rs/src/hocr/spatial/output.rs

@@ -1,63 +0,0 @@
-#![allow(clippy::cast_precision_loss, clippy::cast_sign_loss, clippy::unused_self)]
-//! Markdown table output formatting
-
-/// Convert table to markdown format
-#[must_use]
-pub fn table_to_markdown(table: &[Vec<String>]) -> String {
-    if table.is_empty() {
-        return String::new();
-    }
-
-    let num_cols = table[0].len();
-    if num_cols == 0 {
-        return String::new();
-    }
-
-    let mut markdown = String::new();
-
-    for (row_idx, row) in table.iter().enumerate() {
-        markdown.push('|');
-        for cell in row {
-            markdown.push(' ');
-            markdown.push_str(&cell.replace('|', "\\|"));
-            markdown.push_str(" |");
-        }
-        markdown.push('\n');
-
-        if row_idx == 0 {
-            markdown.push('|');
-            for _ in 0..num_cols {
-                markdown.push_str(" --- |");
-            }
-            markdown.push('\n');
-        }
-    }
-
-    markdown
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_table_to_markdown() {
-        let table = vec![
-            vec!["Header1".to_string(), "Header2".to_string()],
-            vec!["Cell1".to_string(), "Cell2".to_string()],
-        ];
-
-        let markdown = table_to_markdown(&table);
-        assert!(markdown.contains("| Header1 | Header2 |"));
-        assert!(markdown.contains("| --- | --- |"));
-        assert!(markdown.contains("| Cell1 | Cell2 |"));
-    }
-
-    #[test]
-    fn test_table_to_markdown_escape_pipes() {
-        let table = vec![vec!["A|B".to_string(), "C".to_string()]];
-
-        let markdown = table_to_markdown(&table);
-        assert!(markdown.contains("A\\|B"));
-    }
-}

data/vendor/html-to-markdown-rs/src/hocr/types.rs

@@ -1,269 +0,0 @@
-#![allow(clippy::cast_precision_loss, clippy::cast_sign_loss, clippy::unused_self)]
-//! hOCR 1.2 type definitions
-//!
-//! Complete type system for hOCR 1.2 specification elements and properties.
-
-use ahash::AHashMap as HashMap;
-
-/// All hOCR 1.2 element types
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum HocrElementType {
-    /// Document abstract or summary
-    OcrAbstract,
-    /// Author attribution
-    OcrAuthor,
-    /// Block quotation
-    OcrBlockquote,
-    /// Image caption
-    OcrCaption,
-    /// Chapter division
-    OcrChapter,
-    /// Document root element
-    OcrDocument,
-    /// Paragraph text
-    OcrPar,
-    /// Major part or section
-    OcrPart,
-    /// Section heading
-    OcrSection,
-    /// Subsection heading
-    OcrSubsection,
-    /// Subsubsection heading
-    OcrSubsubsection,
-    /// Document title
-    OcrTitle,
-
-    /// Column area of multi-column layout
-    OcrCarea,
-    /// Column within a page
-    OcrColumn,
-    /// Text line
-    OcrLine,
-    /// Linearization element
-    OcrLinear,
-    /// Page element
-    OcrPage,
-    /// Separator or divider
-    OcrSeparator,
-
-    /// Chemical formula
-    OcrChem,
-    /// Display equation or complex content
-    OcrDisplay,
-    /// Float element (typically with caption)
-    OcrFloat,
-    /// Footer area of page
-    OcrFooter,
-    /// Header area of page
-    OcrHeader,
-    /// Image element
-    OcrImage,
-    /// Line drawing or diagram
-    OcrLinedrawing,
-    /// Mathematical formula
-    OcrMath,
-    /// Page number marker
-    OcrPageno,
-    /// Photograph element
-    OcrPhoto,
-    /// Table element
-    OcrTable,
-    /// Text float (floating text box)
-    OcrTextfloat,
-    /// Text image (text rendered as image)
-    OcrTextimage,
-
-    /// Character information
-    OcrCinfo,
-    /// Decorative capital letter
-    OcrDropcap,
-    /// Glyph element
-    OcrGlyph,
-    /// Multiple glyphs
-    OcrGlyphs,
-    /// Noise or artifacts
-    OcrNoise,
-    /// `XyZut` analysis segment
-    OcrXycut,
-
-    /// Block-level element
-    OcrxBlock,
-    /// OCR word line
-    OcrxLine,
-    /// Individual word element
-    OcrxWord,
-}
-
-impl HocrElementType {
-    /// Get element type from class name
-    #[must_use]
-    pub fn from_class(class: &str) -> Option<Self> {
-        match class {
-            "ocr_abstract" => Some(Self::OcrAbstract),
-            "ocr_author" => Some(Self::OcrAuthor),
-            "ocr_blockquote" => Some(Self::OcrBlockquote),
-            "ocr_caption" => Some(Self::OcrCaption),
-            "ocr_chapter" => Some(Self::OcrChapter),
-            "ocr_document" => Some(Self::OcrDocument),
-            "ocr_par" => Some(Self::OcrPar),
-            "ocr_part" => Some(Self::OcrPart),
-            "ocr_section" => Some(Self::OcrSection),
-            "ocr_subsection" => Some(Self::OcrSubsection),
-            "ocr_subsubsection" => Some(Self::OcrSubsubsection),
-            "ocr_title" => Some(Self::OcrTitle),
-
-            "ocr_carea" => Some(Self::OcrCarea),
-            "ocr_column" => Some(Self::OcrColumn),
-            "ocr_line" => Some(Self::OcrLine),
-            "ocr_linear" => Some(Self::OcrLinear),
-            "ocr_page" => Some(Self::OcrPage),
-            "ocr_separator" => Some(Self::OcrSeparator),
-
-            "ocr_chem" => Some(Self::OcrChem),
-            "ocr_display" => Some(Self::OcrDisplay),
-            "ocr_float" => Some(Self::OcrFloat),
-            "ocr_footer" => Some(Self::OcrFooter),
-            "ocr_header" => Some(Self::OcrHeader),
-            "ocr_image" => Some(Self::OcrImage),
-            "ocr_linedrawing" => Some(Self::OcrLinedrawing),
-            "ocr_math" => Some(Self::OcrMath),
-            "ocr_pageno" => Some(Self::OcrPageno),
-            "ocr_photo" => Some(Self::OcrPhoto),
-            "ocr_table" => Some(Self::OcrTable),
-            "ocr_textfloat" => Some(Self::OcrTextfloat),
-            "ocr_textimage" => Some(Self::OcrTextimage),
-
-            "ocr_cinfo" => Some(Self::OcrCinfo),
-            "ocr_dropcap" => Some(Self::OcrDropcap),
-            "ocr_glyph" => Some(Self::OcrGlyph),
-            "ocr_glyphs" => Some(Self::OcrGlyphs),
-            "ocr_noise" => Some(Self::OcrNoise),
-            "ocr_xycut" => Some(Self::OcrXycut),
-
-            "ocrx_block" => Some(Self::OcrxBlock),
-            "ocrx_line" => Some(Self::OcrxLine),
-            "ocrx_word" => Some(Self::OcrxWord),
-
-            _ => None,
-        }
-    }
-}
-
-/// Bounding box with corner coordinates
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct BBox {
-    /// Left edge x-coordinate (pixels)
-    pub x1: u32,
-    /// Top edge y-coordinate (pixels)
-    pub y1: u32,
-    /// Right edge x-coordinate (pixels)
-    pub x2: u32,
-    /// Bottom edge y-coordinate (pixels)
-    pub y2: u32,
-}
-
-impl BBox {
-    /// Calculate the width from left to right edge
-    #[must_use]
-    pub const fn width(&self) -> u32 {
-        self.x2.saturating_sub(self.x1)
-    }
-
-    /// Calculate the height from top to bottom edge
-    #[must_use]
-    pub const fn height(&self) -> u32 {
-        self.y2.saturating_sub(self.y1)
-    }
-}
-
-/// Baseline property for text alignment in OCR elements
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub struct Baseline {
-    /// Baseline slope relative to horizontal
-    pub slope: f64,
-    /// Baseline vertical offset in pixels
-    pub constant: i32,
-}
-
-/// All hOCR properties extracted from element attributes
-#[derive(Debug, Clone, Default)]
-pub struct HocrProperties {
-    /// Bounding box (left, top, right, bottom) coordinates
-    pub bbox: Option<BBox>,
-    /// Baseline properties (slope, constant offset)
-    pub baseline: Option<Baseline>,
-    /// Text rotation angle in degrees
-    pub textangle: Option<f64>,
-    /// Polygon coordinates for non-rectangular regions
-    pub poly: Option<Vec<(i32, i32)>>,
-
-    /// Word-level confidence score (0-100)
-    pub x_wconf: Option<f64>,
-    /// Per-character confidence scores
-    pub x_confs: Vec<f64>,
-    /// Natural language processing results
-    pub nlp: Vec<f64>,
-
-    /// Font name or family
-    pub x_font: Option<String>,
-    /// Font size in points
-    pub x_fsize: Option<u32>,
-
-    /// Reading order index for document structure
-    pub order: Option<u32>,
-    /// Column flow direction (ltr, rtl, etc.)
-    pub cflow: Option<String>,
-    /// Hard line break indicator
-    pub hardbreak: bool,
-
-    /// Cut lines for layout analysis
-    pub cuts: Vec<Vec<u32>>,
-    /// Alternative bounding boxes for multi-part elements
-    pub x_bboxes: Vec<BBox>,
-
-    /// Image path or data URI
-    pub image: Option<String>,
-    /// MD5 hash of image content
-    pub imagemd5: Option<String>,
-    /// Physical page number
-    pub ppageno: Option<u32>,
-    /// Logical page number/label
-    pub lpageno: Option<String>,
-    /// Scanner resolution (`dpi_x`, `dpi_y`)
-    pub scan_res: Option<(u32, u32)>,
-    /// Image source file paths
-    pub x_source: Vec<String>,
-    /// Scanner device identifier
-    pub x_scanner: Option<String>,
-
-    /// Additional custom properties
-    pub other: HashMap<String, String>,
-}
-
-/// A complete hOCR element with type, properties, text content, and child elements
-#[derive(Debug, Clone)]
-pub struct HocrElement {
-    /// The semantic type of this hOCR element
-    pub element_type: HocrElementType,
-    /// All extracted properties (bbox, confidence, etc.)
-    pub properties: HocrProperties,
-    /// Text content of this element
-    pub text: String,
-    /// Child elements in the document tree
-    pub children: Vec<Self>,
-}
-
-/// hOCR document metadata extracted from document properties
-#[derive(Debug, Clone, Default)]
-pub struct HocrMetadata {
-    /// Name and version of the OCR system used
-    pub ocr_system: Option<String>,
-    /// OCR capabilities supported (e.g., "`ocr_page`", "`ocr_carea`")
-    pub ocr_capabilities: Vec<String>,
-    /// Total number of pages in the OCR'd document
-    pub ocr_number_of_pages: Option<u32>,
-    /// Languages detected in the document (ISO 639 codes)
-    pub ocr_langs: Vec<String>,
-    /// Scripts used in the document (ISO 15924 codes)
-    pub ocr_scripts: Vec<String>,
-}