html-to-markdown 2.30.0 → 3.0.1

data/vendor/html-to-markdown-rs/src/hocr/parser.rs

@@ -1,333 +0,0 @@
-#![allow(clippy::cast_precision_loss, clippy::cast_sign_loss, clippy::unused_self)]
-//! hOCR property parser
-//!
-//! Parses hOCR title attributes into structured properties.
-
-use super::types::{BBox, Baseline, HocrProperties};
-use crate::text::decode_html_entities_cow;
-
-/// Parse all properties from hOCR title attribute
-#[must_use]
-pub fn parse_properties(title: &str) -> HocrProperties {
-    let mut props = HocrProperties::default();
-
-    let title = decode_html_entities_cow(title);
-
-    for part in title.as_ref().split(';') {
-        let part = part.trim();
-        if part.is_empty() {
-            continue;
-        }
-
-        let mut tokens = part.split_whitespace();
-        if let Some(key) = tokens.next() {
-            match key {
-                "bbox" => {
-                    if let Some(bbox) = parse_bbox_coords(&mut tokens) {
-                        props.bbox = Some(bbox);
-                    }
-                }
-                "baseline" => {
-                    if let Some(baseline) = parse_baseline(&mut tokens) {
-                        props.baseline = Some(baseline);
-                    }
-                }
-                "textangle" => {
-                    if let Some(angle_str) = tokens.next() {
-                        if let Ok(angle) = angle_str.parse::<f64>() {
-                            props.textangle = Some(angle);
-                        }
-                    }
-                }
-                "poly" => {
-                    props.poly = parse_poly(&mut tokens);
-                }
-                "x_wconf" => {
-                    if let Some(conf_str) = tokens.next() {
-                        if let Ok(conf) = conf_str.parse::<f64>() {
-                            props.x_wconf = Some(conf);
-                        }
-                    }
-                }
-                "x_confs" => {
-                    props.x_confs = parse_float_list(&mut tokens);
-                }
-                "nlp" => {
-                    props.nlp = parse_float_list(&mut tokens);
-                }
-                "x_font" => {
-                    if let Some(font) = parse_quoted_string(part) {
-                        props.x_font = Some(font);
-                    }
-                }
-                "x_fsize" => {
-                    if let Some(size_str) = tokens.next() {
-                        if let Ok(size) = size_str.parse::<u32>() {
-                            props.x_fsize = Some(size);
-                        }
-                    }
-                }
-                "order" => {
-                    if let Some(order_str) = tokens.next() {
-                        if let Ok(order) = order_str.parse::<u32>() {
-                            props.order = Some(order);
-                        }
-                    }
-                }
-                "cflow" => {
-                    if let Some(flow) = parse_quoted_string(part) {
-                        props.cflow = Some(flow);
-                    }
-                }
-                "hardbreak" => {
-                    if let Some(val) = tokens.next() {
-                        props.hardbreak = val == "1";
-                    }
-                }
-                "cuts" => {
-                    props.cuts = parse_cuts(&mut tokens);
-                }
-                "x_bboxes" => {
-                    props.x_bboxes = parse_bboxes_list(&mut tokens);
-                }
-                "image" => {
-                    if let Some(img) = parse_quoted_string(part) {
-                        props.image = Some(img);
-                    }
-                }
-                "imagemd5" => {
-                    if let Some(md5) = parse_quoted_string(part) {
-                        props.imagemd5 = Some(md5);
-                    }
-                }
-                "ppageno" => {
-                    if let Some(page_str) = tokens.next() {
-                        if let Ok(page) = page_str.parse::<u32>() {
-                            props.ppageno = Some(page);
-                        }
-                    }
-                }
-                "lpageno" => {
-                    let rest: Vec<&str> = tokens.collect();
-                    if !rest.is_empty() {
-                        let lpageno_str = rest.join(" ");
-                        if let Some(quoted) = parse_quoted_string(part) {
-                            props.lpageno = Some(quoted);
-                        } else {
-                            props.lpageno = Some(lpageno_str);
-                        }
-                    }
-                }
-                "scan_res" => {
-                    let coords: Vec<&str> = tokens.collect();
-                    if coords.len() >= 2 {
-                        if let (Ok(x), Ok(y)) = (coords[0].parse::<u32>(), coords[1].parse::<u32>()) {
-                            props.scan_res = Some((x, y));
-                        }
-                    }
-                }
-                "x_source" => {
-                    let sources = parse_all_quoted_strings(part);
-                    if !sources.is_empty() {
-                        props.x_source = sources;
-                    }
-                }
-                "x_scanner" => {
-                    if let Some(scanner) = parse_quoted_string(part) {
-                        props.x_scanner = Some(scanner);
-                    }
-                }
-                "x_size" | "x_descenders" | "x_ascenders" => {
-                    let value: Vec<&str> = tokens.collect();
-                    if !value.is_empty() {
-                        props.other.insert(key.to_string(), value.join(" "));
-                    }
-                }
-                _ => {
-                    let value: Vec<&str> = tokens.collect();
-                    if !value.is_empty() {
-                        props.other.insert(key.to_string(), value.join(" "));
-                    }
-                }
-            }
-        }
-    }
-
-    props
-}
-
-fn parse_bbox_coords<'a, I>(tokens: &mut I) -> Option<BBox>
-where
-    I: Iterator<Item = &'a str>,
-{
-    let x1 = tokens.next()?.parse::<u32>().ok()?;
-    let y1 = tokens.next()?.parse::<u32>().ok()?;
-    let x2 = tokens.next()?.parse::<u32>().ok()?;
-    let y2 = tokens.next()?.parse::<u32>().ok()?;
-    Some(BBox { x1, y1, x2, y2 })
-}
-
-fn parse_baseline<'a, I>(tokens: &mut I) -> Option<Baseline>
-where
-    I: Iterator<Item = &'a str>,
-{
-    let slope = tokens.next()?.parse::<f64>().ok()?;
-    let constant = tokens.next()?.parse::<i32>().ok()?;
-    Some(Baseline { slope, constant })
-}
-
-fn parse_poly<'a, I>(tokens: &mut I) -> Option<Vec<(i32, i32)>>
-where
-    I: Iterator<Item = &'a str>,
-{
-    let coords: Vec<&str> = tokens.collect();
-    if coords.len() >= 4 && coords.len() % 2 == 0 {
-        let mut points = Vec::new();
-        for chunk in coords.chunks(2) {
-            if let (Ok(x), Ok(y)) = (chunk[0].parse::<i32>(), chunk[1].parse::<i32>()) {
-                points.push((x, y));
-            } else {
-                return None;
-            }
-        }
-        return Some(points);
-    }
-    None
-}
-
-fn parse_float_list<'a, I>(tokens: &mut I) -> Vec<f64>
-where
-    I: Iterator<Item = &'a str>,
-{
-    tokens.filter_map(|s| s.parse::<f64>().ok()).collect()
-}
-
-fn parse_cuts<'a, I>(tokens: &mut I) -> Vec<Vec<u32>>
-where
-    I: Iterator<Item = &'a str>,
-{
-    let mut cuts = Vec::new();
-    for token in tokens {
-        if token.contains(',') {
-            let parts: Vec<u32> = token.split(',').filter_map(|s| s.parse::<u32>().ok()).collect();
-            cuts.push(parts);
-        } else if let Ok(val) = token.parse::<u32>() {
-            cuts.push(vec![val]);
-        }
-    }
-    cuts
-}
-
-fn parse_bboxes_list<'a, I>(tokens: &mut I) -> Vec<BBox>
-where
-    I: Iterator<Item = &'a str>,
-{
-    let coords: Vec<u32> = tokens.filter_map(|s| s.parse::<u32>().ok()).collect();
-
-    coords
-        .chunks(4)
-        .filter_map(|chunk| {
-            if chunk.len() == 4 {
-                Some(BBox {
-                    x1: chunk[0],
-                    y1: chunk[1],
-                    x2: chunk[2],
-                    y2: chunk[3],
-                })
-            } else {
-                None
-            }
-        })
-        .collect()
-}
-
-fn parse_quoted_string(s: &str) -> Option<String> {
-    if let Some(start) = s.find('"') {
-        if let Some(end) = s[start + 1..].find('"') {
-            return Some(s[start + 1..start + 1 + end].to_string());
-        }
-    }
-    None
-}
-
-fn parse_all_quoted_strings(s: &str) -> Vec<String> {
-    let mut results = Vec::new();
-    let mut remaining = s;
-
-    while let Some(start) = remaining.find('"') {
-        if let Some(end) = remaining[start + 1..].find('"') {
-            results.push(remaining[start + 1..start + 1 + end].to_string());
-            remaining = &remaining[start + 1 + end + 1..];
-        } else {
-            break;
-        }
-    }
-
-    results
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parse_bbox() {
-        let props = parse_properties("bbox 100 50 200 150");
-        assert_eq!(
-            props.bbox,
-            Some(BBox {
-                x1: 100,
-                y1: 50,
-                x2: 200,
-                y2: 150
-            })
-        );
-    }
-
-    #[test]
-    fn test_parse_baseline() {
-        let props = parse_properties("baseline 0.015 -18");
-        assert_eq!(
-            props.baseline,
-            Some(Baseline {
-                slope: 0.015,
-                constant: -18
-            })
-        );
-    }
-
-    #[test]
-    fn test_parse_multiple_properties() {
-        let props = parse_properties("bbox 0 0 100 50; x_wconf 95.5; textangle 7.2");
-        assert_eq!(
-            props.bbox,
-            Some(BBox {
-                x1: 0,
-                y1: 0,
-                x2: 100,
-                y2: 50
-            })
-        );
-        assert_eq!(props.x_wconf, Some(95.5));
-        assert_eq!(props.textangle, Some(7.2));
-    }
-
-    #[test]
-    fn test_parse_quoted_strings() {
-        let props = parse_properties("x_font \"Comic Sans MS\"; x_fsize 12");
-        assert_eq!(props.x_font, Some("Comic Sans MS".to_string()));
-        assert_eq!(props.x_fsize, Some(12));
-    }
-
-    #[test]
-    fn test_parse_poly() {
-        let props = parse_properties("poly 0 0 0 10 10 10 10 0");
-        assert_eq!(props.poly, Some(vec![(0, 0), (0, 10), (10, 10), (10, 0)]));
-    }
-
-    #[test]
-    fn test_parse_x_confs() {
-        let props = parse_properties("x_confs 37.3 51.23 100");
-        assert_eq!(props.x_confs, vec![37.3, 51.23, 100.0]);
-    }
-}

data/vendor/html-to-markdown-rs/src/hocr/spatial/coords.rs

@@ -1,129 +0,0 @@
-#![allow(
-    clippy::cast_precision_loss,
-    clippy::cast_sign_loss,
-    clippy::unused_self,
-    clippy::float_cmp
-)]
-//! Coordinate types and parsing from hOCR bbox attributes
-
-/// Represents a word extracted from hOCR with position and confidence information
-#[derive(Debug, Clone)]
-pub struct HocrWord {
-    /// The text content of the word
-    pub text: String,
-    /// X-coordinate of the left edge (pixels)
-    pub left: u32,
-    /// Y-coordinate of the top edge (pixels)
-    pub top: u32,
-    /// Width of the word bounding box (pixels)
-    pub width: u32,
-    /// Height of the word bounding box (pixels)
-    pub height: u32,
-    /// OCR confidence score (0.0 to 100.0)
-    pub confidence: f64,
-}
-
-impl HocrWord {
-    /// Get the right edge position
-    #[must_use]
-    pub const fn right(&self) -> u32 {
-        self.left + self.width
-    }
-
-    /// Get the bottom edge position
-    #[must_use]
-    pub const fn bottom(&self) -> u32 {
-        self.top + self.height
-    }
-
-    /// Get the vertical center position
-    #[must_use]
-    pub fn y_center(&self) -> f64 {
-        f64::from(self.top) + (f64::from(self.height) / 2.0)
-    }
-
-    /// Get the horizontal center position
-    #[must_use]
-    pub fn x_center(&self) -> f64 {
-        f64::from(self.left) + (f64::from(self.width) / 2.0)
-    }
-}
-
-/// Parse bbox attribute from hOCR title attribute
-///
-/// Example: "bbox 100 50 180 80; `x_wconf` 95" -> (100, 50, 80, 30)
-pub fn parse_bbox(title: &str) -> Option<(u32, u32, u32, u32)> {
-    for part in title.split(';') {
-        let part = part.trim();
-
-        if let Some(bbox_str) = part.strip_prefix("bbox ") {
-            let coords: Vec<&str> = bbox_str.split_whitespace().collect();
-            if coords.len() == 4 {
-                if let (Ok(x1), Ok(y1), Ok(x2), Ok(y2)) = (
-                    coords[0].parse::<u32>(),
-                    coords[1].parse::<u32>(),
-                    coords[2].parse::<u32>(),
-                    coords[3].parse::<u32>(),
-                ) {
-                    let width = x2.saturating_sub(x1);
-                    let height = y2.saturating_sub(y1);
-                    return Some((x1, y1, width, height));
-                }
-            }
-        }
-    }
-    None
-}
-
-/// Parse confidence from hOCR title attribute
-///
-/// Example: "bbox 100 50 180 80; `x_wconf` 95" -> 95.0
-pub fn parse_confidence(title: &str) -> f64 {
-    for part in title.split(';') {
-        let part = part.trim();
-        if let Some(conf_str) = part.strip_prefix("x_wconf ") {
-            if let Ok(conf) = conf_str.trim().parse::<f64>() {
-                return conf;
-            }
-        }
-    }
-    0.0
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parse_bbox() {
-        assert_eq!(parse_bbox("bbox 100 50 180 80"), Some((100, 50, 80, 30)));
-        assert_eq!(parse_bbox("bbox 0 0 100 200"), Some((0, 0, 100, 200)));
-        assert_eq!(parse_bbox("bbox 100 50 180 80; x_wconf 95"), Some((100, 50, 80, 30)));
-        assert_eq!(parse_bbox("invalid"), None);
-        assert_eq!(parse_bbox("bbox 100 50"), None);
-    }
-
-    #[test]
-    fn test_parse_confidence() {
-        assert_eq!(parse_confidence("x_wconf 95.5"), 95.5);
-        assert_eq!(parse_confidence("bbox 100 50 180 80; x_wconf 92"), 92.0);
-        assert_eq!(parse_confidence("invalid"), 0.0);
-    }
-
-    #[test]
-    fn test_hocr_word_methods() {
-        let word = HocrWord {
-            text: "Hello".to_string(),
-            left: 100,
-            top: 50,
-            width: 80,
-            height: 30,
-            confidence: 95.5,
-        };
-
-        assert_eq!(word.right(), 180);
-        assert_eq!(word.bottom(), 80);
-        assert_eq!(word.y_center(), 65.0);
-        assert_eq!(word.x_center(), 140.0);
-    }
-}

data/vendor/html-to-markdown-rs/src/hocr/spatial/grouping.rs

@@ -1,165 +0,0 @@
-#![allow(clippy::cast_precision_loss, clippy::cast_sign_loss, clippy::unused_self)]
-//! Word extraction and DOM processing for hOCR documents
-
-use crate::hocr::spatial::coords::{HocrWord, parse_bbox, parse_confidence};
-
-/// Extract text content from a node
-#[allow(clippy::trivially_copy_pass_by_ref)]
-fn get_text_content(node_handle: &tl::NodeHandle, parser: &tl::Parser) -> String {
-    let mut text = String::new();
-
-    if let Some(node) = node_handle.get(parser) {
-        match node {
-            tl::Node::Raw(bytes) => {
-                text.push_str(&bytes.as_utf8_str());
-            }
-            tl::Node::Tag(tag) => {
-                let children = tag.children();
-                for child_handle in children.top().iter() {
-                    text.push_str(&get_text_content(child_handle, parser));
-                }
-            }
-            tl::Node::Comment(_) => {}
-        }
-    }
-
-    text
-}
-
-/// Extract hOCR words from a DOM tree
-///
-/// Walks the DOM and extracts all elements with `ocrx_word` class,
-/// parsing their bbox and confidence information.
-#[must_use]
-#[allow(clippy::trivially_copy_pass_by_ref)]
-pub fn extract_hocr_words(node_handle: &tl::NodeHandle, parser: &tl::Parser, min_confidence: f64) -> Vec<HocrWord> {
-    let mut words = Vec::new();
-
-    if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
-        let tag_name = tag.name().as_utf8_str();
-        let attrs = tag.attributes();
-
-        let class_attr = attrs.get("class").flatten().map(|v| v.as_utf8_str().to_string());
-
-        // hOCR class validation removed for performance
-
-        if tag_name == "span" {
-            let is_word = class_attr.as_ref().is_some_and(|c| c.contains("ocrx_word"));
-            let title = attrs.get("title").flatten().map(|v| v.as_utf8_str());
-
-            if is_word {
-                let title_str = title.as_deref().unwrap_or("");
-                if let Some((left, top, width, height)) = parse_bbox(title_str) {
-                    let confidence = parse_confidence(title_str);
-
-                    if confidence >= min_confidence {
-                        let text = get_text_content(node_handle, parser).trim().to_string();
-
-                        if !text.is_empty() {
-                            words.push(HocrWord {
-                                text,
-                                left,
-                                top,
-                                width,
-                                height,
-                                confidence,
-                            });
-                        }
-                    }
-                }
-            }
-        }
-
-        let children = tag.children();
-        for child_handle in children.top().iter() {
-            words.extend(extract_hocr_words(child_handle, parser, min_confidence));
-        }
-    }
-
-    words
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_extract_hocr_words() {
-        let hocr = r#"
-            <div class="ocr_page">
-                <span class="ocrx_word" title="bbox 100 50 150 80; x_wconf 95">Hello</span>
-                <span class="ocrx_word" title="bbox 160 50 210 80; x_wconf 92">World</span>
-            </div>
-        "#;
-
-        let dom = tl::parse(hocr, tl::ParserOptions::default()).unwrap();
-        let parser = dom.parser();
-
-        let mut words = Vec::new();
-        for child_handle in dom.children().iter() {
-            words.extend(extract_hocr_words(child_handle, parser, 0.0));
-        }
-
-        assert_eq!(words.len(), 2);
-        assert_eq!(words[0].text, "Hello");
-        assert_eq!(words[0].left, 100);
-        assert!((words[0].confidence - 95.0).abs() < f64::EPSILON);
-
-        assert_eq!(words[1].text, "World");
-        assert_eq!(words[1].left, 160);
-        assert!((words[1].confidence - 92.0).abs() < f64::EPSILON);
-    }
-
-    #[test]
-    fn test_extract_hocr_words_confidence_filter() {
-        let hocr = r#"
-            <div class="ocr_page">
-                <span class="ocrx_word" title="bbox 100 50 150 80; x_wconf 95">HighConf</span>
-                <span class="ocrx_word" title="bbox 160 50 210 80; x_wconf 50">LowConf</span>
-                <span class="ocrx_word" title="bbox 220 50 270 80; x_wconf 98">VeryHigh</span>
-            </div>
-        "#;
-
-        let dom = tl::parse(hocr, tl::ParserOptions::default()).unwrap();
-        let parser = dom.parser();
-
-        let mut words = Vec::new();
-        for child_handle in dom.children().iter() {
-            words.extend(extract_hocr_words(child_handle, parser, 90.0));
-        }
-
-        assert_eq!(words.len(), 2);
-        assert_eq!(words[0].text, "HighConf");
-        assert_eq!(words[1].text, "VeryHigh");
-    }
-
-    #[test]
-    fn test_end_to_end_hocr_table_extraction() {
-        let hocr = r#"
-            <div class="ocr_page">
-                <span class="ocrx_word" title="bbox 100 50 140 70; x_wconf 95">Product</span>
-                <span class="ocrx_word" title="bbox 200 50 240 70; x_wconf 95">Price</span>
-                <span class="ocrx_word" title="bbox 100 100 140 120; x_wconf 95">Apple</span>
-                <span class="ocrx_word" title="bbox 200 100 240 120; x_wconf 95">$1.50</span>
-                <span class="ocrx_word" title="bbox 100 150 140 170; x_wconf 95">Orange</span>
-                <span class="ocrx_word" title="bbox 200 150 240 170; x_wconf 95">$2.00</span>
-            </div>
-        "#;
-
-        let dom = tl::parse(hocr, tl::ParserOptions::default()).unwrap();
-        let parser = dom.parser();
-
-        let mut words = Vec::new();
-        for child_handle in dom.children().iter() {
-            words.extend(extract_hocr_words(child_handle, parser, 0.0));
-        }
-
-        assert_eq!(words.len(), 6);
-        assert_eq!(words[0].text, "Product");
-        assert_eq!(words[1].text, "Price");
-        assert_eq!(words[2].text, "Apple");
-        assert_eq!(words[3].text, "$1.50");
-        assert_eq!(words[4].text, "Orange");
-        assert_eq!(words[5].text, "$2.00");
-    }
-}