html-to-markdown 2.30.0 → 3.0.0

data/vendor/html-to-markdown-rs/src/hocr/converter/code_analysis.rs

@@ -1,254 +0,0 @@
-//! Code block analysis and validation
-
-use super::hierarchy::CodeLineInfo;
-use super::keywords::{contains_keyword_token, is_shell_prompt, starts_with_keyword};
-
-/// Check if a line looks like it's part of a bullet list
-pub fn is_bullet_like(line: &str) -> bool {
-    let trimmed = line.trim_start();
-    if trimmed.is_empty() {
-        return false;
-    }
-
-    if trimmed.starts_with("- ") || trimmed.starts_with("* ") || trimmed.starts_with("+ ") || trimmed.starts_with("•")
-    {
-        return true;
-    }
-
-    let mut chars = trimmed.chars().peekable();
-    let mut digit_count = 0;
-    while let Some(&ch) = chars.peek() {
-        if ch.is_ascii_digit() {
-            digit_count += 1;
-            chars.next();
-            continue;
-        }
-        break;
-    }
-
-    if digit_count > 0 {
-        if let Some(&ch) = chars.peek() {
-            if (ch == '.' || ch == ')') && chars.clone().nth(1).is_some_and(char::is_whitespace) {
-                return true;
-            }
-        }
-    }
-
-    false
-}
-
-/// Determine if lines form a paragraph of code
-pub fn is_code_paragraph(lines: &[CodeLineInfo]) -> bool {
-    if lines.is_empty() {
-        return false;
-    }
-
-    let mut strong_markers = 0;
-    let mut moderate_markers = 0;
-    let mut total = 0;
-
-    for info in lines {
-        let text = info.text.trim();
-        if text.is_empty() {
-            continue;
-        }
-
-        if is_bullet_like(&info.text) {
-            return false;
-        }
-
-        total += 1;
-        let lower = text.to_lowercase();
-        let trimmed = text.trim_start();
-
-        let documentation_tokens = [
-            "definition",
-            "theorem",
-            "lemma",
-            "proof",
-            "corollary",
-            "algorithm",
-            "figure",
-            "table",
-            "appendix",
-        ];
-        if documentation_tokens
-            .iter()
-            .any(|token| contains_keyword_token(&lower, token))
-        {
-            return false;
-        }
-
-        let has_keyword = (starts_with_keyword(trimmed, "function") && text.contains('('))
-            || (starts_with_keyword(trimmed, "return")
-                && trimmed.chars().nth("return".len()).is_none_or(char::is_whitespace))
-            || trimmed.starts_with("console.")
-            || starts_with_keyword(trimmed, "async")
-            || starts_with_keyword(trimmed, "await")
-            || (starts_with_keyword(trimmed, "class") && (text.contains('{') || text.contains(':')))
-            || (starts_with_keyword(trimmed, "struct") && text.contains('{'))
-            || (starts_with_keyword(trimmed, "enum") && text.contains('{'))
-            || (starts_with_keyword(trimmed, "def") && (text.contains('(') || text.contains(':')))
-            || (starts_with_keyword(trimmed, "fn") && text.contains('('))
-            || (starts_with_keyword(trimmed, "pub")
-                && (text.contains("fn") || text.contains("struct") || text.contains("enum")))
-            || starts_with_keyword(trimmed, "import")
-            || starts_with_keyword(trimmed, "using")
-            || starts_with_keyword(trimmed, "namespace")
-            || starts_with_keyword(trimmed, "public")
-            || starts_with_keyword(trimmed, "private")
-            || starts_with_keyword(trimmed, "protected")
-            || starts_with_keyword(trimmed, "static")
-            || starts_with_keyword(trimmed, "void")
-            || starts_with_keyword(trimmed, "try")
-            || starts_with_keyword(trimmed, "catch")
-            || starts_with_keyword(trimmed, "finally")
-            || starts_with_keyword(trimmed, "throw")
-            || starts_with_keyword(trimmed, "typedef")
-            || starts_with_keyword(trimmed, "package")
-            || starts_with_keyword(trimmed, "module");
-
-        let has_symbol = text.contains(';') || text.contains("::");
-
-        if has_keyword || has_symbol {
-            strong_markers += 1;
-            continue;
-        }
-
-        if is_shell_prompt(text) {
-            strong_markers += 1;
-            continue;
-        }
-        let has_assignment = text.contains(" = ")
-            || text.contains("+=")
-            || text.contains("-=")
-            || text.contains("*=")
-            || text.contains("/=")
-            || text.contains(" := ")
-            || text.contains(" == ");
-
-        let has_arrow = text.contains("=>");
-        let has_brace = text.contains('{') || text.contains('}');
-        let has_pointer_arrow = text.contains("->");
-
-        if has_assignment || has_arrow || has_brace || has_pointer_arrow {
-            moderate_markers += 1;
-        }
-    }
-
-    if total == 0 {
-        return false;
-    }
-    if strong_markers == 0 {
-        return false;
-    }
-    if strong_markers * 2 >= total {
-        return true;
-    }
-    (strong_markers + moderate_markers) * 2 >= total
-}
-
-/// Check if code block is likely valid and not just prose
-pub fn is_confident_code_block(lines: &[CodeLineInfo]) -> bool {
-    let mut total = 0;
-    let mut keyword_lines = 0;
-    let mut punctuation_lines = 0;
-    let mut assignment_lines = 0;
-    let mut shell_lines = 0;
-    let mut indent_lines = 0;
-
-    let min_x = lines.iter().map(|info| info.x1).min().unwrap_or_default();
-
-    for info in lines {
-        let text = info.text.trim();
-        if text.is_empty() {
-            continue;
-        }
-        total += 1;
-
-        if is_shell_prompt(text) {
-            shell_lines += 1;
-        }
-
-        let trimmed = text.trim_start();
-
-        if (starts_with_keyword(trimmed, "function") && text.contains('('))
-            || trimmed.starts_with("console.")
-            || (starts_with_keyword(trimmed, "return")
-                && trimmed.chars().nth("return".len()).is_none_or(char::is_whitespace))
-            || starts_with_keyword(trimmed, "async")
-            || starts_with_keyword(trimmed, "await")
-            || (starts_with_keyword(trimmed, "class") && (text.contains('{') || text.contains(':')))
-            || (starts_with_keyword(trimmed, "struct") && text.contains('{'))
-            || (starts_with_keyword(trimmed, "enum") && text.contains('{'))
-            || (starts_with_keyword(trimmed, "def") && (text.contains('(') || text.contains(':')))
-            || (starts_with_keyword(trimmed, "fn") && text.contains('('))
-            || (starts_with_keyword(trimmed, "pub")
-                && (text.contains("fn") || text.contains("struct") || text.contains("enum")))
-            || starts_with_keyword(trimmed, "import")
-            || starts_with_keyword(trimmed, "using")
-            || starts_with_keyword(trimmed, "namespace")
-            || starts_with_keyword(trimmed, "public")
-            || starts_with_keyword(trimmed, "private")
-            || starts_with_keyword(trimmed, "protected")
-            || starts_with_keyword(trimmed, "static")
-            || starts_with_keyword(trimmed, "void")
-            || starts_with_keyword(trimmed, "try")
-            || starts_with_keyword(trimmed, "catch")
-            || starts_with_keyword(trimmed, "finally")
-            || starts_with_keyword(trimmed, "throw")
-            || starts_with_keyword(trimmed, "typedef")
-            || starts_with_keyword(trimmed, "package")
-            || starts_with_keyword(trimmed, "module")
-        {
-            keyword_lines += 1;
-        }
-
-        if text.contains(';')
-            || text.contains('{')
-            || text.contains('}')
-            || text.contains("::")
-            || text.contains("->")
-            || text.contains("=>")
-        {
-            punctuation_lines += 1;
-        }
-
-        if text.contains(" = ")
-            || text.contains("+=")
-            || text.contains("-=")
-            || text.contains("*=")
-            || text.contains("/=")
-            || text.contains(" := ")
-            || text.contains(" == ")
-        {
-            assignment_lines += 1;
-        }
-
-        if info.x1 > min_x + 8 {
-            indent_lines += 1;
-        }
-    }
-
-    if total < 3 {
-        return false;
-    }
-
-    if shell_lines >= 2 && shell_lines * 2 >= total {
-        return true;
-    }
-
-    if keyword_lines >= 2 && assignment_lines >= 1 {
-        return true;
-    }
-
-    if keyword_lines >= 1 && punctuation_lines >= 1 && assignment_lines >= 1 {
-        return true;
-    }
-
-    if indent_lines == total && keyword_lines >= 1 && assignment_lines >= 1 {
-        return true;
-    }
-
-    false
-}

data/vendor/html-to-markdown-rs/src/hocr/converter/core.rs

@@ -1,249 +0,0 @@
-//! Core conversion logic for hOCR to Markdown
-
-use super::elements::convert_element;
-use super::output::{ConvertContext, collapse_extra_newlines};
-use crate::hocr::types::HocrElement;
-
-/// Convert hOCR elements to Markdown with semantic formatting
-///
-/// Transforms hOCR document structure into clean, readable Markdown while preserving
-/// document hierarchy and semantic meaning.
-///
-/// # Arguments
-///
-/// * `elements` - hOCR elements to convert (typically from `extract_hocr_document`)
-/// * `preserve_structure` - If `true`, sorts elements by their `order` property to respect reading order
-///
-/// # Returns
-///
-/// A `String` containing the formatted Markdown output
-///
-/// # Semantic Conversion
-///
-/// All 40 hOCR 1.2 element types are converted with appropriate markdown formatting:
-///
-/// | hOCR Element | Markdown Output |
-/// |--------------|-----------------|
-/// | `ocr_title`, `ocr_chapter` | `# Heading` |
-/// | `ocr_section` | `## Heading` |
-/// | `ocr_subsection` | `### Heading` |
-/// | `ocr_par` | Paragraph with blank lines |
-/// | `ocr_blockquote` | `> Quote` |
-/// | `ocr_abstract` | `**Abstract**` header |
-/// | `ocr_author` | `*Author*` (italic) |
-/// | `ocr_image`, `ocr_photo` | `![alt](path)` |
-/// | `ocr_math`, `ocr_chem` | `` `formula` `` (inline code) |
-/// | `ocr_display` | ` ```equation``` ` (code block) |
-/// | `ocr_separator` | `---` (horizontal rule) |
-/// | `ocr_dropcap` | `**Letter**` (bold) |
-/// | `ocrx_word` | Word with markdown escaping |
-///
-/// # Example
-///
-/// ```rust
-/// use html_to_markdown_rs::hocr::{extract_hocr_document, convert_to_markdown};
-///
-/// let html = r#"<div class="ocr_page">
-///     <h1 class="ocr_title">Document Title</h1>
-///     <p class="ocr_par" title="order 1">
-///         <span class="ocrx_word" title="bbox 10 10 50 30; x_wconf 95">Hello</span>
-///         <span class="ocrx_word" title="bbox 60 10 100 30; x_wconf 92">World</span>
-///     </p>
-/// </div>"#;
-///
-/// let dom = tl::parse(html, tl::ParserOptions::default()).unwrap();
-/// let (elements, _) = extract_hocr_document(&dom);
-/// let markdown = convert_to_markdown(&elements, true);
-/// // Output: "# Document Title\n\nHello World"
-/// ```
-#[must_use]
-pub fn convert_to_markdown(elements: &[HocrElement], preserve_structure: bool) -> String {
-    convert_to_markdown_with_options(elements, preserve_structure, true)
-}
-
-/// Convert hOCR elements to Markdown with advanced options.
-///
-/// Transforms hOCR document structure into clean, readable Markdown with fine-grained
-/// control over structure preservation and spatial table reconstruction behavior.
-///
-/// # Arguments
-///
-/// * `elements` - hOCR elements to convert (typically from `extract_hocr_document`)
-/// * `preserve_structure` - If `true`, sorts elements by their `order` property to respect reading order.
-///   If `false`, elements are processed in their original tree order.
-/// * `enable_spatial_tables` - If `true`, attempts to reconstruct table structure from spatial
-///   positioning of words. If `false`, word positions are ignored and only text content is used.
-///
-/// # Returns
-///
-/// A `String` containing the formatted Markdown output
-///
-/// # Performance
-///
-/// - Spatial table reconstruction is more computationally expensive but produces better table formatting
-/// - For documents without tables, setting `enable_spatial_tables` to `false` improves performance
-/// - Structure preservation requires sorting which adds O(n log n) complexity; disable if not needed
-#[must_use]
-pub fn convert_to_markdown_with_options(
-    elements: &[HocrElement],
-    preserve_structure: bool,
-    enable_spatial_tables: bool,
-) -> String {
-    let mut output = String::new();
-    let mut ctx = ConvertContext::default();
-
-    if preserve_structure && should_sort_children(elements) {
-        let mut sorted_elements: Vec<&HocrElement> = elements.iter().collect();
-        sorted_elements.sort_by_key(|e| e.properties.order.unwrap_or(u32::MAX));
-        for element in sorted_elements {
-            convert_element(
-                element,
-                &mut output,
-                0,
-                preserve_structure,
-                enable_spatial_tables,
-                &mut ctx,
-            );
-        }
-    } else {
-        for element in elements {
-            convert_element(
-                element,
-                &mut output,
-                0,
-                preserve_structure,
-                enable_spatial_tables,
-                &mut ctx,
-            );
-        }
-    }
-
-    collapse_extra_newlines(&mut output);
-    output.trim().to_string()
-}
-
-fn should_sort_children(children: &[HocrElement]) -> bool {
-    let mut last = 0u32;
-    let mut saw_any = false;
-
-    for child in children {
-        let order = child.properties.order.unwrap_or(u32::MAX);
-        if saw_any && order < last {
-            return true;
-        }
-        last = order;
-        saw_any = true;
-    }
-
-    false
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::hocr::types::{BBox, HocrElement, HocrElementType, HocrProperties};
-
-    #[test]
-    fn test_convert_title() {
-        let element = HocrElement {
-            element_type: HocrElementType::OcrTitle,
-            properties: HocrProperties::default(),
-            text: "Document Title".to_string(),
-            children: vec![],
-        };
-
-        let markdown = convert_to_markdown(&[element], true);
-        assert_eq!(markdown, "# Document Title");
-    }
-
-    #[test]
-    fn test_spatial_table_reconstruction_can_be_disabled() {
-        fn word(text: &str, x1: u32, y1: u32) -> HocrElement {
-            HocrElement {
-                element_type: HocrElementType::OcrxWord,
-                properties: HocrProperties {
-                    bbox: Some(BBox {
-                        x1,
-                        y1,
-                        x2: x1 + 40,
-                        y2: y1 + 20,
-                    }),
-                    x_wconf: Some(95.0),
-                    ..HocrProperties::default()
-                },
-                text: text.to_string(),
-                children: vec![],
-            }
-        }
-
-        let paragraph = HocrElement {
-            element_type: HocrElementType::OcrPar,
-            properties: HocrProperties::default(),
-            text: String::new(),
-            children: vec![
-                word("A", 10, 10),
-                word("B", 120, 10),
-                word("C", 230, 10),
-                word("D", 12, 60),
-                word("E", 122, 60),
-                word("F", 232, 60),
-            ],
-        };
-
-        let markdown_with_tables = convert_to_markdown_with_options(std::slice::from_ref(&paragraph), true, true);
-        assert!(
-            markdown_with_tables.contains("| --- |"),
-            "Expected spatial table reconstruction to produce a markdown table"
-        );
-
-        let markdown_without_tables = convert_to_markdown_with_options(std::slice::from_ref(&paragraph), true, false);
-        assert!(
-            !markdown_without_tables.contains('|'),
-            "Table reconstruction should be disabled when the flag is false"
-        );
-        assert!(
-            markdown_without_tables.contains("A B C"),
-            "Plain text output should retain original word order"
-        );
-    }
-
-    #[test]
-    fn test_convert_paragraph_with_words() {
-        let par = HocrElement {
-            element_type: HocrElementType::OcrPar,
-            properties: HocrProperties::default(),
-            text: String::new(),
-            children: vec![
-                HocrElement {
-                    element_type: HocrElementType::OcrxWord,
-                    properties: HocrProperties::default(),
-                    text: "Hello".to_string(),
-                    children: vec![],
-                },
-                HocrElement {
-                    element_type: HocrElementType::OcrxWord,
-                    properties: HocrProperties::default(),
-                    text: "World".to_string(),
-                    children: vec![],
-                },
-            ],
-        };
-
-        let markdown = convert_to_markdown(&[par], true);
-        assert!(markdown.contains("Hello"));
-        assert!(markdown.contains("World"));
-    }
-
-    #[test]
-    fn test_convert_blockquote() {
-        let quote = HocrElement {
-            element_type: HocrElementType::OcrBlockquote,
-            properties: HocrProperties::default(),
-            text: "This is a quote".to_string(),
-            children: vec![],
-        };
-
-        let markdown = convert_to_markdown(&[quote], true);
-        assert!(markdown.starts_with("> "));
-    }
-}