html-to-markdown 3.4.0 → 3.6.0.pre.rc.23

data/vendor/html-to-markdown-rs/src/validation.rs

@@ -1,158 +0,0 @@
-//! Input validation module for HTML to Markdown conversion.
-//!
-//! Provides validation functions to detect and reject binary data,
-//! corrupted input, and other non-text content.
-
-use crate::error::{ConversionError, Result};
-
-const BINARY_SCAN_LIMIT: usize = 8192;
-const BINARY_CONTROL_RATIO: f64 = 0.3;
-const BINARY_UTF16_NULL_RATIO: f64 = 0.2;
-const BINARY_NUL_RATIO: f64 = 0.01;
-const BINARY_NUL_MAX: usize = 8;
-
-const BINARY_MAGIC_PREFIXES: &[(&[u8], &str)] = &[
-    (b"\x1F\x8B", "gzip-compressed data"),
-    (b"\x28\xB5\x2F\xFD", "zstd-compressed data"),
-    (b"PK\x03\x04", "zip archive"),
-    (b"PK\x05\x06", "zip archive"),
-    (b"PK\x07\x08", "zip archive"),
-    (b"%PDF-", "PDF data"),
-];
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum Utf16Encoding {
-    BomLe,
-    BomBe,
-    NoBomLe,
-    NoBomBe,
-}
-
-/// Validate HTML input and reject binary/corrupted data.
-///
-/// # Errors
-///
-/// Returns `ConversionError::InvalidInput` if the input is detected as binary,
-/// compressed, or contains too many control characters.
-#[allow(clippy::cast_precision_loss)]
-pub fn validate_input(html: &str) -> Result<()> {
-    let bytes = html.as_bytes();
-    if bytes.is_empty() {
-        return Ok(());
-    }
-
-    if let Some(label) = detect_binary_magic(bytes) {
-        return Err(ConversionError::InvalidInput(format!(
-            "binary data detected ({label}); decode/decompress to UTF-8 HTML first"
-        )));
-    }
-
-    let sample_len = bytes.len().min(BINARY_SCAN_LIMIT);
-    let mut control_count = 0usize;
-    let mut nul_count = 0usize;
-
-    for &byte in &bytes[..sample_len] {
-        if byte == 0 {
-            nul_count += 1;
-        }
-        let is_control = (byte < 0x09) || (0x0E..0x20).contains(&byte);
-        if is_control {
-            control_count += 1;
-        }
-    }
-
-    if nul_count > 0 {
-        if let Some(encoding) = detect_utf16_encoding(bytes) {
-            let label = utf16_label(encoding);
-            return Err(ConversionError::InvalidInput(format!(
-                "binary data detected ({label}); decode to UTF-8 HTML first"
-            )));
-        }
-        let nul_ratio = nul_count as f64 / sample_len as f64;
-        if nul_ratio > BINARY_NUL_RATIO && nul_count > BINARY_NUL_MAX {
-            return Err(ConversionError::InvalidInput(
-                "binary data detected (excess NUL bytes)".to_string(),
-            ));
-        }
-    }
-
-    if sample_len >= 32 {
-        let control_ratio = control_count as f64 / sample_len as f64;
-        if control_ratio > BINARY_CONTROL_RATIO {
-            return Err(ConversionError::InvalidInput(
-                "binary data detected (excess control bytes)".to_string(),
-            ));
-        }
-    }
-
-    Ok(())
-}
-
-fn detect_binary_magic(bytes: &[u8]) -> Option<&'static str> {
-    for (prefix, label) in BINARY_MAGIC_PREFIXES {
-        if bytes.starts_with(prefix) {
-            return Some(*label);
-        }
-    }
-    None
-}
-
-#[allow(clippy::cast_precision_loss)]
-pub fn detect_utf16_encoding(bytes: &[u8]) -> Option<Utf16Encoding> {
-    if bytes.len() >= 2 {
-        if bytes.starts_with(b"\xFF\xFE") {
-            return Some(Utf16Encoding::BomLe);
-        }
-        if bytes.starts_with(b"\xFE\xFF") {
-            return Some(Utf16Encoding::BomBe);
-        }
-    }
-
-    let sample_len = bytes.len().min(BINARY_SCAN_LIMIT);
-    if sample_len < 4 {
-        return None;
-    }
-
-    let mut nul_count = 0usize;
-    let mut even_nul_count = 0usize;
-    let mut odd_nul_count = 0usize;
-
-    for (idx, &byte) in bytes[..sample_len].iter().enumerate() {
-        if byte == 0 {
-            nul_count += 1;
-            if idx % 2 == 0 {
-                even_nul_count += 1;
-            } else {
-                odd_nul_count += 1;
-            }
-        }
-    }
-
-    if nul_count < 2 {
-        return None;
-    }
-
-    let nul_ratio = nul_count as f64 / sample_len as f64;
-    if nul_ratio < BINARY_UTF16_NULL_RATIO {
-        return None;
-    }
-
-    let dominant_ratio = (even_nul_count.max(odd_nul_count) as f64) / nul_count as f64;
-    if dominant_ratio < 0.9 {
-        return None;
-    }
-
-    if odd_nul_count >= even_nul_count {
-        Some(Utf16Encoding::NoBomLe)
-    } else {
-        Some(Utf16Encoding::NoBomBe)
-    }
-}
-
-fn utf16_label(encoding: Utf16Encoding) -> &'static str {
-    match encoding {
-        Utf16Encoding::BomLe => "UTF-16LE BOM",
-        Utf16Encoding::BomBe => "UTF-16BE BOM",
-        Utf16Encoding::NoBomLe | Utf16Encoding::NoBomBe => "UTF-16 data without BOM",
-    }
-}

data/vendor/html-to-markdown-rs/src/visitor/default_impl.rs

@@ -1,63 +0,0 @@
-//! Default visitor implementations and utilities.
-//!
-//! This module provides standard visitor patterns and helpers for common use cases.
-
-use std::cell::RefCell;
-use std::rc::Rc;
-
-/// Type alias for a visitor handle (Rc-wrapped `RefCell` for interior mutability).
-///
-/// This allows visitors to be passed around and shared while still being mutable.
-pub type VisitorHandle = Rc<RefCell<dyn super::traits::HtmlVisitor>>;
-
-#[cfg(test)]
-mod tests {
-    use super::super::traits::HtmlVisitor;
-    use super::super::types::{NodeContext, NodeType, VisitResult};
-    use std::collections::BTreeMap;
-
-    #[derive(Debug)]
-    struct TrackingVisitor {
-        element_count: usize,
-        text_count: usize,
-    }
-
-    impl HtmlVisitor for TrackingVisitor {
-        fn visit_element_start(&mut self, _ctx: &NodeContext) -> VisitResult {
-            self.element_count += 1;
-            VisitResult::Continue
-        }
-
-        fn visit_text(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-            self.text_count += 1;
-            VisitResult::Continue
-        }
-    }
-
-    #[test]
-    fn test_visitor_handle_creation() {
-        let visitor = TrackingVisitor {
-            element_count: 0,
-            text_count: 0,
-        };
-
-        let handle = std::rc::Rc::new(std::cell::RefCell::new(visitor));
-
-        {
-            let mut v = handle.borrow_mut();
-            let ctx = NodeContext {
-                node_type: NodeType::Text,
-                tag_name: "p".to_string(),
-                attributes: BTreeMap::new(),
-                depth: 1,
-                index_in_parent: 0,
-                parent_tag: Some("div".to_string()),
-                is_inline: false,
-            };
-            v.visit_text(&ctx, "test");
-        }
-
-        let v = handle.borrow();
-        assert_eq!(v.text_count, 1);
-    }
-}

data/vendor/html-to-markdown-rs/src/visitor/mod.rs

@@ -1,41 +0,0 @@
-//! Visitor pattern for HTML to Markdown conversion.
-//!
-//! This module provides a comprehensive visitor trait that allows users to intervene
-//! in the HTML→Markdown transformation logic at any point. Visitors can inspect,
-//! modify, or replace the default conversion behavior for any HTML element type.
-//!
-//! # Design Philosophy
-//!
-//! - **Flexibility over performance**: Visitors prioritize giving users full control
-//! - **Zero-cost when unused**: No performance impact if visitor feature is disabled
-//! - **Comprehensive coverage**: All 60+ HTML element types have dedicated visitor methods
-//! - **Pre/post hooks**: Both element entry and exit points are exposed
-//!
-//! # Example
-//!
-//! ```text
-//! use html_to_markdown_rs::visitor::{HtmlVisitor, NodeContext, VisitResult};
-//!
-//! struct CustomVisitor;
-//!
-//! impl HtmlVisitor for CustomVisitor {
-//!     fn visit_link(&mut self, ctx: &NodeContext, href: &str, text: &str, title: Option<&str>) -> VisitResult {
-//!         // Convert all links to plain text with URL in parentheses
-//!         VisitResult::Custom(format!("{} ({})", text, href))
-//!     }
-//!
-//!     fn visit_image(&mut self, ctx: &NodeContext, src: &str, alt: &str, title: Option<&str>) -> VisitResult {
-//!         // Skip all images
-//!         VisitResult::Skip
-//!     }
-//! }
-//! ```
-
-mod default_impl;
-mod traits;
-mod types;
-
-// Re-export all public items from submodules
-pub use default_impl::VisitorHandle;
-pub use traits::HtmlVisitor;
-pub use types::{NodeContext, NodeType, VisitResult};

data/vendor/html-to-markdown-rs/src/visitor/traits.rs

@@ -1,370 +0,0 @@
-//! Visitor traits for HTML to Markdown conversion.
-//!
-//! This module contains the synchronous visitor trait.
-
-use super::types::{NodeContext, VisitResult};
-
-/// Visitor trait for HTML→Markdown conversion.
-///
-/// Implement this trait to customize the conversion behavior for any HTML element type.
-/// All methods have default implementations that return `VisitResult::Continue`, allowing
-/// selective override of only the elements you care about.
-///
-/// # Method Naming Convention
-///
-/// - `visit_*_start`: Called before entering an element (pre-order traversal)
-/// - `visit_*_end`: Called after exiting an element (post-order traversal)
-/// - `visit_*`: Called for specific element types (e.g., `visit_link`, `visit_image`)
-///
-/// # Execution Order
-///
-/// For a typical element like `<div><p>text</p></div>`:
-/// 1. `visit_element_start` for `<div>`
-/// 2. `visit_element_start` for `<p>`
-/// 3. `visit_text` for "text"
-/// 4. `visit_element_end` for `<p>`
-/// 5. `visit_element_end` for `</div>`
-///
-/// # Performance Notes
-///
-/// - `visit_text` is the most frequently called method (~100+ times per document)
-/// - Return `VisitResult::Continue` quickly for elements you don't need to customize
-/// - Avoid heavy computation in visitor methods; consider caching if needed
-pub trait HtmlVisitor: std::fmt::Debug {
-    /// Called before entering any element.
-    ///
-    /// This is the first callback invoked for every HTML element, allowing
-    /// visitors to implement generic element handling before tag-specific logic.
-    fn visit_element_start(&mut self, _ctx: &NodeContext) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Called after exiting any element.
-    ///
-    /// Receives the default markdown output that would be generated.
-    /// Visitors can inspect or replace this output.
-    fn visit_element_end(&mut self, _ctx: &NodeContext, _output: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit text nodes (most frequent callback - ~100+ per document).
-    ///
-    /// # Arguments
-    /// - `ctx`: Node context (will have `node_type: NodeType::Text`)
-    /// - `text`: The raw text content (HTML entities already decoded)
-    fn visit_text(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit anchor links `<a href="...">`.
-    ///
-    /// # Arguments
-    /// - `ctx`: Node context with link element metadata
-    /// - `href`: The link URL (from `href` attribute)
-    /// - `text`: The link text content (already converted to markdown)
-    /// - `title`: Optional title attribute
-    fn visit_link(&mut self, _ctx: &NodeContext, _href: &str, _text: &str, _title: Option<&str>) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit images `<img src="...">`.
-    ///
-    /// # Arguments
-    /// - `ctx`: Node context with image element metadata
-    /// - `src`: The image source URL
-    /// - `alt`: The alt text
-    /// - `title`: Optional title attribute
-    fn visit_image(&mut self, _ctx: &NodeContext, _src: &str, _alt: &str, _title: Option<&str>) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit heading elements `<h1>` through `<h6>`.
-    ///
-    /// # Arguments
-    /// - `ctx`: Node context with heading metadata
-    /// - `level`: Heading level (1-6)
-    /// - `text`: The heading text content
-    /// - `id`: Optional id attribute (for anchor links)
-    fn visit_heading(&mut self, _ctx: &NodeContext, _level: u32, _text: &str, _id: Option<&str>) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit code blocks `<pre><code>`.
-    ///
-    /// # Arguments
-    /// - `ctx`: Node context
-    /// - `lang`: Optional language specifier (from class attribute)
-    /// - `code`: The code content
-    fn visit_code_block(&mut self, _ctx: &NodeContext, _lang: Option<&str>, _code: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit inline code `<code>`.
-    ///
-    /// # Arguments
-    /// - `ctx`: Node context
-    /// - `code`: The code content
-    fn visit_code_inline(&mut self, _ctx: &NodeContext, _code: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit list items `<li>`.
-    ///
-    /// # Arguments
-    /// - `ctx`: Node context
-    /// - `ordered`: Whether this is an ordered list item
-    /// - `marker`: The list marker (e.g., "-", "1.", "a)")
-    /// - `text`: The list item content (already converted)
-    fn visit_list_item(&mut self, _ctx: &NodeContext, _ordered: bool, _marker: &str, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Called before processing a list `<ul>` or `<ol>`.
-    fn visit_list_start(&mut self, _ctx: &NodeContext, _ordered: bool) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Called after processing a list `</ul>` or `</ol>`.
-    fn visit_list_end(&mut self, _ctx: &NodeContext, _ordered: bool, _output: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Called before processing a table `<table>`.
-    fn visit_table_start(&mut self, _ctx: &NodeContext) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit table rows `<tr>`.
-    ///
-    /// # Arguments
-    /// - `ctx`: Node context
-    /// - `cells`: Cell contents (already converted to markdown)
-    /// - `is_header`: Whether this row is in `<thead>`
-    fn visit_table_row(&mut self, _ctx: &NodeContext, _cells: &[String], _is_header: bool) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Called after processing a table `</table>`.
-    fn visit_table_end(&mut self, _ctx: &NodeContext, _output: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit blockquote elements `<blockquote>`.
-    ///
-    /// # Arguments
-    /// - `ctx`: Node context
-    /// - `content`: The blockquote content (already converted)
-    /// - `depth`: Nesting depth (for nested blockquotes)
-    fn visit_blockquote(&mut self, _ctx: &NodeContext, _content: &str, _depth: usize) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit strong/bold elements `<strong>`, `<b>`.
-    fn visit_strong(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit emphasis/italic elements `<em>`, `<i>`.
-    fn visit_emphasis(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit strikethrough elements `<s>`, `<del>`, `<strike>`.
-    fn visit_strikethrough(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit underline elements `<u>`, `<ins>`.
-    fn visit_underline(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit subscript elements `<sub>`.
-    fn visit_subscript(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit superscript elements `<sup>`.
-    fn visit_superscript(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit mark/highlight elements `<mark>`.
-    fn visit_mark(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit line break elements `<br>`.
-    fn visit_line_break(&mut self, _ctx: &NodeContext) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit horizontal rule elements `<hr>`.
-    fn visit_horizontal_rule(&mut self, _ctx: &NodeContext) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit custom elements (web components) or unknown tags.
-    ///
-    /// # Arguments
-    /// - `ctx`: Node context
-    /// - `tag_name`: The custom element's tag name
-    /// - `html`: The raw HTML of this element
-    fn visit_custom_element(&mut self, _ctx: &NodeContext, _tag_name: &str, _html: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit definition list `<dl>`.
-    fn visit_definition_list_start(&mut self, _ctx: &NodeContext) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit definition term `<dt>`.
-    fn visit_definition_term(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit definition description `<dd>`.
-    fn visit_definition_description(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Called after processing a definition list `</dl>`.
-    fn visit_definition_list_end(&mut self, _ctx: &NodeContext, _output: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit form elements `<form>`.
-    fn visit_form(&mut self, _ctx: &NodeContext, _action: Option<&str>, _method: Option<&str>) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit input elements `<input>`.
-    fn visit_input(
-        &mut self,
-        _ctx: &NodeContext,
-        _input_type: &str,
-        _name: Option<&str>,
-        _value: Option<&str>,
-    ) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit button elements `<button>`.
-    fn visit_button(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit audio elements `<audio>`.
-    fn visit_audio(&mut self, _ctx: &NodeContext, _src: Option<&str>) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit video elements `<video>`.
-    fn visit_video(&mut self, _ctx: &NodeContext, _src: Option<&str>) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit iframe elements `<iframe>`.
-    fn visit_iframe(&mut self, _ctx: &NodeContext, _src: Option<&str>) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit details elements `<details>`.
-    fn visit_details(&mut self, _ctx: &NodeContext, _open: bool) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit summary elements `<summary>`.
-    fn visit_summary(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit figure elements `<figure>`.
-    fn visit_figure_start(&mut self, _ctx: &NodeContext) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Visit figcaption elements `<figcaption>`.
-    fn visit_figcaption(&mut self, _ctx: &NodeContext, _text: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-
-    /// Called after processing a figure `</figure>`.
-    fn visit_figure_end(&mut self, _ctx: &NodeContext, _output: &str) -> VisitResult {
-        VisitResult::Continue
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::collections::BTreeMap;
-
-    #[derive(Debug)]
-    struct NoOpVisitor;
-
-    impl HtmlVisitor for NoOpVisitor {}
-
-    #[test]
-    fn test_default_visitor_implementation() {
-        let mut visitor = NoOpVisitor;
-
-        let ctx = NodeContext {
-            node_type: super::super::types::NodeType::Text,
-            tag_name: String::new(),
-            attributes: BTreeMap::new(),
-            depth: 0,
-            index_in_parent: 0,
-            parent_tag: None,
-            is_inline: true,
-        };
-
-        matches!(visitor.visit_element_start(&ctx), VisitResult::Continue);
-        matches!(visitor.visit_element_end(&ctx, "output"), VisitResult::Continue);
-        matches!(visitor.visit_text(&ctx, "text"), VisitResult::Continue);
-        matches!(visitor.visit_link(&ctx, "href", "text", None), VisitResult::Continue);
-        matches!(visitor.visit_image(&ctx, "src", "alt", None), VisitResult::Continue);
-        matches!(visitor.visit_heading(&ctx, 1, "text", None), VisitResult::Continue);
-        matches!(visitor.visit_code_block(&ctx, None, "code"), VisitResult::Continue);
-        matches!(visitor.visit_code_inline(&ctx, "code"), VisitResult::Continue);
-    }
-
-    #[derive(Debug)]
-    struct CustomLinkVisitor;
-
-    impl HtmlVisitor for CustomLinkVisitor {
-        fn visit_link(&mut self, _ctx: &NodeContext, href: &str, text: &str, _title: Option<&str>) -> VisitResult {
-            VisitResult::Custom(format!("{} ({})", text, href))
-        }
-
-        fn visit_image(&mut self, _ctx: &NodeContext, _src: &str, _alt: &str, _title: Option<&str>) -> VisitResult {
-            VisitResult::Skip
-        }
-    }
-
-    #[test]
-    fn test_custom_visitor_implementation() {
-        let mut visitor = CustomLinkVisitor;
-
-        let ctx = NodeContext {
-            node_type: super::super::types::NodeType::Link,
-            tag_name: "a".to_string(),
-            attributes: BTreeMap::new(),
-            depth: 1,
-            index_in_parent: 0,
-            parent_tag: Some("p".to_string()),
-            is_inline: true,
-        };
-
-        let result = visitor.visit_link(&ctx, "https://example.com", "Example", None);
-        if let VisitResult::Custom(output) = result {
-            assert_eq!(output, "Example (https://example.com)");
-        } else {
-            panic!("Expected Custom result");
-        }
-
-        let img_result = visitor.visit_image(&ctx, "image.jpg", "Alt text", None);
-        matches!(img_result, VisitResult::Skip);
-    }
-}