html-to-markdown 3.4.0 → 3.6.0.pre.rc.24

data/vendor/html-to-markdown-rs/src/converter/preprocessing_helpers.rs

@@ -1,201 +0,0 @@
-//! HTML preprocessing and validation helpers.
-//!
-//! This module contains helper functions for preprocessing HTML before conversion,
-//! including validation and normalization checks.
-
-use crate::converter::dom_context::DomContext;
-use crate::converter::main_helpers::is_inline_element;
-use crate::converter::utility::attributes::{attribute_matches_any, element_has_navigation_hint};
-use crate::converter::utility::content::normalized_tag_name;
-use crate::options::ConversionOptions;
-
-/// Check if an inline ancestor element is allowed to contain block-level elements.
-pub fn inline_ancestor_allows_block(tag_name: &str) -> bool {
-    matches!(tag_name, "a" | "ins" | "del")
-}
-
-/// Detect block elements that were incorrectly nested under inline ancestors.
-///
-/// Excludes elements inside `<pre>` or `<code>` blocks, as they have special
-/// whitespace preservation rules and should not be repaired.
-///
-/// Also detects table structural elements (`td`, `tr`, `th`) nested under `<p>` —
-/// a structural impossibility in valid HTML that signals the `tl` parser absorbed
-/// a table into a paragraph because of an unclosed `<p>` (common in Word/Outlook
-/// HTML such as `<p class='MsoNormal'>` cells). Issue #336.
-pub fn has_inline_block_misnest(dom_ctx: &DomContext, parser: &tl::Parser) -> bool {
-    for handle in dom_ctx.node_map.iter().flatten() {
-        if let Some(tl::Node::Tag(_tag)) = handle.get(parser) {
-            let node_id = handle.get_inner();
-            let Some(info) = dom_ctx.tag_info(node_id, parser) else {
-                continue;
-            };
-
-            // Table elements under <p>: tl misparsed an unclosed <p> in <td>.
-            if matches!(info.name.as_str(), "td" | "tr" | "th") && has_p_ancestor(dom_ctx, parser, node_id) {
-                return true;
-            }
-
-            if !info.is_block {
-                continue;
-            }
-
-            // Check if this block element or any ancestor is pre/code
-            let mut check_parent = Some(node_id);
-            let mut inside_preformatted = false;
-            while let Some(check_id) = check_parent {
-                if let Some(info) = dom_ctx.tag_info(check_id, parser) {
-                    if matches!(info.name.as_str(), "pre" | "code") {
-                        inside_preformatted = true;
-                        break;
-                    }
-                }
-                check_parent = dom_ctx.parent_of(check_id);
-            }
-
-            // Skip misnesting check for elements inside pre/code blocks
-            if inside_preformatted {
-                continue;
-            }
-
-            let mut current = dom_ctx.parent_of(node_id);
-            while let Some(parent_id) = current {
-                if let Some(parent_info) = dom_ctx.tag_info(parent_id, parser) {
-                    if is_inline_element(&parent_info.name) && !inline_ancestor_allows_block(&parent_info.name) {
-                        return true;
-                    }
-                } else if let Some(parent_handle) = dom_ctx.node_handle(parent_id) {
-                    if let Some(tl::Node::Tag(parent_tag)) = parent_handle.get(parser) {
-                        let parent_name = normalized_tag_name(parent_tag.name().as_utf8_str());
-                        if is_inline_element(&parent_name) && !inline_ancestor_allows_block(&parent_name) {
-                            return true;
-                        }
-                    }
-                }
-                current = dom_ctx.parent_of(parent_id);
-            }
-        }
-    }
-
-    false
-}
-
-/// Walk ancestors of `node_id` looking for a `<p>` element.
-///
-/// Stops ascending once it leaves the table hierarchy (`table`/`body`/`html`)
-/// to avoid false positives where a `<p>` legitimately wraps a `<table>`.
-fn has_p_ancestor(dom_ctx: &DomContext, parser: &tl::Parser, node_id: u32) -> bool {
-    let mut current = dom_ctx.parent_of(node_id);
-    while let Some(parent_id) = current {
-        if let Some(parent_info) = dom_ctx.tag_info(parent_id, parser) {
-            if parent_info.name == "p" {
-                return true;
-            }
-            if matches!(parent_info.name.as_str(), "table" | "body" | "html") {
-                return false;
-            }
-        }
-        current = dom_ctx.parent_of(parent_id);
-    }
-    false
-}
-
-/// Determine if a node should be dropped during preprocessing.
-///
-/// Behavior depends on the [`PreprocessingPreset`]:
-///
-/// - **Minimal**: Only scripts/styles are stripped (handled elsewhere). This function
-///   drops nothing — all structural elements are preserved.
-/// - **Standard** (default): Drops `<nav>` unconditionally. Drops `<header>`, `<footer>`,
-///   and `<aside>` only when they have navigation hints (class/role/aria attributes
-///   indicating site chrome). Drops `<form>` when `remove_forms` is enabled.
-/// - **Aggressive**: All of Standard, plus: drops `<footer>`, `<aside>`, `<noscript>`
-///   unconditionally. Drops ANY element with navigation hints in class/id/role
-///   (e.g. `<div class="sidebar">`). Drops elements with noise-related classes/roles.
-pub fn should_drop_for_preprocessing(tag_name: &str, tag: &tl::HTMLTag, options: &ConversionOptions) -> bool {
-    use crate::options::PreprocessingPreset;
-
-    if !options.preprocessing.enabled {
-        return false;
-    }
-
-    let preset = options.preprocessing.preset;
-
-    // Minimal preset: drop nothing here (scripts/styles handled in earlier pipeline stage).
-    if preset == PreprocessingPreset::Minimal {
-        return false;
-    }
-
-    // Form removal — applies to both Standard and Aggressive when enabled.
-    if options.preprocessing.remove_forms && tag_name == "form" {
-        return true;
-    }
-
-    let is_aggressive = preset == PreprocessingPreset::Aggressive;
-
-    // Aggressive: drop <noscript> — its content is fallback for no-JS browsers.
-    if is_aggressive && tag_name == "noscript" {
-        return true;
-    }
-
-    // Navigation removal — only when the flag is enabled.
-    if !options.preprocessing.remove_navigation {
-        return false;
-    }
-
-    let has_nav_hint = element_has_navigation_hint(tag);
-
-    // <nav> is always navigation — drop in both Standard and Aggressive.
-    if tag_name == "nav" {
-        return true;
-    }
-
-    if tag_name == "header" {
-        // Drop <header> only with navigation hints (e.g. class="site-header",
-        // role="navigation"). A plain <header> often wraps article titles like
-        // <header><h1>Title</h1></header> — dropping it loses content.
-        return has_nav_hint;
-    }
-
-    if tag_name == "footer" || tag_name == "aside" {
-        // Standard: drop only with navigation hints.
-        // Aggressive: drop unconditionally.
-        return is_aggressive || has_nav_hint;
-    }
-
-    // Aggressive: drop ANY element that has navigation hints in class/id/role.
-    // This catches <div class="sidebar">, <div class="menu">, <section class="navigation">,
-    // and similar non-semantic navigation containers.
-    if is_aggressive && has_nav_hint {
-        return true;
-    }
-
-    // Aggressive: drop elements with noise-related roles.
-    if is_aggressive {
-        if element_has_noise_hint(tag) {
-            return true;
-        }
-    }
-
-    false
-}
-
-/// Check if an element has noise-related hints (ads, cookie banners, social sharing).
-fn element_has_noise_hint(tag: &tl::HTMLTag) -> bool {
-    const NOISE_KEYWORDS: &[&str] = &[
-        "cookie",
-        "consent",
-        "gdpr",
-        "banner",
-        "advertisement",
-        "ad-container",
-        "advert",
-        "social-share",
-        "share-buttons",
-        "popup",
-        "modal-overlay",
-        "newsletter-signup",
-    ];
-
-    attribute_matches_any(tag, "class", NOISE_KEYWORDS) || attribute_matches_any(tag, "id", NOISE_KEYWORDS)
-}

data/vendor/html-to-markdown-rs/src/converter/reference_collector.rs

@@ -1,69 +0,0 @@
-//! Collector for reference-style link definitions.
-
-use std::cell::RefCell;
-use std::collections::HashMap;
-use std::rc::Rc;
-
-/// Shared handle for passing the collector through the conversion context.
-pub type ReferenceCollectorHandle = Rc<RefCell<ReferenceCollector>>;
-
-#[derive(Debug, Clone, Hash, Eq, PartialEq)]
-struct ReferenceKey {
-    url: String,
-    title: Option<String>,
-}
-
-/// Collects link/image references during conversion and produces a reference
-/// definitions section at the end of the document.
-#[derive(Debug, Default)]
-pub struct ReferenceCollector {
-    map: HashMap<ReferenceKey, usize>,
-    entries: Vec<(usize, String, Option<String>)>,
-}
-
-impl ReferenceCollector {
-    /// Create a new, empty reference collector.
-    pub fn new() -> Self {
-        Self::default()
-    }
-
-    /// Register a URL (and optional title) and return its 1-based reference number.
-    ///
-    /// If the same URL+title pair was already registered, the existing number is returned.
-    pub fn get_or_insert(&mut self, url: &str, title: Option<&str>) -> usize {
-        let key = ReferenceKey {
-            url: url.to_string(),
-            title: title.map(String::from),
-        };
-        if let Some(&num) = self.map.get(&key) {
-            return num;
-        }
-        let num = self.entries.len() + 1;
-        self.map.insert(key, num);
-        self.entries.push((num, url.to_string(), title.map(String::from)));
-        num
-    }
-
-    /// Produce the reference definitions section.
-    ///
-    /// Returns an empty string when no references were collected.
-    pub fn finish(&self) -> String {
-        if self.entries.is_empty() {
-            return String::new();
-        }
-        let mut out = String::new();
-        for (num, url, title) in &self.entries {
-            out.push('[');
-            out.push_str(&num.to_string());
-            out.push_str("]: ");
-            out.push_str(url);
-            if let Some(t) = title {
-                out.push_str(" \"");
-                out.push_str(&t.replace('"', "\\\""));
-                out.push('"');
-            }
-            out.push('\n');
-        }
-        out
-    }
-}

data/vendor/html-to-markdown-rs/src/converter/semantic/attributes.rs

@@ -1,269 +0,0 @@
-//! Handlers for semantic inline elements with attributes.
-//!
-//! Processes semantic inline elements that often carry semantic meaning through attributes:
-//! - `<cite>` - Citation/reference to a source
-//! - `<q>` - Inline quotation
-//! - `<abbr>` - Abbreviation with optional title explanation
-//! - `<dfn>` - Definition of a term
-//! - `<time>` - Machine-readable date/time
-//! - `<data>` - Machine-readable data value
-//!
-//! These elements carry semantic meaning in HTML5 but often have minimal
-//! visual distinction in rendered Markdown. Some are formatted with emphasis
-//! or have their attributes included in the output.
-
-// Note: Context and DomContext are defined in converter.rs
-// walk_node is also defined there and must be called via the parent module
-
-use crate::converter::utility::content::chomp_inline;
-
-/// Handles the `<dfn>` element.
-///
-/// A dfn element marks a term that is being defined. The content represents
-/// the term, and its definition would typically appear in surrounding context.
-/// It is rendered as emphasized (italic) text.
-///
-/// # Behavior
-///
-/// - Content is collected from children
-/// - Non-empty content is wrapped with the configured emphasis symbol (default: `*`)
-/// - Inline suffix handling is applied (e.g., footnote references)
-pub fn handle_dfn(
-    _tag_name: &str,
-    node_handle: &tl::NodeHandle,
-    parser: &tl::Parser,
-    output: &mut String,
-    options: &crate::options::ConversionOptions,
-    ctx: &super::Context,
-    depth: usize,
-    dom_ctx: &super::DomContext,
-) {
-    if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
-        let mut content = String::with_capacity(32);
-        let children = tag.children();
-        {
-            for child_handle in children.top().iter() {
-                super::walk_node(child_handle, parser, &mut content, options, ctx, depth + 1, dom_ctx);
-            }
-        }
-
-        let (prefix, suffix, trimmed) = chomp_inline(&content);
-        if !trimmed.is_empty() {
-            output.push_str(prefix);
-            output.push(options.strong_em_symbol);
-            output.push_str(trimmed);
-            output.push(options.strong_em_symbol);
-            append_inline_suffix(output, suffix, !trimmed.is_empty(), node_handle, parser, dom_ctx);
-        }
-    }
-}
-
-/// Handles the `<abbr>` element.
-///
-/// An abbr element marks an abbreviation or acronym. The `title` attribute
-/// provides the expansion of the abbreviation, which is appended in parentheses
-/// if present.
-///
-/// # Behavior
-///
-/// - Content is collected from children
-/// - Non-empty content is output as-is
-/// - If `title` attribute exists, it is appended in parentheses: `abbr (title)`
-///
-/// # Example
-///
-/// ```html
-/// <abbr title="HyperText Markup Language">HTML</abbr>
-/// ```
-///
-/// Produces: `HTML (HyperText Markup Language)`
-pub fn handle_abbr(
-    _tag_name: &str,
-    node_handle: &tl::NodeHandle,
-    parser: &tl::Parser,
-    output: &mut String,
-    options: &crate::options::ConversionOptions,
-    ctx: &super::Context,
-    depth: usize,
-    dom_ctx: &super::DomContext,
-) {
-    if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
-        let mut content = String::with_capacity(32);
-        let children = tag.children();
-        {
-            for child_handle in children.top().iter() {
-                super::walk_node(child_handle, parser, &mut content, options, ctx, depth + 1, dom_ctx);
-            }
-        }
-
-        let trimmed = content.trim();
-        if !trimmed.is_empty() {
-            output.push_str(trimmed);
-
-            // Append title attribute if present
-            if let Some(title) = tag.attributes().get("title").flatten().map(|v| v.as_utf8_str()) {
-                let trimmed_title = title.trim();
-                if !trimmed_title.is_empty() {
-                    output.push_str(" (");
-                    output.push_str(trimmed_title);
-                    output.push(')');
-                }
-            }
-        }
-    }
-}
-
-/// Handles the `<time>` and `<data>` elements.
-///
-/// Time and data elements contain machine-readable content in their attributes
-/// and human-readable content in their text. For Markdown purposes, we output
-/// only the human-readable text content, as Markdown doesn't have a way to
-/// preserve machine-readable metadata.
-///
-/// # Behavior
-///
-/// - Content is extracted from children and output as-is
-/// - Attributes (datetime, value) are not rendered in Markdown output
-pub fn handle_time_data(
-    _tag_name: &str,
-    node_handle: &tl::NodeHandle,
-    parser: &tl::Parser,
-    output: &mut String,
-    options: &crate::options::ConversionOptions,
-    ctx: &super::Context,
-    depth: usize,
-    dom_ctx: &super::DomContext,
-) {
-    if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
-        let children = tag.children();
-        {
-            for child_handle in children.top().iter() {
-                super::walk_node(child_handle, parser, output, options, ctx, depth + 1, dom_ctx);
-            }
-        }
-    }
-}
-
-/// Handles the `<cite>` element.
-///
-/// A cite element marks the title of a cited work (book, article, website, etc.).
-/// It is rendered as emphasized (italic) text in block mode, or as plain text in inline mode.
-///
-/// # Behavior
-///
-/// - **Block mode**: Content is wrapped with emphasis markers (default: `*`)
-/// - **Inline mode**: Content is output as-is without formatting
-pub fn handle_cite(
-    _tag_name: &str,
-    node_handle: &tl::NodeHandle,
-    parser: &tl::Parser,
-    output: &mut String,
-    options: &crate::options::ConversionOptions,
-    ctx: &super::Context,
-    depth: usize,
-    dom_ctx: &super::DomContext,
-) {
-    if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
-        let mut content = String::with_capacity(32);
-        let children = tag.children();
-        {
-            for child_handle in children.top().iter() {
-                super::walk_node(child_handle, parser, &mut content, options, ctx, depth + 1, dom_ctx);
-            }
-        }
-
-        let trimmed = content.trim();
-        if !trimmed.is_empty() {
-            if ctx.convert_as_inline {
-                output.push_str(trimmed);
-            } else {
-                output.push('*');
-                output.push_str(trimmed);
-                output.push('*');
-            }
-        }
-    }
-}
-
-/// Handles the `<q>` element.
-///
-/// A q element marks an inline quotation. In Markdown, it is rendered as
-/// quoted text enclosed in double quotes. Backslashes and quotes within
-/// the content are escaped.
-///
-/// # Behavior
-///
-/// - **Block mode**: Content is wrapped in escaped double quotes: `"content"`
-/// - **Inline mode**: Content is output as-is without quotes
-///
-/// # Escaping
-///
-/// Internal backslashes and double quotes are escaped:
-/// - `\` → `\\`
-/// - `"` → `\"`
-pub fn handle_q(
-    _tag_name: &str,
-    node_handle: &tl::NodeHandle,
-    parser: &tl::Parser,
-    output: &mut String,
-    options: &crate::options::ConversionOptions,
-    ctx: &super::Context,
-    depth: usize,
-    dom_ctx: &super::DomContext,
-) {
-    if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
-        let mut content = String::with_capacity(32);
-        let children = tag.children();
-        {
-            for child_handle in children.top().iter() {
-                super::walk_node(child_handle, parser, &mut content, options, ctx, depth + 1, dom_ctx);
-            }
-        }
-
-        let trimmed = content.trim();
-        if !trimmed.is_empty() {
-            output.push('"');
-            output.push_str(trimmed);
-            output.push('"');
-        }
-    }
-}
-
-/// Dispatcher for semantic inline attribute elements.
-///
-/// Routes `<cite>`, `<q>`, `<abbr>`, `<dfn>`, `<time>`, and `<data>` elements
-/// to their respective handlers.
-pub fn handle(
-    tag_name: &str,
-    node_handle: &tl::NodeHandle,
-    parser: &tl::Parser,
-    output: &mut String,
-    options: &crate::options::ConversionOptions,
-    ctx: &super::Context,
-    depth: usize,
-    dom_ctx: &super::DomContext,
-) {
-    match tag_name {
-        "dfn" => handle_dfn(tag_name, node_handle, parser, output, options, ctx, depth, dom_ctx),
-        "abbr" => handle_abbr(tag_name, node_handle, parser, output, options, ctx, depth, dom_ctx),
-        "time" | "data" => handle_time_data(tag_name, node_handle, parser, output, options, ctx, depth, dom_ctx),
-        "cite" => handle_cite(tag_name, node_handle, parser, output, options, ctx, depth, dom_ctx),
-        "q" => handle_q(tag_name, node_handle, parser, output, options, ctx, depth, dom_ctx),
-        _ => {}
-    }
-}
-
-/// Appends inline suffix to the output.
-///
-/// This is a placeholder for integrating with other inline formatting systems
-/// (e.g., footnote references). For now, it simply outputs the suffix.
-fn append_inline_suffix(
-    output: &mut String,
-    suffix: &str,
-    _is_nonempty: bool,
-    _node_handle: &tl::NodeHandle,
-    _parser: &tl::Parser,
-    _dom_ctx: &super::DomContext,
-) {
-    output.push_str(suffix);
-}