html-to-markdown 3.4.0 → 3.6.0.pre.rc.23

data/vendor/html-to-markdown-rs/src/converter/text_node.rs

@@ -1,269 +0,0 @@
-//! Text node processing for HTML to Markdown conversion.
-//!
-//! Handles raw text nodes with:
-//! - HTML entity decoding
-//! - Whitespace normalization and stripping
-//! - Text escaping with configurable escape modes
-//! - Visitor callbacks (when feature enabled)
-//! - List item indentation
-
-use std::borrow::Cow;
-
-use crate::converter::dom_context::DomContext;
-use crate::converter::main_helpers::{has_more_than_one_char, is_inline_element};
-use crate::converter::utility::siblings::{
-    get_next_sibling_tag, next_sibling_is_inline_tag, previous_sibling_is_inline_tag,
-};
-use crate::options::ConversionOptions;
-use crate::text;
-
-// Type aliases for Context to avoid circular imports
-type Context = crate::converter::Context;
-
-/// Process a raw text node during HTML to Markdown conversion.
-///
-/// Handles:
-/// - HTML entity decoding
-/// - Whitespace normalization and stripping
-/// - Text escaping with configurable escape modes
-/// - Visitor callbacks (when feature enabled)
-/// - List item indentation
-#[allow(clippy::too_many_lines)]
-#[cfg_attr(not(feature = "visitor"), allow(unused_variables))]
-pub fn process_text_node(
-    raw: &str,
-    node_handle: &tl::NodeHandle,
-    parser: &tl::Parser,
-    output: &mut String,
-    options: &ConversionOptions,
-    ctx: &Context,
-    depth: usize,
-    dom_ctx: &DomContext,
-) {
-    let mut text = text::decode_html_entities_cow(raw);
-
-    if text.is_empty() {
-        return;
-    }
-
-    let text_ref = text.as_ref();
-    let had_newlines = text_ref.contains('\n');
-    let has_double_newline = text_ref.contains("\n\n") || text_ref.contains("\r\n\r\n");
-
-    if options.strip_newlines && (text.contains('\r') || text.contains('\n')) {
-        text = Cow::Owned(text.replace(['\r', '\n'], " "));
-    }
-
-    if text.trim().is_empty() {
-        if ctx.in_code {
-            output.push_str(text.as_ref());
-            return;
-        }
-
-        if options.whitespace_mode == crate::options::WhitespaceMode::Strict {
-            if ctx.convert_as_inline || ctx.in_table_cell || ctx.in_list_item {
-                output.push_str(text.as_ref());
-                return;
-            }
-            if has_double_newline {
-                if !output.ends_with("\n\n") {
-                    output.push('\n');
-                }
-                return;
-            }
-            output.push_str(text.as_ref());
-            return;
-        }
-
-        if had_newlines {
-            if output.is_empty() {
-                return;
-            }
-            if !output.ends_with("\n\n") {
-                if let Some(next_tag) = get_next_sibling_tag(node_handle, parser, dom_ctx) {
-                    if is_inline_element(next_tag) {
-                        // Newlines between inline elements collapse to a single space
-                        // in HTML rendering (per CSS white-space: normal). Preserve
-                        // this word boundary so adjacent inline content doesn't merge.
-                        if !output.ends_with(' ') && !output.ends_with('\n') {
-                            output.push(' ');
-                        }
-                        return;
-                    }
-                }
-            }
-            return;
-        }
-
-        if previous_sibling_is_inline_tag(node_handle, parser, dom_ctx)
-            && next_sibling_is_inline_tag(node_handle, parser, dom_ctx)
-        {
-            if has_more_than_one_char(text.as_ref()) {
-                if !output.ends_with(' ') {
-                    output.push(' ');
-                }
-            } else {
-                output.push_str(text.as_ref());
-            }
-        } else {
-            output.push_str(text.as_ref());
-        }
-        return;
-    }
-
-    let processed_text = if ctx.in_code || ctx.in_ruby {
-        text.into_owned()
-    } else if ctx.in_table_cell {
-        // Always escape * and _ in table cells to prevent unintended emphasis.
-        let escaped = if options.whitespace_mode == crate::options::WhitespaceMode::Normalized {
-            let normalized_text = text::normalize_whitespace_cow(text.as_ref());
-            let escaped_result = text::escape(
-                normalized_text.as_ref(),
-                options.escape_misc,
-                true,
-                true,
-                options.escape_ascii,
-            );
-            escaped_result.into_owned()
-        } else {
-            text::escape(text.as_ref(), options.escape_misc, true, true, options.escape_ascii).into_owned()
-        };
-        if options.escape_misc {
-            escaped
-        } else {
-            escaped.replace('|', r"\|")
-        }
-    } else if options.whitespace_mode == crate::options::WhitespaceMode::Strict {
-        text::escape(
-            text.as_ref(),
-            options.escape_misc,
-            options.escape_asterisks,
-            options.escape_underscores,
-            options.escape_ascii,
-        )
-        .into_owned()
-    } else {
-        let has_double_newline = text.contains("\n\n") || text.contains("\r\n\r\n");
-        let has_trailing_single_newline =
-            text.ends_with('\n') && !text.ends_with("\n\n") && !text.ends_with("\r\n\r\n");
-
-        let normalized_text = text::normalize_whitespace_cow(text.as_ref());
-
-        let (prefix, suffix, core) = text::chomp(normalized_text.as_ref());
-
-        let skip_prefix = output.ends_with("\n\n")
-            || output.ends_with("* ")
-            || output.ends_with("- ")
-            || output.ends_with(". ")
-            || output.ends_with("] ")
-            || (output.ends_with('\n') && prefix == " ")
-            || (output.ends_with(' ')
-                && prefix == " "
-                && !previous_sibling_is_inline_tag(node_handle, parser, dom_ctx));
-
-        let mut final_text = String::with_capacity(prefix.len() + core.len() + suffix.len() + 2);
-        if !skip_prefix && !prefix.is_empty() {
-            final_text.push_str(prefix);
-        }
-
-        let escaped_core = text::escape(
-            core,
-            options.escape_misc,
-            options.escape_asterisks,
-            options.escape_underscores,
-            options.escape_ascii,
-        );
-        final_text.push_str(&escaped_core);
-
-        if !suffix.is_empty() {
-            final_text.push_str(suffix);
-        } else if has_trailing_single_newline {
-            // Check if the "\n\n" at the end of the output buffer came from within
-            // the current block's content, not from a previous block's closing.
-            // Without this distinction, the second paragraph after a "\n\n" boundary
-            // would incorrectly suppress the trailing space before inline elements.
-            let safe_start = ctx.block_content_start.min(output.len());
-            let safe_start = crate::converter::utility::content::floor_char_boundary(output, safe_start);
-            let current_block_output = &output[safe_start..];
-            let at_paragraph_break = current_block_output.ends_with("\n\n");
-            if !at_paragraph_break {
-                if has_double_newline {
-                    final_text.push('\n');
-                } else if let Some(next_tag) = get_next_sibling_tag(node_handle, parser, dom_ctx) {
-                    if matches!(next_tag, "span") {
-                    } else if ctx.inline_depth > 0 || ctx.convert_as_inline || ctx.in_paragraph {
-                        final_text.push(' ');
-                    } else {
-                        final_text.push('\n');
-                    }
-                } else if ctx.inline_depth > 0 || ctx.convert_as_inline || ctx.in_paragraph {
-                    final_text.push(' ');
-                } else {
-                    final_text.push('\n');
-                }
-            }
-        }
-
-        final_text
-    };
-
-    #[cfg(feature = "visitor")]
-    let final_text = if let Some(ref visitor_handle) = ctx.visitor {
-        use crate::visitor::{NodeContext, NodeType, VisitResult};
-        use std::collections::BTreeMap;
-
-        let node_id = node_handle.get_inner();
-        let parent_tag = dom_ctx.parent_tag_name(node_id, parser);
-        let index_in_parent = dom_ctx.get_sibling_index(node_id).unwrap_or(0);
-
-        let node_ctx = NodeContext {
-            node_type: NodeType::Text,
-            tag_name: String::new(),
-            attributes: BTreeMap::new(),
-            depth,
-            index_in_parent,
-            parent_tag,
-            is_inline: true,
-        };
-
-        let mut visitor = visitor_handle.borrow_mut();
-        match visitor.visit_text(&node_ctx, &processed_text) {
-            VisitResult::Continue => processed_text,
-            VisitResult::Custom(custom) => {
-                if ctx.inline_depth > 0 || ctx.in_heading {
-                    processed_text
-                } else {
-                    custom
-                }
-            }
-            VisitResult::Skip => return,
-            VisitResult::Error(err) => {
-                if ctx.visitor_error.borrow().is_none() {
-                    *ctx.visitor_error.borrow_mut() = Some(err);
-                }
-                return;
-            }
-            VisitResult::PreserveHtml => processed_text,
-        }
-    } else {
-        processed_text
-    };
-
-    #[cfg(not(feature = "visitor"))]
-    let final_text = processed_text;
-
-    if ctx.in_list_item && final_text.contains("\n\n") {
-        let indent = " ".repeat(4 * ctx.list_depth);
-        let mut first = true;
-        for part in final_text.split("\n\n") {
-            if !first {
-                output.push_str("\n\n");
-                output.push_str(&indent);
-            }
-            first = false;
-            output.push_str(part.trim());
-        }
-    } else {
-        output.push_str(&final_text);
-    }
-}

data/vendor/html-to-markdown-rs/src/converter/utility/attributes.rs

@@ -1,151 +0,0 @@
-//! Attribute handling and extraction utilities.
-//!
-//! Functions for working with element attributes, semantic detection, and hOCR document detection.
-
-use crate::converter::DomContext;
-use crate::converter::utility::content::normalized_tag_name;
-
-/// Check if a tag has main content semantics based on role or class.
-pub fn tag_has_main_semantics(tag: &tl::HTMLTag) -> bool {
-    if let Some(Some(role)) = tag.attributes().get("role") {
-        let lowered = role.as_utf8_str().to_ascii_lowercase();
-        if matches!(lowered.as_str(), "main" | "article" | "document" | "region") {
-            return true;
-        }
-    }
-
-    if let Some(Some(class_bytes)) = tag.attributes().get("class") {
-        let class_value = class_bytes.as_utf8_str().to_ascii_lowercase();
-        const MAIN_CLASS_HINTS: &[&str] = &[
-            "mw-body",
-            "mw-parser-output",
-            "content-body",
-            "content-container",
-            "article-body",
-            "article-content",
-            "main-content",
-            "page-content",
-            "entry-content",
-            "post-content",
-            "document-body",
-        ];
-        if MAIN_CLASS_HINTS.iter().any(|hint| class_value.contains(hint)) {
-            return true;
-        }
-    }
-
-    false
-}
-
-/// Check if an element has navigation-related hints in its attributes.
-pub fn element_has_navigation_hint(tag: &tl::HTMLTag) -> bool {
-    if attribute_matches_any(tag, "role", &["navigation", "menubar", "tablist", "toolbar"]) {
-        return true;
-    }
-
-    if attribute_contains_any(
-        tag,
-        "aria-label",
-        &["navigation", "menu", "contents", "table of contents", "toc"],
-    ) {
-        return true;
-    }
-
-    const NAV_KEYWORDS: &[&str] = &[
-        "nav",
-        "navigation",
-        "navbar",
-        "breadcrumbs",
-        "breadcrumb",
-        "toc",
-        "sidebar",
-        "sidenav",
-        "menu",
-        "menubar",
-        "mainmenu",
-        "subnav",
-        "tabs",
-        "tablist",
-        "toolbar",
-        "pager",
-        "pagination",
-        "skipnav",
-        "skip-link",
-        "skiplinks",
-        "site-nav",
-        "site-menu",
-        "site-header",
-        "site-footer",
-        "topbar",
-        "bottombar",
-        "masthead",
-        "vector-nav",
-        "vector-header",
-        "vector-footer",
-    ];
-
-    attribute_matches_any(tag, "class", NAV_KEYWORDS) || attribute_matches_any(tag, "id", NAV_KEYWORDS)
-}
-
-/// Check if an attribute value matches any of the given keywords (space or custom-separator aware).
-pub fn attribute_matches_any(tag: &tl::HTMLTag, attr: &str, keywords: &[&str]) -> bool {
-    let Some(attr_value) = tag.attributes().get(attr) else {
-        return false;
-    };
-    let Some(value) = attr_value else {
-        return false;
-    };
-    let raw = value.as_utf8_str();
-    raw.split_whitespace()
-        .map(|token| {
-            token
-                .chars()
-                .map(|c| match c {
-                    '_' | ':' | '.' | '/' => '-',
-                    _ => c,
-                })
-                .collect::<String>()
-                .to_ascii_lowercase()
-        })
-        .filter(|token| !token.is_empty())
-        .any(|token| keywords.iter().any(|kw| token == *kw))
-}
-
-/// Check if an attribute contains any of the given keywords (substring match).
-#[allow(clippy::trivially_copy_pass_by_ref)]
-pub fn attribute_contains_any(tag: &tl::HTMLTag, attr: &str, keywords: &[&str]) -> bool {
-    let Some(attr_value) = tag.attributes().get(attr) else {
-        return false;
-    };
-    let Some(value) = attr_value else {
-        return false;
-    };
-    let lower = value.as_utf8_str().to_ascii_lowercase();
-    keywords.iter().any(|kw| lower.contains(*kw))
-}
-
-/// Check if a node has a semantic content ancestor (main, article, section).
-#[allow(clippy::trivially_copy_pass_by_ref)]
-pub fn has_semantic_content_ancestor(node_handle: &tl::NodeHandle, parser: &tl::Parser, dom_ctx: &DomContext) -> bool {
-    let mut current_id = node_handle.get_inner();
-    while let Some(parent_id) = dom_ctx.parent_of(current_id) {
-        if let Some(parent_info) = dom_ctx.tag_info(parent_id, parser) {
-            if matches!(parent_info.name.as_str(), "main" | "article" | "section") {
-                return true;
-            }
-        }
-        if let Some(parent_handle) = dom_ctx.node_handle(parent_id) {
-            if let Some(tl::Node::Tag(parent_tag)) = parent_handle.get(parser) {
-                let parent_name = normalized_tag_name(parent_tag.name().as_utf8_str());
-                if matches!(parent_name.as_ref(), "main" | "article" | "section") {
-                    return true;
-                }
-                if tag_has_main_semantics(parent_tag) {
-                    return true;
-                }
-            }
-        }
-        current_id = parent_id;
-    }
-    false
-}

data/vendor/html-to-markdown-rs/src/converter/utility/caching.rs

@@ -1,74 +0,0 @@
-//! Performance caching utilities.
-//!
-//! Caching mechanisms for expensive operations during conversion, including
-//! DOM context building and cache capacity management.
-
-use crate::converter::DomContext;
-use std::num::NonZeroUsize;
-
-/// Build a DOM context with hierarchical node information.
-///
-/// Pre-computes parent-child relationships, sibling indices, and caches
-/// tag information for efficient DOM navigation during conversion.
-pub fn build_dom_context(dom: &tl::VDom, parser: &tl::Parser, input_len: usize) -> DomContext {
-    let cache_capacity = text_cache_capacity_for_input(input_len);
-    let mut ctx = DomContext {
-        parent_map: Vec::new(),
-        children_map: Vec::new(),
-        sibling_index_map: Vec::new(),
-        root_children: dom.children().to_vec(),
-        node_map: Vec::new(),
-        tag_info_map: Vec::new(),
-        prev_inline_like_map: Vec::new(),
-        next_inline_like_map: Vec::new(),
-        next_tag_map: Vec::new(),
-        next_whitespace_map: Vec::new(),
-        text_cache: std::cell::RefCell::new(lru::LruCache::new(cache_capacity)),
-    };
-
-    for (index, child_handle) in dom.children().iter().enumerate() {
-        let id = child_handle.get_inner();
-        ctx.ensure_capacity(id);
-        ctx.sibling_index_map[id as usize] = Some(index);
-        record_node_hierarchy(*child_handle, None, parser, &mut ctx);
-    }
-
-    ctx
-}
-
-/// Calculate appropriate cache capacity based on input size.
-///
-/// Returns a cache capacity between 32 and TEXT_CACHE_CAPACITY,
-/// scaled proportionally to input size (1KB = 1 slot).
-pub fn text_cache_capacity_for_input(input_len: usize) -> NonZeroUsize {
-    const TEXT_CACHE_CAPACITY: usize = 256;
-    // `clamp(32, TEXT_CACHE_CAPACITY)` guarantees `target >= 32 > 0`, so `new` always returns Some.
-    let target = (input_len / 1024).clamp(32, TEXT_CACHE_CAPACITY);
-    NonZeroUsize::new(target).unwrap_or(NonZeroUsize::MIN)
-}
-
-/// Recursively record node hierarchy into DOM context.
-///
-/// Builds the complete parent-child relationship map for efficient tree traversal.
-pub fn record_node_hierarchy(
-    node_handle: tl::NodeHandle,
-    parent: Option<u32>,
-    parser: &tl::Parser,
-    ctx: &mut DomContext,
-) {
-    let id = node_handle.get_inner();
-    ctx.ensure_capacity(id);
-    ctx.parent_map[id as usize] = parent;
-    ctx.node_map[id as usize] = Some(node_handle);
-
-    if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
-        let children: Vec<_> = tag.children().top().iter().copied().collect();
-        for (index, child) in children.iter().enumerate() {
-            let child_id = child.get_inner();
-            ctx.ensure_capacity(child_id);
-            ctx.sibling_index_map[child_id as usize] = Some(index);
-            record_node_hierarchy(*child, Some(id), parser, ctx);
-        }
-        ctx.children_map[id as usize] = Some(children);
-    }
-}