html-to-markdown 2.29.0 → 3.0.0

data/vendor/html-to-markdown-rs/src/converter/text/escaping.rs

@@ -19,42 +19,7 @@ use std::borrow::Cow;
 /// "Text \\[escaped\\]" → "Text \\[escaped\\]"
 /// ```
 pub fn escape_link_label(text: &str) -> String {
-    if text.is_empty() {
-        return String::new();
-    }
-
-    let mut result = String::with_capacity(text.len());
-    let mut backslash_count = 0usize;
-    let mut bracket_depth = 0usize;
-
-    for ch in text.chars() {
-        if ch == '\\' {
-            result.push('\\');
-            backslash_count += 1;
-            continue;
-        }
-
-        let is_escaped = backslash_count % 2 == 1;
-        backslash_count = 0;
-
-        match ch {
-            '[' if !is_escaped => {
-                bracket_depth = bracket_depth.saturating_add(1);
-                result.push('[');
-            }
-            ']' if !is_escaped => {
-                if bracket_depth == 0 {
-                    result.push('\\');
-                } else {
-                    bracket_depth -= 1;
-                }
-                result.push(']');
-            }
-            _ => result.push(ch),
-        }
-    }
-
-    result
+    crate::converter::utility::content::escape_link_label(text)
 }
 
 /// Escape malformed angle brackets in markdown output.

data/vendor/html-to-markdown-rs/src/converter/text/mod.rs

@@ -8,7 +8,5 @@ mod normalization;
 mod processing;
 
 pub use escaping::{escape_link_label, escape_malformed_angle_brackets};
-pub use normalization::{
-    chomp_inline, normalize_heading_text, trim_line_end_whitespace, trim_trailing_whitespace, truncate_at_char_boundary,
-};
+pub use normalization::{normalize_heading_text, trim_line_end_whitespace, truncate_at_char_boundary};
 pub use processing::dedent_code_block;

data/vendor/html-to-markdown-rs/src/converter/text/normalization.rs

@@ -5,59 +5,6 @@
 
 use std::borrow::Cow;
 
-/// Chomp whitespace from inline element content, preserving line breaks.
-///
-/// Returns (prefix, suffix, trimmed_text) where:
-/// - prefix: leading whitespace (space or tab)
-/// - suffix: trailing whitespace (including soft breaks like "  \n" or "\\\n")
-/// - trimmed_text: the trimmed content
-///
-/// # Examples
-///
-/// ```text
-/// " text  \n" → (" ", "  \n", "text")
-/// "  text  " → (" ", " ", "text")
-/// "text" → ("", "", "text")
-/// ```
-pub fn chomp_inline(text: &str) -> (&str, &str, &str) {
-    if text.is_empty() {
-        return ("", "", "");
-    }
-
-    let prefix = if text.starts_with(&[' ', '\t'][..]) { " " } else { "" };
-
-    let has_trailing_linebreak = text.ends_with("  \n") || text.ends_with("\\\n");
-
-    let suffix = if has_trailing_linebreak {
-        if text.ends_with("  \n") { "  \n" } else { "\\\n" }
-    } else if text.ends_with(&[' ', '\t'][..]) {
-        " "
-    } else {
-        ""
-    };
-
-    let trimmed = if has_trailing_linebreak {
-        text.strip_suffix("  \n").map_or_else(
-            || text.strip_suffix("\\\n").map_or_else(|| text.trim(), |s| s.trim()),
-            |s| s.trim(),
-        )
-    } else {
-        text.trim()
-    };
-
-    (prefix, suffix, trimmed)
-}
-
-/// Remove trailing spaces and tabs from output string.
-///
-/// This is used before adding block separators or newlines to ensure
-/// clean Markdown output without spurious whitespace.
-pub fn trim_trailing_whitespace(output: &mut String) {
-    while output.ends_with(' ') || output.ends_with('\t') {
-        output.pop();
-    }
-}
-
 /// Remove trailing spaces/tabs from every line while preserving newlines.
 pub fn trim_line_end_whitespace(output: &mut String) {
     if output.is_empty() {

data/vendor/html-to-markdown-rs/src/converter/text_node.rs

@@ -50,7 +50,7 @@ pub fn process_text_node(
     let had_newlines = text_ref.contains('\n');
     let has_double_newline = text_ref.contains("\n\n") || text_ref.contains("\r\n\r\n");
 
-    if options.strip_newlines {
+    if options.strip_newlines && (text.contains('\r') || text.contains('\n')) {
         text = Cow::Owned(text.replace(['\r', '\n'], " "));
     }
 

data/vendor/html-to-markdown-rs/src/converter/utility/attributes.rs

@@ -153,44 +153,3 @@ pub(crate) fn has_semantic_content_ancestor(
     }
     false
 }
-
-/// Check if a document might be an hOCR document (has relevant attributes).
-pub(crate) fn may_be_hocr(input: &str) -> bool {
-    const HOCR_MARKERS: [&[u8]; 3] = [b"class=\"ocr", b"class='ocr", b"ocr_page"];
-    HOCR_MARKERS
-        .iter()
-        .any(|marker| input.as_bytes().windows(marker.len()).any(|w| w == *marker))
-}
-
-/// Check if a node is an hOCR document by examining its root tag.
-pub(crate) fn is_hocr_document(node_handle: tl::NodeHandle, parser: &tl::Parser) -> bool {
-    if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
-        let tag_name = tag.name().as_utf8_str();
-        if tag_name != "html" {
-            return false;
-        }
-
-        // Check for hOCR class on root or first child
-        if let Some(Some(class_bytes)) = tag.attributes().get("class") {
-            let class = class_bytes.as_utf8_str();
-            if class.contains("ocr_document") || class.contains("ocr_page") {
-                return true;
-            }
-        }
-
-        // Check children
-        let children = tag.children();
-        for child_handle in children.top().iter() {
-            if let Some(tl::Node::Tag(child_tag)) = child_handle.get(parser) {
-                if let Some(Some(class_bytes)) = child_tag.attributes().get("class") {
-                    let class = class_bytes.as_utf8_str();
-                    if class.contains("ocr_document") || class.contains("ocr_page") {
-                        return true;
-                    }
-                }
-            }
-        }
-    }
-
-    false
-}

data/vendor/html-to-markdown-rs/src/converter/utility/caching.rs

@@ -42,8 +42,9 @@ pub(crate) fn build_dom_context(dom: &tl::VDom, parser: &tl::Parser, input_len:
 /// scaled proportionally to input size (1KB = 1 slot).
 pub(crate) fn text_cache_capacity_for_input(input_len: usize) -> NonZeroUsize {
     const TEXT_CACHE_CAPACITY: usize = 256;
+    // `clamp(32, TEXT_CACHE_CAPACITY)` guarantees `target >= 32 > 0`, so `new` always returns Some.
     let target = (input_len / 1024).clamp(32, TEXT_CACHE_CAPACITY);
-    NonZeroUsize::new(target).unwrap_or_else(|| NonZeroUsize::new(32).unwrap())
+    NonZeroUsize::new(target).unwrap_or(NonZeroUsize::MIN)
 }
 
 /// Recursively record node hierarchy into DOM context.

data/vendor/html-to-markdown-rs/src/converter/utility/content.rs

@@ -5,10 +5,24 @@
 
 use crate::text;
 use std::borrow::Cow;
+#[cfg(feature = "visitor")]
+use std::collections::BTreeMap;
 
 // Forward declare DomContext from parent module to avoid circular imports
 pub(crate) use crate::converter::DomContext;
 
+/// Collect all attributes from an HTML tag as a `BTreeMap<String, String>`.
+///
+/// Boolean attributes (those with `None` as the value) are skipped; only
+/// attributes that carry an explicit value are included.
+#[cfg(feature = "visitor")]
+pub(crate) fn collect_tag_attributes(tag: &tl::HTMLTag) -> BTreeMap<String, String> {
+    tag.attributes()
+        .iter()
+        .filter_map(|(k, v)| v.as_ref().map(|val| (k.to_string(), val.to_string())))
+        .collect()
+}
+
 /// Chomp whitespace from inline element content, preserving line breaks.
 ///
 /// Similar to `text::chomp` but handles line breaks from `<br>` tags specially.
@@ -131,31 +145,6 @@ pub(crate) fn normalize_link_label(label: &str) -> String {
     normalized.as_ref().trim().to_string()
 }
 
-/// Check if an inline element is considered empty (no meaningful content).
-pub(crate) fn is_empty_inline_element(node_handle: &tl::NodeHandle, parser: &tl::Parser, dom_ctx: &DomContext) -> bool {
-    const EMPTY_WHEN_NO_CONTENT_TAGS: &[&str] = &[
-        "abbr", "var", "ins", "dfn", "time", "data", "cite", "q", "mark", "small", "u",
-    ];
-
-    let tag_name: Option<Cow<'_, str>> = dom_ctx
-        .tag_info(node_handle.get_inner(), parser)
-        .map(|info| Cow::Borrowed(info.name.as_str()))
-        .or_else(|| {
-            if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
-                Some(normalized_tag_name(tag.name().as_utf8_str()))
-            } else {
-                None
-            }
-        });
-
-    if let Some(tag_name) = tag_name {
-        if EMPTY_WHEN_NO_CONTENT_TAGS.contains(&tag_name.as_ref()) {
-            return get_text_content(node_handle, parser, dom_ctx).trim().is_empty();
-        }
-    }
-    false
-}
-
 /// Normalize a tag name to lowercase, preserving borrowed input when possible.
 pub(crate) fn normalized_tag_name(raw: Cow<'_, str>) -> Cow<'_, str> {
     if raw.as_bytes().iter().any(u8::is_ascii_uppercase) {
@@ -167,81 +156,9 @@ pub(crate) fn normalized_tag_name(raw: Cow<'_, str>) -> Cow<'_, str> {
     }
 }
 
-/// Check if an element is inline (not block-level).
-fn is_inline_element(tag_name: &str) -> bool {
-    matches!(
-        tag_name,
-        "a" | "abbr"
-            | "b"
-            | "bdi"
-            | "bdo"
-            | "br"
-            | "cite"
-            | "code"
-            | "data"
-            | "dfn"
-            | "em"
-            | "i"
-            | "kbd"
-            | "mark"
-            | "q"
-            | "rp"
-            | "rt"
-            | "ruby"
-            | "s"
-            | "samp"
-            | "small"
-            | "span"
-            | "strong"
-            | "sub"
-            | "sup"
-            | "time"
-            | "u"
-            | "var"
-            | "wbr"
-            | "del"
-            | "ins"
-            | "img"
-            | "map"
-            | "area"
-            | "audio"
-            | "video"
-            | "picture"
-            | "source"
-            | "track"
-            | "embed"
-            | "object"
-            | "param"
-            | "input"
-            | "label"
-            | "button"
-            | "select"
-            | "textarea"
-            | "output"
-            | "progress"
-            | "meter"
-    )
-}
-
 /// Check if an element is block-level (not inline).
 pub(crate) fn is_block_level_element(tag_name: &str) -> bool {
-    is_block_level_name(tag_name, is_inline_element(tag_name))
-}
-
-/// Truncate a string to a maximum length at a valid UTF-8 character boundary.
-///
-/// Ensures the string is not longer than `max_len` bytes, truncating at the last
-/// valid character boundary if necessary to preserve valid UTF-8.
-pub(crate) fn truncate_at_char_boundary(value: &mut String, max_len: usize) {
-    if value.len() <= max_len {
-        return;
-    }
-
-    let mut new_len = max_len.min(value.len());
-    while new_len > 0 && !value.is_char_boundary(new_len) {
-        new_len -= 1;
-    }
-    value.truncate(new_len);
+    is_block_level_name(tag_name, crate::converter::main_helpers::is_inline_element(tag_name))
 }
 
 /// Returns the largest valid char boundary index at or before `index`.

data/vendor/html-to-markdown-rs/src/converter/utility/preprocessing.rs

@@ -176,10 +176,13 @@ pub(crate) fn find_closing_tag_bytes(bytes: &[u8], start: usize, tag: &[u8]) ->
     const MAX_SCAN: usize = 100_000_000; // 100MB limit per tag - prevents pathological cases
 
     while idx < len && (idx - start) < MAX_SCAN {
-        // Optimization: skip forward to next '<' quickly
+        // Optimization: skip forward to next '<' quickly using memchr
         if bytes[idx] != b'<' {
-            idx += 1;
-            continue;
+            if let Some(pos) = memchr::memchr(b'<', &bytes[idx..]) {
+                idx += pos;
+            } else {
+                break;
+            }
         }
 
         // Check for </ pattern
@@ -291,7 +294,11 @@ pub(crate) fn preprocess_html(input: &str) -> Cow<'_, str> {
                             out.push_str(&input[last..idx]);
                             out.push_str(&input[idx..open_end]);
                             out.push_str("</");
-                            out.push_str(str::from_utf8(tag).unwrap());
+                            // `TAGS` contains only ASCII byte literals (`b"script"`, `b"style"`),
+                            // which are always valid UTF-8; `from_utf8` cannot fail here.
+                            if let Ok(tag_str) = str::from_utf8(tag) {
+                                out.push_str(tag_str);
+                            }
                             out.push('>');
 
                             last = remove_end;
@@ -573,6 +580,108 @@ pub(crate) fn sanitize_markdown_url(url: &str) -> Cow<'_, str> {
     Cow::Owned(url[paren_start..paren_end].to_string())
 }
 
+/// Strip elements with the `hidden` attribute from HTML.
+///
+/// Scans for opening tags containing the `hidden` attribute, finds their
+/// matching closing tag, and removes the entire element (tag + content).
+/// Self-closing tags with `hidden` are also removed.
+pub(crate) fn strip_hidden_elements(input: &str) -> Cow<'_, str> {
+    let bytes = input.as_bytes();
+    let len = bytes.len();
+
+    if len == 0 || !bytes.contains(&b'<') {
+        return Cow::Borrowed(input);
+    }
+
+    let mut idx = 0;
+    let mut last = 0;
+    let mut output: Option<String> = None;
+
+    while idx < len {
+        if bytes[idx] == b'<' && idx + 1 < len && bytes[idx + 1] != b'/' && bytes[idx + 1] != b'!' {
+            // Find the end of this opening tag
+            if let Some(tag_end) = find_tag_end(bytes, idx + 1) {
+                let tag_slice = &input[idx..tag_end];
+                if tag_has_hidden_attribute(tag_slice) {
+                    // Extract the tag name
+                    let name_start = idx + 1;
+                    let mut name_end = name_start;
+                    while name_end < len
+                        && !bytes[name_end].is_ascii_whitespace()
+                        && bytes[name_end] != b'>'
+                        && bytes[name_end] != b'/'
+                    {
+                        name_end += 1;
+                    }
+                    let tag_name = &bytes[name_start..name_end];
+
+                    // Check if it's a self-closing tag (e.g., <br hidden> or <br hidden/>)
+                    let is_self_closing = tag_slice.ends_with("/>")
+                        || tag_name.eq_ignore_ascii_case(b"br")
+                        || tag_name.eq_ignore_ascii_case(b"hr")
+                        || tag_name.eq_ignore_ascii_case(b"img")
+                        || tag_name.eq_ignore_ascii_case(b"input");
+
+                    let remove_end = if is_self_closing {
+                        tag_end
+                    } else {
+                        // Find the closing tag
+                        find_closing_tag_bytes(bytes, tag_end, tag_name).unwrap_or(tag_end)
+                    };
+
+                    let out = output.get_or_insert_with(|| String::with_capacity(len));
+                    out.push_str(&input[last..idx]);
+                    last = remove_end;
+                    idx = remove_end;
+                    continue;
+                }
+            }
+        }
+        idx += 1;
+    }
+
+    if let Some(mut out) = output {
+        if last < len {
+            out.push_str(&input[last..]);
+        }
+        Cow::Owned(out)
+    } else {
+        Cow::Borrowed(input)
+    }
+}
+
+/// Check if an opening tag string contains the `hidden` attribute.
+///
+/// Handles: `hidden`, `hidden=""`, `hidden="hidden"`, `hidden="true"`.
+/// Does NOT match attributes like `data-hidden` or `aria-hidden`.
+fn tag_has_hidden_attribute(tag: &str) -> bool {
+    let bytes = tag.as_bytes();
+    let len = bytes.len();
+    let needle = b"hidden";
+    let nlen = needle.len();
+
+    let mut i = 0;
+    // Skip past the tag name
+    while i < len && bytes[i] != b' ' && bytes[i] != b'\t' && bytes[i] != b'\n' && bytes[i] != b'>' {
+        i += 1;
+    }
+
+    while i + nlen <= len {
+        if bytes[i..i + nlen].eq_ignore_ascii_case(needle) {
+            // Check that the character before is whitespace (attribute boundary)
+            let before_ok = i == 0 || bytes[i - 1].is_ascii_whitespace();
+            // Check that the character after is whitespace, '>', '=', or '/'
+            let after = bytes.get(i + nlen).copied();
+            let after_ok = matches!(after, None | Some(b' ' | b'\t' | b'\n' | b'\r' | b'>' | b'=' | b'/'));
+            if before_ok && after_ok {
+                return true;
+            }
+        }
+        i += 1;
+    }
+    false
+}
+
 #[cfg(test)]
 mod tests {
     use super::sanitize_markdown_url;

data/vendor/html-to-markdown-rs/src/converter/utility/serialization.rs

@@ -7,6 +7,7 @@ use crate::converter::utility::content::normalized_tag_name;
 
 /// Serialize an element to HTML string (for SVG and Math elements).
 #[allow(clippy::trivially_copy_pass_by_ref)]
+#[allow(dead_code)] // used with visitor feature
 pub(crate) fn serialize_element(node_handle: &tl::NodeHandle, parser: &tl::Parser) -> String {
     if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
         let tag_name = normalized_tag_name(tag.name().as_utf8_str());
@@ -46,6 +47,7 @@ pub(crate) fn serialize_element(node_handle: &tl::NodeHandle, parser: &tl::Parse
 
 /// Serialize a node to HTML string.
 #[allow(clippy::trivially_copy_pass_by_ref)]
+#[allow(dead_code)] // used with visitor feature
 pub(crate) fn serialize_node(node_handle: &tl::NodeHandle, parser: &tl::Parser) -> String {
     if let Some(node) = node_handle.get(parser) {
         match node {
@@ -67,6 +69,7 @@ pub(crate) fn serialize_tag_to_html(handle: &tl::NodeHandle, parser: &tl::Parser
 
 /// Recursively serialize a node to HTML.
 #[allow(clippy::trivially_copy_pass_by_ref)]
+#[allow(dead_code)] // used with visitor feature
 pub(crate) fn serialize_node_to_html(handle: &tl::NodeHandle, parser: &tl::Parser, output: &mut String) {
     match handle.get(parser) {
         Some(tl::Node::Tag(tag)) => {

data/vendor/html-to-markdown-rs/src/converter/visitor_hooks.rs

@@ -6,6 +6,8 @@
 
 use std::collections::BTreeMap;
 
+#[cfg(feature = "visitor")]
+use crate::converter::utility::content::collect_tag_attributes;
 use crate::converter::utility::content::is_block_level_element;
 use crate::visitor::{NodeContext, NodeType, VisitResult};
 
@@ -48,11 +50,7 @@ pub fn handle_visitor_element_start(
     depth: usize,
     dom_ctx: &crate::converter::DomContext,
 ) -> VisitAction {
-    let attributes: BTreeMap<String, String> = tag
-        .attributes()
-        .iter()
-        .filter_map(|(k, v)| v.as_ref().map(|val| (k.to_string(), val.to_string())))
-        .collect();
+    let attributes: BTreeMap<String, String> = collect_tag_attributes(tag);
 
     let node_id = node_handle.get_inner();
     let parent_tag = dom_ctx.parent_tag_name(node_id, parser);
@@ -131,11 +129,7 @@ pub fn handle_visitor_element_end(
         return;
     }
 
-    let attributes: BTreeMap<String, String> = tag
-        .attributes()
-        .iter()
-        .filter_map(|(k, v)| v.as_ref().map(|val| (k.to_string(), val.to_string())))
-        .collect();
+    let attributes: BTreeMap<String, String> = collect_tag_attributes(tag);
 
     let node_id = node_handle.get_inner();
     let parent_tag = dom_ctx.parent_tag_name(node_id, parser);

data/vendor/html-to-markdown-rs/src/exports.rs

@@ -13,7 +13,7 @@ pub use crate::inline_images::{
 
 #[cfg(feature = "metadata")]
 pub use crate::metadata::{
-    DEFAULT_MAX_STRUCTURED_DATA_SIZE, DocumentMetadata, ExtendedMetadata, HeaderMetadata, ImageMetadata, ImageType,
+    DEFAULT_MAX_STRUCTURED_DATA_SIZE, DocumentMetadata, HeaderMetadata, HtmlMetadata, ImageMetadata, ImageType,
     LinkMetadata, LinkType, MetadataConfig, MetadataConfigUpdate, StructuredData, StructuredDataType, TextDirection,
 };
 
@@ -21,6 +21,3 @@ pub use crate::options::{
     CodeBlockStyle, ConversionOptions, ConversionOptionsUpdate, HeadingStyle, HighlightStyle, ListIndentType,
     NewlineStyle, OutputFormat, PreprocessingOptions, PreprocessingOptionsUpdate, PreprocessingPreset, WhitespaceMode,
 };
-
-#[cfg(feature = "async-visitor")]
-pub use crate::visitor_helpers::AsyncVisitorHandle;

data/vendor/html-to-markdown-rs/src/inline_images.rs

@@ -172,7 +172,7 @@ pub struct InlineImageWarning {
     pub message: String,
 }
 
-/// Output of `convert_with_inline_images`.
+/// Output containing extracted inline images from `convert()` when `extract_images` is enabled.
 #[derive(Debug, Clone)]
 pub struct HtmlExtraction {
     /// Converted markdown output.