html-to-markdown 3.4.0 → 3.6.0.pre.rc.24

data/vendor/html-to-markdown-rs/src/converter/media/svg.rs

@@ -1,280 +0,0 @@
-//! SVG and MathML element handling with serialization and base64 encoding.
-
-use crate::converter::main_helpers::tag_name_eq;
-use crate::converter::utility::content::normalized_tag_name;
-#[allow(unused_imports)]
-use std::collections::BTreeMap;
-use tl::{NodeHandle, Parser};
-
-#[cfg(feature = "inline-images")]
-use crate::inline_images::{InlineImageCollector, InlineImageFormat, InlineImageSource};
-
-#[cfg(feature = "inline-images")]
-type InlineCollectorHandle = std::rc::Rc<std::cell::RefCell<InlineImageCollector>>;
-
-/// Handle inline SVG elements with size limits and base64 encoding.
-///
-/// # Features
-/// - SVG serialization to HTML string
-/// - Size validation with configurable limits
-/// - Base64 encoding for data URI
-/// - Metadata extraction (aria-label, title, dimensions)
-#[cfg(feature = "inline-images")]
-#[allow(clippy::trivially_copy_pass_by_ref)]
-#[allow(clippy::needless_pass_by_value)]
-#[allow(clippy::option_if_let_else)]
-pub fn handle_inline_svg(
-    collector_ref: &InlineCollectorHandle,
-    node_handle: &NodeHandle,
-    parser: &Parser,
-    title_opt: Option<String>,
-    attributes: BTreeMap<String, String>,
-) {
-    let max_size = {
-        let borrow = collector_ref.borrow();
-        if !borrow.capture_svg() {
-            return;
-        }
-        borrow.max_decoded_size()
-    };
-
-    if max_size == 0 {
-        let mut collector = collector_ref.borrow_mut();
-        let index = collector.next_index();
-        collector.warn_skip(index, "max SVG payload size is zero");
-        return;
-    }
-
-    let mut collector = collector_ref.borrow_mut();
-    let index = collector.next_index();
-
-    let serialized = serialize_element(node_handle, parser);
-    if serialized.is_empty() {
-        collector.warn_skip(index, "unable to serialize SVG element");
-        return;
-    }
-
-    let data = serialized.into_bytes();
-    if data.len() as u64 > max_size {
-        collector.warn_skip(
-            index,
-            format!(
-                "serialized SVG payload ({} bytes) exceeds configured max ({})",
-                data.len(),
-                max_size
-            ),
-        );
-        return;
-    }
-
-    let description = attributes
-        .get("aria-label")
-        .and_then(|value| non_empty_trimmed(value))
-        .or_else(|| title_opt.as_deref().and_then(non_empty_trimmed));
-
-    let filename_candidate = attributes
-        .get("data-filename")
-        .cloned()
-        .or_else(|| attributes.get("filename").cloned())
-        .or_else(|| attributes.get("data-name").cloned());
-
-    let image = collector.build_image(
-        data,
-        InlineImageFormat::Svg,
-        filename_candidate,
-        description,
-        None,
-        InlineImageSource::SvgElement,
-        attributes,
-    );
-
-    collector.push_image(index, image);
-}
-
-/// Serialize an element to HTML string (for SVG and Math elements).
-#[allow(clippy::trivially_copy_pass_by_ref)]
-pub fn serialize_element(node_handle: &NodeHandle, parser: &Parser) -> String {
-    if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
-        let tag_name = normalized_tag_name(tag.name().as_utf8_str());
-        let mut html = String::with_capacity(256);
-        html.push('<');
-        html.push_str(&tag_name);
-
-        for (key, value_opt) in tag.attributes().iter() {
-            html.push(' ');
-            html.push_str(&key);
-            if let Some(value) = value_opt {
-                html.push_str("=\"");
-                html.push_str(&value);
-                html.push('"');
-            }
-        }
-
-        let has_children = !tag.children().top().is_empty();
-        if has_children {
-            html.push('>');
-            let children = tag.children();
-            {
-                for child_handle in children.top().iter() {
-                    html.push_str(&serialize_node(child_handle, parser));
-                }
-            }
-            html.push_str("</");
-            html.push_str(&tag_name);
-            html.push('>');
-        } else {
-            html.push_str(" />");
-        }
-        return html;
-    }
-    String::new()
-}
-
-/// Serialize a node to HTML string.
-#[allow(clippy::trivially_copy_pass_by_ref)]
-pub fn serialize_node(node_handle: &NodeHandle, parser: &Parser) -> String {
-    if let Some(node) = node_handle.get(parser) {
-        match node {
-            tl::Node::Raw(bytes) => bytes.as_utf8_str().to_string(),
-            tl::Node::Tag(_) => serialize_element(node_handle, parser),
-            _ => String::new(),
-        }
-    } else {
-        String::new()
-    }
-}
-
-/// Extract non-empty trimmed string or return None.
-#[cfg(feature = "inline-images")]
-fn non_empty_trimmed(value: &str) -> Option<String> {
-    let trimmed = value.trim();
-    if trimmed.is_empty() {
-        None
-    } else {
-        Some(trimmed.to_string())
-    }
-}
-
-/// Handle SVG element conversion to Markdown.
-///
-/// Extracts title from child elements, handles inline image collection,
-/// and outputs either the title text (in inline mode) or a base64-encoded image.
-#[allow(clippy::too_many_arguments)]
-pub fn handle_svg(
-    node_handle: &NodeHandle,
-    tag: &tl::HTMLTag,
-    parser: &Parser,
-    output: &mut String,
-    options: &crate::options::ConversionOptions,
-    ctx: &super::Context,
-    _depth: usize,
-    dom_ctx: &super::DomContext,
-) {
-    use crate::converter::utility::content::get_text_content;
-
-    let mut title = String::from("SVG Image");
-    let children = tag.children();
-    for child_handle in children.top().iter() {
-        if let Some(tl::Node::Tag(child_tag)) = child_handle.get(parser) {
-            if tag_name_eq(child_tag.name().as_utf8_str(), "title") {
-                title = get_text_content(child_handle, parser, dom_ctx).trim().to_string();
-                break;
-            }
-        }
-    }
-
-    #[cfg(feature = "inline-images")]
-    if let Some(ref collector_ref) = ctx.inline_collector {
-        let title_opt = if title == "SVG Image" {
-            None
-        } else {
-            Some(title.clone())
-        };
-        let mut attributes_map = BTreeMap::new();
-        for (key, value_opt) in tag.attributes().iter() {
-            let key_str = key.to_string();
-            let keep = key_str == "width"
-                || key_str == "height"
-                || key_str == "filename"
-                || key_str == "aria-label"
-                || key_str.starts_with("data-");
-            if keep {
-                let value = value_opt.map(|value| value.to_string()).unwrap_or_default();
-                attributes_map.insert(key_str, value);
-            }
-        }
-        handle_inline_svg(collector_ref, node_handle, parser, title_opt, attributes_map);
-    }
-
-    if options.skip_images {
-        return;
-    }
-
-    if ctx.convert_as_inline {
-        output.push_str(&title);
-    } else {
-        use base64::{Engine as _, engine::general_purpose::STANDARD};
-
-        let svg_html = serialize_element(node_handle, parser);
-        let base64_svg = STANDARD.encode(svg_html.as_bytes());
-
-        output.push_str("![");
-        output.push_str(&title);
-        output.push_str("](data:image/svg+xml;base64,");
-        output.push_str(&base64_svg);
-        output.push(')');
-    }
-}
-
-/// Handle MathML element conversion to Markdown.
-///
-/// Serializes MathML to HTML comment and outputs text content with escaping.
-#[allow(clippy::too_many_arguments)]
-pub fn handle_math(
-    node_handle: &NodeHandle,
-    tag: &tl::HTMLTag,
-    parser: &Parser,
-    output: &mut String,
-    options: &crate::options::ConversionOptions,
-    ctx: &super::Context,
-    _depth: usize,
-    dom_ctx: &super::DomContext,
-) {
-    use crate::converter::utility::content::get_text_content;
-    use crate::text;
-
-    let text_content = get_text_content(node_handle, parser, dom_ctx).trim().to_string();
-
-    if text_content.is_empty() {
-        return;
-    }
-
-    let math_html = serialize_element(node_handle, parser);
-
-    let escaped_text = text::escape(
-        &text_content,
-        options.escape_misc,
-        options.escape_asterisks,
-        options.escape_underscores,
-        options.escape_ascii,
-    );
-
-    let is_display_block = tag
-        .attributes()
-        .get("display")
-        .flatten()
-        .is_some_and(|v| v.as_utf8_str() == "block");
-
-    if is_display_block && !ctx.in_paragraph && !ctx.convert_as_inline {
-        output.push_str("\n\n");
-    }
-
-    output.push_str("<!-- MathML: ");
-    output.push_str(&math_html);
-    output.push_str(" --> ");
-    output.push_str(&escaped_text);
-
-    if is_display_block && !ctx.in_paragraph && !ctx.convert_as_inline {
-        output.push_str("\n\n");
-    }
-}

data/vendor/html-to-markdown-rs/src/converter/metadata.rs

@@ -1,220 +0,0 @@
-//! Handler for metadata and script elements (head, script, style, math).
-//!
-//! Converts various metadata-related elements:
-//! - **head**: Document metadata container; processes script[type="application/ld+json"]
-//! - **script**: Script elements; extracts JSON-LD structured data when appropriate
-//! - **style**: CSS stylesheet elements; skipped in conversion
-//! - **math**: MathML elements with serialization and HTML comments for preservation
-
-use crate::converter::media::svg::serialize_element;
-use crate::options::ConversionOptions;
-#[cfg(feature = "metadata")]
-use crate::text::decode_html_entities;
-use crate::text::escape;
-use tl::{NodeHandle, Parser};
-
-// Type aliases for Context and DomContext to avoid circular imports
-type Context = crate::converter::Context;
-type DomContext = crate::converter::DomContext;
-
-/// Handles metadata elements: head, script, style, math.
-///
-/// Processes various metadata-related elements:
-/// - head: Scans for structured data in script[type="application/ld+json"]
-/// - script: Extracts JSON-LD for structured data collection
-/// - style: Skipped (CSS not relevant in markdown)
-/// - math: Preserves MathML as HTML comments with text content
-pub fn handle(
-    tag_name: &str,
-    node_handle: &NodeHandle,
-    parser: &Parser,
-    output: &mut String,
-    options: &ConversionOptions,
-    ctx: &Context,
-    depth: usize,
-    dom_ctx: &DomContext,
-) {
-    match tag_name {
-        "head" => {
-            handle_head(node_handle, parser, output, options, ctx, depth, dom_ctx);
-        }
-        "script" => {
-            handle_script(node_handle, parser, output, options, ctx);
-        }
-        "style" => {
-            // Style elements are skipped - no output
-        }
-        "math" => {
-            handle_math(node_handle, parser, output, options, ctx, dom_ctx);
-        }
-        _ => {}
-    }
-}
-
-/// Handle head element.
-///
-/// Head elements contain metadata. We process them to extract structured data from
-/// nested script[type="application/ld+json"] elements if metadata collection is enabled.
-fn handle_head(
-    node_handle: &NodeHandle,
-    parser: &Parser,
-    output: &mut String,
-    options: &ConversionOptions,
-    ctx: &Context,
-    depth: usize,
-    dom_ctx: &DomContext,
-) {
-    use crate::converter::walk_node;
-
-    let Some(node) = node_handle.get(parser) else { return };
-
-    let tag = match node {
-        tl::Node::Tag(tag) => tag,
-        _ => return,
-    };
-
-    let children = tag.children();
-    let has_body_like = children.top().iter().any(|child_handle| {
-        if let Some(child_name) = dom_ctx.tag_name_for(*child_handle, parser) {
-            matches!(
-                child_name.as_ref(),
-                "body" | "main" | "article" | "section" | "div" | "p"
-            )
-        } else {
-            false
-        }
-    });
-
-    #[cfg(feature = "metadata")]
-    if ctx.metadata_wants_structured_data {
-        if let Some(ref collector) = ctx.metadata_collector {
-            for child_handle in children.top().iter() {
-                if let Some(tl::Node::Tag(child_tag)) = child_handle.get(parser) {
-                    let child_name = dom_ctx
-                        .tag_name_for(*child_handle, parser)
-                        .unwrap_or_else(|| crate::converter::normalized_tag_name(child_tag.name().as_utf8_str()));
-                    if child_name.as_ref() == "script" {
-                        if let Some(type_attr) = child_tag.attributes().get("type").flatten() {
-                            let type_value = type_attr.as_utf8_str();
-                            let type_value = type_value.as_ref();
-                            let type_value = type_value.split(';').next().unwrap_or(type_value);
-                            if type_value.trim().eq_ignore_ascii_case("application/ld+json") {
-                                let json = child_tag.inner_text(parser);
-                                let json = json.trim();
-                                if !json.is_empty() {
-                                    let json = decode_html_entities(json);
-                                    if !json.is_empty() {
-                                        collector.borrow_mut().add_json_ld(json);
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    // If head contains body-like elements (malformed HTML), process them
-    if has_body_like {
-        for child_handle in children.top().iter() {
-            walk_node(child_handle, parser, output, options, ctx, depth + 1, dom_ctx);
-        }
-    }
-}
-
-/// Handle script element.
-///
-/// Script elements are processed to extract JSON-LD structured data when
-/// the type is "application/ld+json" and metadata collection is enabled.
-#[cfg_attr(not(feature = "metadata"), allow(unused_variables))]
-fn handle_script(
-    node_handle: &NodeHandle,
-    parser: &Parser,
-    _output: &mut String,
-    _options: &ConversionOptions,
-    ctx: &Context,
-) {
-    let Some(node) = node_handle.get(parser) else { return };
-
-    let tag = match node {
-        tl::Node::Tag(tag) => tag,
-        _ => return,
-    };
-
-    #[cfg(feature = "metadata")]
-    if let Some(type_attr) = tag.attributes().get("type").flatten() {
-        let type_value = type_attr.as_utf8_str();
-        let type_value = type_value.as_ref();
-        let type_value = type_value.split(';').next().unwrap_or(type_value);
-        if type_value.trim().eq_ignore_ascii_case("application/ld+json") && ctx.metadata_wants_structured_data {
-            if let Some(ref collector) = ctx.metadata_collector {
-                let json = tag.inner_text(parser);
-                let json = json.trim();
-                if !json.is_empty() {
-                    let json = decode_html_entities(json);
-                    if !json.is_empty() {
-                        collector.borrow_mut().add_json_ld(json);
-                    }
-                }
-            }
-        }
-    }
-}
-
-/// Handle math element.
-///
-/// MathML elements are serialized to HTML and wrapped in a comment to preserve them.
-/// The text content of the element is also output as plain text.
-fn handle_math(
-    node_handle: &NodeHandle,
-    parser: &Parser,
-    output: &mut String,
-    options: &ConversionOptions,
-    ctx: &Context,
-    dom_ctx: &DomContext,
-) {
-    let text_content = crate::converter::get_text_content(node_handle, parser, dom_ctx)
-        .trim()
-        .to_string();
-
-    if text_content.is_empty() {
-        return;
-    }
-
-    let math_html = serialize_element(node_handle, parser);
-
-    let escaped_text = escape(
-        &text_content,
-        options.escape_misc,
-        options.escape_asterisks,
-        options.escape_underscores,
-        options.escape_ascii,
-    );
-
-    let Some(node) = node_handle.get(parser) else { return };
-
-    let tag = match node {
-        tl::Node::Tag(tag) => tag,
-        _ => return,
-    };
-
-    let is_display_block = tag
-        .attributes()
-        .get("display")
-        .flatten()
-        .is_some_and(|v| v.as_utf8_str() == "block");
-
-    if is_display_block && !ctx.in_paragraph && !ctx.convert_as_inline {
-        output.push_str("\n\n");
-    }
-
-    output.push_str("<!-- MathML: ");
-    output.push_str(&math_html);
-    output.push_str(" --> ");
-    output.push_str(&escaped_text);
-
-    if is_display_block && !ctx.in_paragraph && !ctx.convert_as_inline {
-        output.push_str("\n\n");
-    }
-}

data/vendor/html-to-markdown-rs/src/converter/mod.rs

@@ -1,156 +0,0 @@
-//! HTML to Markdown conversion engine with modular architecture.
-//!
-//! This module provides the complete conversion pipeline for transforming HTML documents
-//! into Markdown format. It follows a modular, type-safe design where HTML element handling
-//! is organized by semantic category (block, inline, list, table, etc.) with dispatch functions
-//! routing elements to their specialized handlers.
-//!
-//! # Module Organization
-//!
-//! The converter module is organized into semantic categories:
-//!
-//! - **[block]**: Block-level elements (headings, paragraphs, blockquotes, preformatted text, tables)
-//! - **[inline]**: Inline formatting (emphasis, links, code, semantic formatting)
-//! - **[list]**: List structures (ordered, unordered, definition lists)
-//! - **[table]**: Accessible via `block::table` submodule
-//! - **[media]**: Media elements (images, video, audio, embedded content, SVG)
-//! - **[semantic]**: Semantic HTML5 elements (sectioning, figures, interactive elements)
-//! - **[form]**: Form elements (inputs, selects, buttons, fieldsets)
-//! - **[utility]**: Helper functions (DOM traversal, caching, serialization, attributes)
-//! - **[text]**: Text processing and escaping (via crate::text module)
-//!
-//! # Public Types
-//!
-//! The main context types used across the conversion pipeline:
-//!
-//! - **[Context]**: Stateful conversion context tracking (e.g., list nesting, code blocks, in_heading)
-//! - **[DomContext]**: DOM relationship cache for efficient tree navigation
-//!
-//! # Conversion Flow
-//!
-//! The conversion process follows these steps:
-//!
-//! 1. **Parse HTML**: Input HTML is parsed into a DOM tree using the astral-tl parser
-//! 2. **Walk Tree**: Recursive tree walk starting from the root document node
-//! 3. **Dispatch**: Each element is dispatched to its handler based on tag name
-//! 4. **Convert**: Handler transforms the element to Markdown representation
-//! 5. **Post-process**: Text escaping and whitespace normalization
-//!
-//! # Handler Pattern
-//!
-//! Each submodule (block, inline, list, etc.) follows a consistent pattern:
-//!
-//! ```text
-//! // Module declares handlers for specific element types
-//! pub fn dispatch_<category>_handler(
-//!     tag_name: &str,
-//!     node_handle: &NodeHandle,
-//!     parser: &Parser,
-//!     output: &mut String,
-//!     options: &ConversionOptions,
-//!     ctx: &Context,
-//!     depth: usize,
-//!     dom_ctx: &DomContext,
-//! ) -> bool {
-//!     // Route to appropriate handler, return true if handled
-//! }
-//! ```
-//!
-//! # Visibility Rules
-//!
-//! - **Context & DomContext**: Public types for external module coordination
-//! - **Dispatch functions**: Public for main walk_node caller
-//! - **Individual handlers**: Typically pub for direct access if needed
-//! - **Internal utilities**: pub(crate) or pub(super) for module-internal use
-//!
-//! # Feature Support
-//!
-//! - Inline image extraction (`inline-images` feature)
-//! - Metadata collection (`metadata` feature)
-//! - Custom visitor callbacks (`visitor` feature)
-//!
-//! # Example Integration
-//!
-//! Once `converter.rs` is refactored to use `converter/main.rs`, the walk_node function
-//! will use dispatch functions like:
-//!
-//! ```text
-//! use crate::converter::{block, inline, list, media, semantic, form};
-//!
-//! fn walk_node(...) {
-//!     // Try each dispatcher in order
-//!     if block::dispatch_block_handler(&tag, ...) { return; }
-//!     if inline::dispatch_inline_handler(&tag, ...) { return; }
-//!     if list::dispatch_list_handler(&tag, ...) { return; }
-//!     if media::dispatch_media_handler(&tag, ...) { return; }
-//!     if semantic::dispatch_semantic_handler(&tag, ...) { return; }
-//!     if form::dispatch_form_handler(&tag, ...) { return; }
-//!     // Default handling for unrecognized tags
-//! }
-//! ```
-
-pub mod block;
-pub mod context;
-pub mod dom_context;
-pub mod form;
-pub mod format;
-pub mod handlers;
-pub mod inline;
-pub mod list;
-pub mod main;
-mod main_helpers;
-pub mod media;
-mod metadata;
-pub mod plain_text;
-pub mod preprocessing_helpers;
-pub mod reference_collector;
-pub mod semantic;
-pub mod text;
-mod text_node;
-pub mod utility;
-
-#[cfg(feature = "visitor")]
-pub mod visitor_hooks;
-
-// Import and re-export public types and functions from the main module
-pub use self::context::Context;
-pub use self::dom_context::DomContext;
-
-// Import the tree walker and utility functions from main and main_helpers
-pub use self::main::{convert_html_impl, walk_node};
-pub use self::main_helpers::trim_trailing_whitespace;
-
-// Re-export helper functions from utility modules (migrated from converter_legacy)
-pub use crate::converter::utility::content::{chomp_inline, get_text_content, normalized_tag_name};
-#[allow(unused_imports)]
-pub use crate::converter::utility::serialization::{serialize_node, serialize_node_to_html};
-
-// Helper functions migrated to utility modules
-pub use crate::converter::utility::siblings::append_inline_suffix;
-
-// Caching functions migrated to utility/caching
-
-// Content functions migrated to utility/content
-
-// Heading functions migrated to block/heading
-pub use crate::converter::block::heading::find_single_heading_child;
-
-// Link functions migrated to inline/link
-
-// Re-export dispatch functions for routing elements to handlers
-// Media module doesn't have a dispatcher - it exports utility functions
-
-// Re-export utility submodules for public access to their types
-// NOTE: utility::preprocessing is deliberately not re-exported to avoid naming conflict
-// with preprocessing_helpers module. Users should access utility::preprocessing directly.
-
-// Re-export format renderer types
-
-// Block and inline handlers are internal - only dispatchers are exposed
-// Individual handlers are pub(crate) and not meant to be part of the public API
-
-// Re-export media utilities for internal use (crate-private)
-
-// Re-export list utilities for internal use (crate-private)
-
-// Semantic and form handlers are also internal (pub(crate))