html-to-markdown 3.4.0 → 3.6.0.pre.rc.23

data/vendor/html-to-markdown-rs/src/text.rs

@@ -1,358 +0,0 @@
-#![allow(clippy::cast_precision_loss, clippy::cast_sign_loss, clippy::unused_self)]
-//! Text processing utilities for Markdown conversion.
-
-use regex::Regex;
-use std::borrow::Cow;
-use std::sync::LazyLock;
-
-/// Regex for escaping miscellaneous characters
-static ESCAPE_MISC_RE: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"([\\&<`\[\]>~#=+|\-])").expect("valid regex pattern"));
-
-/// Regex for escaping numbered lists
-static ESCAPE_NUMBERED_LIST_RE: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"([0-9])([.)])").expect("valid regex pattern"));
-
-/// Regex for escaping ASCII punctuation (CommonMark spec example 12)
-/// Matches: `! " # $ % & ' ( ) * + , - . / : ; < = > ? @ [ \ ] ^ _ \` { | } ~`
-static ESCAPE_ASCII_RE: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"([!\x22#$%&\x27()*+,\-./:;<=>?@\[\\\]^_`{|}~])").expect("valid regex pattern"));
-
-/// Escape Markdown special characters in text.
-///
-/// # Arguments
-///
-/// * `text` - Text to escape
-/// * `escape_misc` - Escape miscellaneous characters (`\` `&` `<` `` ` `` `[` `>` `~` `#` `=` `+` `|` `-`)
-/// * `escape_asterisks` - Escape asterisks (`*`)
-/// * `escape_underscores` - Escape underscores (`_`)
-/// * `escape_ascii` - Escape all ASCII punctuation (for `CommonMark` spec compliance)
-///
-/// # Returns
-///
-/// Escaped text
-#[allow(clippy::fn_params_excessive_bools)]
-pub fn escape(
-    text: &str,
-    escape_misc: bool,
-    escape_asterisks: bool,
-    escape_underscores: bool,
-    escape_ascii: bool,
-) -> Cow<'_, str> {
-    if text.is_empty() {
-        return Cow::Borrowed("");
-    }
-
-    if !escape_misc && !escape_asterisks && !escape_underscores && !escape_ascii {
-        return Cow::Borrowed(text);
-    }
-
-    if escape_ascii
-        && !text.as_bytes().iter().any(|b| {
-            matches!(
-                b,
-                b'!' | b'"'
-                    | b'#'
-                    | b'$'
-                    | b'%'
-                    | b'&'
-                    | b'\''
-                    | b'('
-                    | b')'
-                    | b'*'
-                    | b'+'
-                    | b','
-                    | b'-'
-                    | b'.'
-                    | b'/'
-                    | b':'
-                    | b';'
-                    | b'<'
-                    | b'='
-                    | b'>'
-                    | b'?'
-                    | b'@'
-                    | b'['
-                    | b'\\'
-                    | b']'
-                    | b'^'
-                    | b'_'
-                    | b'`'
-                    | b'{'
-                    | b'|'
-                    | b'}'
-                    | b'~'
-            )
-        })
-    {
-        return Cow::Borrowed(text);
-    }
-
-    if !escape_ascii && escape_misc && !escape_asterisks && !escape_underscores {
-        let needs_misc = text.as_bytes().iter().any(|b| {
-            matches!(
-                b,
-                b'\\' | b'&' | b'<' | b'`' | b'[' | b']' | b'>' | b'~' | b'#' | b'=' | b'+' | b'|' | b'-'
-            )
-        });
-        let needs_numbered = text.as_bytes().iter().any(|b| matches!(b, b'.' | b')'));
-        if !needs_misc && !needs_numbered {
-            return Cow::Borrowed(text);
-        }
-    }
-
-    let mut result: Cow<'_, str> = Cow::Borrowed(text);
-
-    if escape_ascii {
-        result = match ESCAPE_ASCII_RE.replace_all(result.as_ref(), r"\$1") {
-            Cow::Borrowed(_) => result,
-            Cow::Owned(s) => Cow::Owned(s),
-        };
-        return result;
-    }
-
-    if escape_misc {
-        result = match ESCAPE_MISC_RE.replace_all(result.as_ref(), r"\$1") {
-            Cow::Borrowed(_) => result,
-            Cow::Owned(s) => Cow::Owned(s),
-        };
-
-        result = match ESCAPE_NUMBERED_LIST_RE.replace_all(result.as_ref(), r"$1\$2") {
-            Cow::Borrowed(_) => result,
-            Cow::Owned(s) => Cow::Owned(s),
-        };
-    }
-
-    if escape_asterisks && result.contains('*') {
-        result = Cow::Owned(result.replace('*', r"\*"));
-    }
-
-    if escape_underscores && result.contains('_') {
-        result = Cow::Owned(result.replace('_', r"\_"));
-    }
-
-    result
-}
-
-/// Extract boundary whitespace from text (chomp).
-///
-/// Returns (prefix, suffix, `trimmed_text`) tuple.
-/// Prefix/suffix are " " if original text had leading/trailing whitespace.
-/// However, suffix is "" if the trailing whitespace is only newlines (not spaces/tabs).
-/// This prevents trailing newlines from becoming trailing spaces in the output.
-/// The trimmed text has all leading/trailing whitespace removed.
-#[must_use]
-pub fn chomp(text: &str) -> (&str, &str, &str) {
-    if text.is_empty() {
-        return ("", "", "");
-    }
-
-    let prefix = if text.starts_with(|c: char| c.is_whitespace()) {
-        " "
-    } else {
-        ""
-    };
-
-    let suffix = if text.ends_with("\n\n") || text.ends_with("\r\n\r\n") {
-        "\n\n"
-    } else if text.ends_with([' ', '\t']) {
-        " "
-    } else {
-        ""
-    };
-
-    let trimmed = if suffix == "\n\n" {
-        text.trim_end_matches("\n\n").trim_end_matches("\r\n\r\n").trim()
-    } else {
-        text.trim()
-    };
-
-    (prefix, suffix, trimmed)
-}
-
-/// Normalize whitespace by collapsing consecutive spaces and tabs.
-///
-/// Multiple spaces and tabs are replaced with a single space.
-/// Newlines are preserved.
-/// Unicode spaces are normalized to ASCII spaces.
-///
-/// # Arguments
-///
-/// * `text` - The text to normalize
-///
-/// # Returns
-///
-/// Normalized text with collapsed spaces/tabs but preserved newlines
-#[must_use]
-pub fn normalize_whitespace(text: &str) -> String {
-    let mut result = String::with_capacity(text.len());
-    let mut prev_was_space = false;
-
-    for ch in text.chars() {
-        let is_space = ch == ' ' || ch == '\t' || is_unicode_space(ch);
-
-        if is_space {
-            if !prev_was_space {
-                result.push(' ');
-                prev_was_space = true;
-            }
-        } else {
-            result.push(ch);
-            prev_was_space = false;
-        }
-    }
-
-    result
-}
-
-/// Normalize whitespace in text, returning borrowed or owned result as needed.
-///
-/// This function optimizes memory by returning a borrowed reference when no normalization
-/// is needed, and only allocating a new string when whitespace changes are necessary.
-///
-/// Multiple consecutive spaces, tabs, and Unicode space characters are replaced with
-/// a single ASCII space. Newlines are preserved as-is.
-///
-/// # Arguments
-///
-/// * `text` - The text to normalize
-///
-/// # Returns
-///
-/// `Cow::Borrowed` if text is already normalized, or `Cow::Owned` with normalized text
-#[must_use]
-pub fn normalize_whitespace_cow(text: &str) -> Cow<'_, str> {
-    let mut prev_was_space = false;
-
-    for ch in text.chars() {
-        let is_space = ch == ' ' || ch == '\t' || is_unicode_space(ch);
-        if is_space {
-            if prev_was_space || ch != ' ' {
-                return Cow::Owned(normalize_whitespace(text));
-            }
-            prev_was_space = true;
-        } else {
-            prev_was_space = false;
-        }
-    }
-
-    Cow::Borrowed(text)
-}
-
-/// Decode common HTML entities.
-///
-/// Decodes the most common HTML entities to their character equivalents:
-/// - `&quot;` → `"`
-/// - `&apos;` → `'`
-/// - `&lt;` → `<`
-/// - `&gt;` → `>`
-/// - `&amp;` → `&` (must be last to avoid double-decoding)
-///
-/// # Arguments
-///
-/// * `text` - Text containing HTML entities
-///
-/// # Returns
-///
-/// Text with entities decoded
-#[must_use]
-pub fn decode_html_entities(text: &str) -> String {
-    html_escape::decode_html_entities(text).into_owned()
-}
-
-/// Decode HTML entities in text, returning borrowed or owned result as needed.
-///
-/// This function optimizes memory by returning a borrowed reference when no HTML
-/// entities are present, and only allocating a new string when entity decoding
-/// is necessary.
-///
-/// Decodes common HTML entities like:
-/// - `&quot;` → `"`
-/// - `&apos;` → `'`
-/// - `&lt;` → `<`
-/// - `&gt;` → `>`
-/// - `&amp;` → `&` (decoded last to avoid double-decoding)
-///
-/// # Arguments
-///
-/// * `text` - Text potentially containing HTML entities
-///
-/// # Returns
-///
-/// `Cow::Borrowed` if no entities found, or `Cow::Owned` with entities decoded
-#[must_use]
-pub fn decode_html_entities_cow(text: &str) -> Cow<'_, str> {
-    if !text.contains('&') {
-        return Cow::Borrowed(text);
-    }
-
-    html_escape::decode_html_entities(text)
-}
-
-/// Check if a character is a unicode space character.
-///
-/// Includes: non-breaking space, various width spaces, etc.
-const fn is_unicode_space(ch: char) -> bool {
-    matches!(
-        ch,
-        '\u{00A0}'
-            | '\u{1680}'
-            | '\u{2000}'
-            | '\u{2001}'
-            | '\u{2002}'
-            | '\u{2003}'
-            | '\u{2004}'
-            | '\u{2005}'
-            | '\u{2006}'
-            | '\u{2007}'
-            | '\u{2008}'
-            | '\u{2009}'
-            | '\u{200A}'
-            | '\u{202F}'
-            | '\u{205F}'
-            | '\u{3000}'
-    )
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_escape_misc() {
-        assert_eq!(escape("foo & bar", true, false, false, false), r"foo \& bar");
-        assert_eq!(escape("foo [bar]", true, false, false, false), r"foo \[bar\]");
-        assert_eq!(escape("1. Item", true, false, false, false), r"1\. Item");
-        assert_eq!(escape("1) Item", true, false, false, false), r"1\) Item");
-    }
-
-    #[test]
-    fn test_escape_asterisks() {
-        assert_eq!(escape("foo * bar", false, true, false, false), r"foo \* bar");
-        assert_eq!(escape("**bold**", false, true, false, false), r"\*\*bold\*\*");
-    }
-
-    #[test]
-    fn test_escape_underscores() {
-        assert_eq!(escape("foo_bar", false, false, true, false), r"foo\_bar");
-        assert_eq!(escape("__bold__", false, false, true, false), r"\_\_bold\_\_");
-    }
-
-    #[test]
-    fn test_escape_ascii() {
-        assert_eq!(escape(r##"!"#$%&"##, false, false, false, true), r#"\!\"\#\$\%\&"#);
-        assert_eq!(escape("*+,-./", false, false, false, true), r"\*\+\,\-\.\/");
-        assert_eq!(escape("<=>?@", false, false, false, true), r"\<\=\>\?\@");
-        assert_eq!(escape(r"[\]^_`", false, false, false, true), r"\[\\\]\^\_\`");
-        assert_eq!(escape("{|}~", false, false, false, true), r"\{\|\}\~");
-    }
-
-    #[test]
-    fn test_chomp() {
-        assert_eq!(chomp("  text  "), (" ", " ", "text"));
-        assert_eq!(chomp("text"), ("", "", "text"));
-        assert_eq!(chomp(" text"), (" ", "", "text"));
-        assert_eq!(chomp("text "), ("", " ", "text"));
-        assert_eq!(chomp(""), ("", "", ""));
-    }
-}

data/vendor/html-to-markdown-rs/src/types/document.rs

@@ -1,191 +0,0 @@
-//! Structured document tree types aligned with kreuzberg's `DocumentStructure`.
-
-use std::collections::HashMap;
-
-#[cfg(feature = "serde")]
-use serde::{Deserialize, Serialize};
-
-use super::tables::TableGrid;
-
-/// A structured document tree representing the semantic content of an HTML document.
-///
-/// Uses a flat node array with index-based parent/child references for efficient traversal.
-#[derive(Debug, Clone, PartialEq, Eq)]
-#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-pub struct DocumentStructure {
-    /// All nodes in document reading order.
-    pub nodes: Vec<DocumentNode>,
-    /// The source format (always "html" for this crate).
-    #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
-    pub source_format: Option<String>,
-}
-
-/// A single node in the document tree.
-#[derive(Debug, Clone, PartialEq, Eq)]
-#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-pub struct DocumentNode {
-    /// Deterministic node identifier.
-    pub id: String,
-    /// The semantic content of this node.
-    pub content: NodeContent,
-    /// Index of the parent node (None for root nodes).
-    #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
-    pub parent: Option<u32>,
-    /// Indices of child nodes in reading order.
-    #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Vec::is_empty", default))]
-    pub children: Vec<u32>,
-    /// Inline formatting annotations (bold, italic, links, etc.) with byte offsets into the text.
-    #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Vec::is_empty", default))]
-    pub annotations: Vec<TextAnnotation>,
-    /// Format-specific attributes (e.g. class, id, data-* attributes).
-    #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
-    pub attributes: Option<HashMap<String, String>>,
-}
-
-/// The semantic content type of a document node.
-///
-/// Uses internally tagged representation (`"node_type": "heading"`) for JSON serialization.
-#[derive(Debug, Clone, PartialEq, Eq)]
-#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "serde", serde(tag = "node_type", rename_all = "snake_case"))]
-pub enum NodeContent {
-    /// A heading element (h1-h6).
-    Heading {
-        /// Heading level (1-6).
-        level: u8,
-        /// The heading text content.
-        text: String,
-    },
-    /// A paragraph of text.
-    Paragraph {
-        /// The paragraph text content.
-        text: String,
-    },
-    /// A list container (ordered or unordered). Children are `ListItem` nodes.
-    List {
-        /// Whether this is an ordered list.
-        ordered: bool,
-    },
-    /// A single list item.
-    ListItem {
-        /// The list item text content.
-        text: String,
-    },
-    /// A table with structured cell data.
-    Table {
-        /// The table grid structure.
-        grid: TableGrid,
-    },
-    /// An image element.
-    Image {
-        /// Alt text or caption.
-        #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
-        description: Option<String>,
-        /// Image source URL.
-        #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
-        src: Option<String>,
-        /// Index into `ConversionResult.images` when image extraction is enabled.
-        #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
-        image_index: Option<u32>,
-    },
-    /// A code block or inline code.
-    Code {
-        /// The code text content.
-        text: String,
-        /// Programming language (from class="language-*" or similar).
-        #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
-        language: Option<String>,
-    },
-    /// A block quote container.
-    Quote,
-    /// A definition list container.
-    DefinitionList,
-    /// A definition list entry with term and description.
-    DefinitionItem {
-        /// The term being defined.
-        term: String,
-        /// The definition text.
-        definition: String,
-    },
-    /// A raw block preserved as-is (e.g. `<script>`, `<style>` content).
-    RawBlock {
-        /// The format of the raw content (e.g. "html", "css", "javascript").
-        format: String,
-        /// The raw content.
-        content: String,
-    },
-    /// A block of key-value metadata pairs (from `<head>` meta tags).
-    MetadataBlock {
-        /// Key-value metadata pairs.
-        entries: Vec<(String, String)>,
-    },
-    /// A section grouping container (auto-generated from heading hierarchy).
-    Group {
-        /// Optional section label.
-        #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
-        label: Option<String>,
-        /// The heading level that created this group.
-        #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
-        heading_level: Option<u8>,
-        /// The heading text that created this group.
-        #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
-        heading_text: Option<String>,
-    },
-}
-
-/// An inline text annotation with byte-range offsets.
-///
-/// Annotations describe formatting (bold, italic, etc.) and links within a node's text content.
-#[derive(Debug, Clone, PartialEq, Eq)]
-#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-pub struct TextAnnotation {
-    /// Start byte offset (inclusive) into the parent node's text.
-    pub start: u32,
-    /// End byte offset (exclusive) into the parent node's text.
-    pub end: u32,
-    /// The type of annotation.
-    pub kind: AnnotationKind,
-}
-
-/// The type of an inline text annotation.
-///
-/// Uses internally tagged representation (`"annotation_type": "bold"`) for JSON serialization.
-#[derive(Debug, Clone, PartialEq, Eq, Default)]
-#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "serde", serde(tag = "annotation_type", rename_all = "snake_case"))]
-pub enum AnnotationKind {
-    /// Bold / strong emphasis.
-    #[default]
-    Bold,
-    /// Italic / emphasis.
-    Italic,
-    /// Underline.
-    Underline,
-    /// Strikethrough / deleted text.
-    Strikethrough,
-    /// Inline code.
-    Code,
-    /// Subscript text.
-    Subscript,
-    /// Superscript text.
-    Superscript,
-    /// Highlighted / marked text.
-    Highlight,
-    /// A hyperlink.
-    Link {
-        /// The link URL.
-        url: String,
-        /// Optional link title attribute.
-        #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
-        title: Option<String>,
-    },
-}
-
-impl Default for NodeContent {
-    fn default() -> Self {
-        Self::Heading {
-            level: 1,
-            text: String::new(),
-        }
-    }
-}

data/vendor/html-to-markdown-rs/src/types/mod.rs

@@ -1,17 +0,0 @@
-//! Core types for structured HTML extraction results.
-//!
-//! These types are aligned with kreuzberg's `DocumentStructure` model for seamless integration.
-
-mod document;
-mod result;
-pub mod structure_builder;
-pub mod structure_collector;
-mod tables;
-mod warnings;
-
-pub use document::{AnnotationKind, DocumentNode, DocumentStructure, NodeContent, TextAnnotation};
-pub use result::ConversionResult;
-pub use structure_builder::build_document_structure;
-pub use structure_collector::{StructureCollector, StructureCollectorHandle};
-pub use tables::{GridCell, TableData, TableGrid};
-pub use warnings::{ProcessingWarning, WarningKind};

data/vendor/html-to-markdown-rs/src/types/result.rs

@@ -1,54 +0,0 @@
-//! The primary result type for HTML conversion and extraction.
-
-#[cfg(feature = "serde")]
-use serde::{Deserialize, Serialize};
-
-use super::document::DocumentStructure;
-use super::tables::TableData;
-use super::warnings::ProcessingWarning;
-
-/// The primary result of HTML conversion and extraction.
-///
-/// Contains the converted text output, optional structured document tree,
-/// metadata, extracted tables, images, and processing warnings.
-///
-/// # Example
-///
-/// ```text
-/// use html_to_markdown_rs::{convert, ConversionOptions};
-///
-/// let result = convert("<h1>Hello</h1><p>World</p>", None)?;
-/// assert!(result.content.is_some());
-/// assert!(result.warnings.is_empty());
-/// ```
-#[derive(Debug, Clone, Default)]
-#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-pub struct ConversionResult {
-    /// Converted text output (markdown, djot, or plain text).
-    ///
-    /// `None` when `output_format` is set to `OutputFormat::None`,
-    /// indicating extraction-only mode.
-    pub content: Option<String>,
-
-    /// Structured document tree with semantic elements.
-    ///
-    /// Populated when `include_document_structure` is `true` in options.
-    pub document: Option<DocumentStructure>,
-
-    /// Extracted HTML metadata (title, OG, links, images, structured data).
-    #[cfg(feature = "metadata")]
-    pub metadata: crate::metadata::HtmlMetadata,
-
-    /// Extracted tables with structured cell data and markdown representation.
-    pub tables: Vec<TableData>,
-
-    /// Extracted inline images (data URIs and SVGs).
-    ///
-    /// Populated when `extract_images` is `true` in options.
-    #[cfg(feature = "inline-images")]
-    #[cfg_attr(feature = "serde", serde(skip))]
-    pub images: Vec<crate::inline_images::InlineImage>,
-
-    /// Non-fatal processing warnings.
-    pub warnings: Vec<ProcessingWarning>,
-}