html-to-markdown 2.30.0 → 3.0.1

data/vendor/html-to-markdown-rs/src/options/preprocessing.rs

@@ -42,7 +42,10 @@ impl PreprocessingPreset {
     any(feature = "serde", feature = "metadata"),
     derive(serde::Serialize, serde::Deserialize)
 )]
-#[cfg_attr(any(feature = "serde", feature = "metadata"), serde(rename_all = "camelCase"))]
+#[cfg_attr(
+    any(feature = "serde", feature = "metadata"),
+    serde(rename_all = "camelCase", deny_unknown_fields)
+)]
 pub struct PreprocessingOptions {
     /// Enable HTML preprocessing globally
     pub enabled: bool,
@@ -67,7 +70,10 @@ pub struct PreprocessingOptions {
     any(feature = "serde", feature = "metadata"),
     derive(serde::Serialize, serde::Deserialize)
 )]
-#[cfg_attr(any(feature = "serde", feature = "metadata"), serde(rename_all = "camelCase"))]
+#[cfg_attr(
+    any(feature = "serde", feature = "metadata"),
+    serde(rename_all = "camelCase", deny_unknown_fields)
+)]
 pub struct PreprocessingOptionsUpdate {
     /// Optional global preprocessing enablement override
     pub enabled: Option<bool>,

data/vendor/html-to-markdown-rs/src/prelude.rs

@@ -9,18 +9,4 @@
 pub use crate::convert;
 pub use crate::error::{ConversionError, Result};
 pub use crate::options::{ConversionOptions, HeadingStyle};
-
-#[cfg(feature = "inline-images")]
-pub use crate::convert_with_inline_images;
-
-#[cfg(feature = "metadata")]
-pub use crate::convert_with_metadata;
-
-#[cfg(feature = "visitor")]
-pub use crate::convert_with_visitor;
-
-#[cfg(feature = "visitor")]
-pub use crate::{ConversionWithTables, TableData, convert_with_tables};
-
-#[cfg(feature = "async-visitor")]
-pub use crate::convert_with_async_visitor;
+pub use crate::types::ConversionResult;

data/vendor/html-to-markdown-rs/src/rcdom.rs

@@ -67,7 +67,10 @@ pub enum NodeData {
     /// [dtd wiki]: https://en.wikipedia.org/wiki/Document_type_declaration
     Doctype {
         name: StrTendril,
+        // Fields required by html5ever's DOM model; not accessed during conversion.
+        #[allow(dead_code)]
         public_id: StrTendril,
+        #[allow(dead_code)]
         system_id: StrTendril,
     },
 
@@ -386,7 +389,10 @@ impl TreeSink for RcDom {
             let previous_parent = child.parent.replace(Some(Rc::downgrade(new_parent)));
             assert!(Rc::ptr_eq(
                 node,
-                &previous_parent.unwrap().upgrade().expect("dangling weak")
+                &previous_parent
+                    .expect("invariant: child must have a parent during reparenting")
+                    .upgrade()
+                    .expect("dangling weak")
             ))
         }
         new_children.extend(mem::take(&mut *children));

data/vendor/html-to-markdown-rs/src/text.rs

@@ -6,15 +6,17 @@ use std::borrow::Cow;
 use std::sync::LazyLock;
 
 /// Regex for escaping miscellaneous characters
-static ESCAPE_MISC_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"([\\&<`\[\]>~#=+|\-])").unwrap());
+static ESCAPE_MISC_RE: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"([\\&<`\[\]>~#=+|\-])").expect("valid regex pattern"));
 
 /// Regex for escaping numbered lists
-static ESCAPE_NUMBERED_LIST_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"([0-9])([.)])").unwrap());
+static ESCAPE_NUMBERED_LIST_RE: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"([0-9])([.)])").expect("valid regex pattern"));
 
 /// Regex for escaping ASCII punctuation (CommonMark spec example 12)
 /// Matches: `! " # $ % & ' ( ) * + , - . / : ; < = > ? @ [ \ ] ^ _ \` { | } ~`
 static ESCAPE_ASCII_RE: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"([!\x22#$%&\x27()*+,\-./:;<=>?@\[\\\]^_`{|}~])").unwrap());
+    LazyLock::new(|| Regex::new(r"([!\x22#$%&\x27()*+,\-./:;<=>?@\[\\\]^_`{|}~])").expect("valid regex pattern"));
 
 /// Escape Markdown special characters in text.
 ///
@@ -99,28 +101,37 @@ pub fn escape(
         }
     }
 
-    let mut result = text.to_string();
+    let mut result: Cow<'_, str> = Cow::Borrowed(text);
 
     if escape_ascii {
-        result = ESCAPE_ASCII_RE.replace_all(&result, r"\$1").to_string();
-        return Cow::Owned(result);
+        result = match ESCAPE_ASCII_RE.replace_all(result.as_ref(), r"\$1") {
+            Cow::Borrowed(_) => result,
+            Cow::Owned(s) => Cow::Owned(s),
+        };
+        return result;
     }
 
     if escape_misc {
-        result = ESCAPE_MISC_RE.replace_all(&result, r"\$1").to_string();
-
-        result = ESCAPE_NUMBERED_LIST_RE.replace_all(&result, r"$1\$2").to_string();
+        result = match ESCAPE_MISC_RE.replace_all(result.as_ref(), r"\$1") {
+            Cow::Borrowed(_) => result,
+            Cow::Owned(s) => Cow::Owned(s),
+        };
+
+        result = match ESCAPE_NUMBERED_LIST_RE.replace_all(result.as_ref(), r"$1\$2") {
+            Cow::Borrowed(_) => result,
+            Cow::Owned(s) => Cow::Owned(s),
+        };
     }
 
-    if escape_asterisks {
-        result = result.replace('*', r"\*");
+    if escape_asterisks && result.contains('*') {
+        result = Cow::Owned(result.replace('*', r"\*"));
     }
 
-    if escape_underscores {
-        result = result.replace('_', r"\_");
+    if escape_underscores && result.contains('_') {
+        result = Cow::Owned(result.replace('_', r"\_"));
     }
 
-    Cow::Owned(result)
+    result
 }
 
 /// Extract boundary whitespace from text (chomp).

data/vendor/html-to-markdown-rs/src/types/document.rs

@@ -0,0 +1,175 @@
+//! Structured document tree types aligned with kreuzberg's `DocumentStructure`.
+
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use super::tables::TableGrid;
+
+/// A structured document tree representing the semantic content of an HTML document.
+///
+/// Uses a flat node array with index-based parent/child references for efficient traversal.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DocumentStructure {
+    /// All nodes in document reading order.
+    pub nodes: Vec<DocumentNode>,
+    /// The source format (always "html" for this crate).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub source_format: Option<String>,
+}
+
+/// A single node in the document tree.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DocumentNode {
+    /// Deterministic node identifier.
+    pub id: String,
+    /// The semantic content of this node.
+    pub content: NodeContent,
+    /// Index of the parent node (None for root nodes).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub parent: Option<u32>,
+    /// Indices of child nodes in reading order.
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    pub children: Vec<u32>,
+    /// Inline formatting annotations (bold, italic, links, etc.) with byte offsets into the text.
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    pub annotations: Vec<TextAnnotation>,
+    /// Format-specific attributes (e.g. class, id, data-* attributes).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub attributes: Option<HashMap<String, String>>,
+}
+
+/// The semantic content type of a document node.
+///
+/// Uses internally tagged representation (`"node_type": "heading"`) for JSON serialization.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "node_type", rename_all = "snake_case")]
+pub enum NodeContent {
+    /// A heading element (h1-h6).
+    Heading {
+        /// Heading level (1-6).
+        level: u8,
+        /// The heading text content.
+        text: String,
+    },
+    /// A paragraph of text.
+    Paragraph {
+        /// The paragraph text content.
+        text: String,
+    },
+    /// A list container (ordered or unordered). Children are `ListItem` nodes.
+    List {
+        /// Whether this is an ordered list.
+        ordered: bool,
+    },
+    /// A single list item.
+    ListItem {
+        /// The list item text content.
+        text: String,
+    },
+    /// A table with structured cell data.
+    Table {
+        /// The table grid structure.
+        grid: TableGrid,
+    },
+    /// An image element.
+    Image {
+        /// Alt text or caption.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        description: Option<String>,
+        /// Image source URL.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        src: Option<String>,
+        /// Index into `ConversionResult.images` when image extraction is enabled.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        image_index: Option<u32>,
+    },
+    /// A code block or inline code.
+    Code {
+        /// The code text content.
+        text: String,
+        /// Programming language (from class="language-*" or similar).
+        #[serde(skip_serializing_if = "Option::is_none")]
+        language: Option<String>,
+    },
+    /// A block quote container.
+    Quote,
+    /// A definition list container.
+    DefinitionList,
+    /// A definition list entry with term and description.
+    DefinitionItem {
+        /// The term being defined.
+        term: String,
+        /// The definition text.
+        definition: String,
+    },
+    /// A raw block preserved as-is (e.g. `<script>`, `<style>` content).
+    RawBlock {
+        /// The format of the raw content (e.g. "html", "css", "javascript").
+        format: String,
+        /// The raw content.
+        content: String,
+    },
+    /// A block of key-value metadata pairs (from `<head>` meta tags).
+    MetadataBlock {
+        /// Key-value metadata pairs.
+        entries: Vec<(String, String)>,
+    },
+    /// A section grouping container (auto-generated from heading hierarchy).
+    Group {
+        /// Optional section label.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        label: Option<String>,
+        /// The heading level that created this group.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        heading_level: Option<u8>,
+        /// The heading text that created this group.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        heading_text: Option<String>,
+    },
+}
+
+/// An inline text annotation with byte-range offsets.
+///
+/// Annotations describe formatting (bold, italic, etc.) and links within a node's text content.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TextAnnotation {
+    /// Start byte offset (inclusive) into the parent node's text.
+    pub start: u32,
+    /// End byte offset (exclusive) into the parent node's text.
+    pub end: u32,
+    /// The type of annotation.
+    pub kind: AnnotationKind,
+}
+
+/// The type of an inline text annotation.
+///
+/// Uses internally tagged representation (`"annotation_type": "bold"`) for JSON serialization.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "annotation_type", rename_all = "snake_case")]
+pub enum AnnotationKind {
+    /// Bold / strong emphasis.
+    Bold,
+    /// Italic / emphasis.
+    Italic,
+    /// Underline.
+    Underline,
+    /// Strikethrough / deleted text.
+    Strikethrough,
+    /// Inline code.
+    Code,
+    /// Subscript text.
+    Subscript,
+    /// Superscript text.
+    Superscript,
+    /// Highlighted / marked text.
+    Highlight,
+    /// A hyperlink.
+    Link {
+        /// The link URL.
+        url: String,
+        /// Optional link title attribute.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        title: Option<String>,
+    },
+}

data/vendor/html-to-markdown-rs/src/types/mod.rs

@@ -0,0 +1,17 @@
+//! Core types for structured HTML extraction results.
+//!
+//! These types are aligned with kreuzberg's `DocumentStructure` model for seamless integration.
+
+mod document;
+mod result;
+pub mod structure_builder;
+pub mod structure_collector;
+mod tables;
+mod warnings;
+
+pub use document::{AnnotationKind, DocumentNode, DocumentStructure, NodeContent, TextAnnotation};
+pub use result::ConversionResult;
+pub use structure_builder::build_document_structure;
+pub use structure_collector::{StructureCollector, StructureCollectorHandle};
+pub use tables::{GridCell, TableData, TableGrid};
+pub use warnings::{ProcessingWarning, WarningKind};

data/vendor/html-to-markdown-rs/src/types/result.rs

@@ -0,0 +1,49 @@
+//! The primary result type for HTML conversion and extraction.
+
+use super::document::DocumentStructure;
+use super::tables::TableData;
+use super::warnings::ProcessingWarning;
+
+/// The primary result of HTML conversion and extraction.
+///
+/// Contains the converted text output, optional structured document tree,
+/// metadata, extracted tables, images, and processing warnings.
+///
+/// # Example
+///
+/// ```rust,ignore
+/// use html_to_markdown_rs::{convert, ConversionOptions};
+///
+/// let result = convert("<h1>Hello</h1><p>World</p>", None)?;
+/// assert!(result.content.is_some());
+/// assert!(result.warnings.is_empty());
+/// ```
+#[derive(Debug, Clone, Default)]
+pub struct ConversionResult {
+    /// Converted text output (markdown, djot, or plain text).
+    ///
+    /// `None` when `output_format` is set to `OutputFormat::None`,
+    /// indicating extraction-only mode.
+    pub content: Option<String>,
+
+    /// Structured document tree with semantic elements.
+    ///
+    /// Populated when `include_document_structure` is `true` in options.
+    pub document: Option<DocumentStructure>,
+
+    /// Extracted HTML metadata (title, OG, links, images, structured data).
+    #[cfg(feature = "metadata")]
+    pub metadata: crate::metadata::HtmlMetadata,
+
+    /// Extracted tables with structured cell data and markdown representation.
+    pub tables: Vec<TableData>,
+
+    /// Extracted inline images (data URIs and SVGs).
+    ///
+    /// Populated when `extract_images` is `true` in options.
+    #[cfg(feature = "inline-images")]
+    pub images: Vec<crate::inline_images::InlineImage>,
+
+    /// Non-fatal processing warnings.
+    pub warnings: Vec<ProcessingWarning>,
+}