html-to-markdown 2.29.0 → 3.0.0

data/vendor/html-to-markdown-rs/src/types/structure_collector.rs

@@ -0,0 +1,442 @@
+//! Collector that builds a [`DocumentStructure`] during the converter's HTML DOM walk.
+//!
+//! Follows the same single-pass collector pattern used by [`crate::metadata::MetadataCollector`]:
+//! an `Rc<RefCell<StructureCollector>>` handle is threaded through the [`crate::converter::Context`]
+//! and individual element handlers call `push_*` methods as they encounter content.
+//!
+//! # Design
+//!
+//! - **Flat node array** with index-based parent/child links (matches [`DocumentStructure`]).
+//! - **section_stack** tracks the currently-open heading groups (`(level, group_node_index)`).
+//! - **container_stack** tracks open blockquote containers.
+//! - **list_stack** tracks open list containers so `push_list_item` attaches items to the right list.
+//! - IDs are deterministic hashes of `(node_type, text_prefix, index)`.
+
+use std::cell::RefCell;
+use std::rc::Rc;
+
+use super::document::{DocumentNode, DocumentStructure, NodeContent};
+use super::tables::TableGrid;
+
+/// Shared mutable handle used in [`crate::converter::Context`].
+pub type StructureCollectorHandle = Rc<RefCell<StructureCollector>>;
+
+/// Incremental builder for [`DocumentStructure`] during a single DOM walk.
+pub struct StructureCollector {
+    /// Accumulated nodes in document order.
+    nodes: Vec<DocumentNode>,
+    /// Open heading-group stack: `(heading_level, node_index)`.
+    /// Mirrors the `group_stack` in `structure_builder`.
+    section_stack: Vec<(u8, u32)>,
+    /// Open blockquote container indices (innermost last).
+    container_stack: Vec<u32>,
+    /// Open list container indices (innermost last).
+    list_stack: Vec<u32>,
+}
+
+impl StructureCollector {
+    /// Create a new empty collector.
+    pub fn new() -> Self {
+        Self {
+            nodes: Vec::new(),
+            section_stack: Vec::new(),
+            container_stack: Vec::new(),
+            list_stack: Vec::new(),
+        }
+    }
+
+    // ── Public push methods ──────────────────────────────────────────────────
+
+    /// Record a heading element.
+    ///
+    /// Creates a [`NodeContent::Group`] (which owns all subsequent sibling content until a
+    /// heading of equal or higher rank closes it) followed by a [`NodeContent::Heading`] child.
+    ///
+    /// Returns the index of the **heading** node (the group node is one before it).
+    pub fn push_heading(&mut self, level: u8, text: &str, id: Option<&str>) -> u32 {
+        // Close any open groups at the same or deeper heading level.
+        while let Some(&(open_level, _)) = self.section_stack.last() {
+            if open_level >= level {
+                self.section_stack.pop();
+            } else {
+                break;
+            }
+        }
+
+        // The group's parent is the surrounding open group or a container/list (if any).
+        let group_parent = self.current_structural_parent();
+
+        // Create the Group node.
+        let group_id = Self::generate_id("group", text, self.nodes.len() as u32);
+        let group_idx = self.raw_push(DocumentNode {
+            id: group_id,
+            content: NodeContent::Group {
+                label: Some(text.to_string()),
+                heading_level: Some(level),
+                heading_text: Some(text.to_string()),
+            },
+            parent: group_parent,
+            children: Vec::new(),
+            annotations: Vec::new(),
+            attributes: None,
+        });
+        if let Some(gp) = group_parent {
+            self.add_child(gp, group_idx);
+        }
+        self.section_stack.push((level, group_idx));
+
+        // Create the Heading node as a child of the new group.
+        let heading_id = Self::generate_id("heading", text, self.nodes.len() as u32);
+        let heading_idx = self.raw_push(DocumentNode {
+            id: heading_id,
+            content: NodeContent::Heading {
+                level,
+                text: text.to_string(),
+            },
+            parent: Some(group_idx),
+            children: Vec::new(),
+            annotations: Vec::new(),
+            attributes: id.map(|v| {
+                let mut m = std::collections::HashMap::new();
+                m.insert("id".to_string(), v.to_string());
+                m
+            }),
+        });
+        self.add_child(group_idx, heading_idx);
+        heading_idx
+    }
+
+    /// Record a paragraph element.
+    ///
+    /// Returns the node index.
+    pub fn push_paragraph(&mut self, text: &str) -> u32 {
+        if text.is_empty() {
+            return u32::MAX;
+        }
+        let parent = self.current_structural_parent();
+        let id = Self::generate_id("paragraph", text, self.nodes.len() as u32);
+        let idx = self.raw_push(DocumentNode {
+            id,
+            content: NodeContent::Paragraph { text: text.to_string() },
+            parent,
+            children: Vec::new(),
+            annotations: Vec::new(),
+            attributes: None,
+        });
+        if let Some(p) = parent {
+            self.add_child(p, idx);
+        }
+        idx
+    }
+
+    /// Open a list container.
+    ///
+    /// Returns the node index; call [`push_list_end`] to close it.
+    pub fn push_list_start(&mut self, ordered: bool) -> u32 {
+        let parent = self.current_structural_parent();
+        let label = if ordered { "ordered" } else { "unordered" };
+        let id = Self::generate_id("list", label, self.nodes.len() as u32);
+        let idx = self.raw_push(DocumentNode {
+            id,
+            content: NodeContent::List { ordered },
+            parent,
+            children: Vec::new(),
+            annotations: Vec::new(),
+            attributes: None,
+        });
+        if let Some(p) = parent {
+            self.add_child(p, idx);
+        }
+        self.list_stack.push(idx);
+        idx
+    }
+
+    /// Close the innermost open list container.
+    pub fn push_list_end(&mut self) {
+        self.list_stack.pop();
+    }
+
+    /// Record a list item under the current open list.
+    ///
+    /// If there is no open list, the item is parented under the current section/container.
+    /// Returns the node index.
+    pub fn push_list_item(&mut self, text: &str) -> u32 {
+        let parent = self
+            .list_stack
+            .last()
+            .copied()
+            .or_else(|| self.current_structural_parent());
+        let id = Self::generate_id("list_item", text, self.nodes.len() as u32);
+        let idx = self.raw_push(DocumentNode {
+            id,
+            content: NodeContent::ListItem { text: text.to_string() },
+            parent,
+            children: Vec::new(),
+            annotations: Vec::new(),
+            attributes: None,
+        });
+        if let Some(p) = parent {
+            self.add_child(p, idx);
+        }
+        idx
+    }
+
+    /// Record a table.
+    ///
+    /// Returns the node index.
+    pub fn push_table(&mut self, grid: TableGrid) -> u32 {
+        let parent = self.current_structural_parent();
+        let label = grid.rows.to_string();
+        let id = Self::generate_id("table", &label, self.nodes.len() as u32);
+        let idx = self.raw_push(DocumentNode {
+            id,
+            content: NodeContent::Table { grid },
+            parent,
+            children: Vec::new(),
+            annotations: Vec::new(),
+            attributes: None,
+        });
+        if let Some(p) = parent {
+            self.add_child(p, idx);
+        }
+        idx
+    }
+
+    /// Record an image element.
+    ///
+    /// Returns the node index.
+    pub fn push_image(&mut self, src: Option<&str>, alt: Option<&str>) -> u32 {
+        let parent = self.current_structural_parent();
+        let label = src.unwrap_or("img");
+        let id = Self::generate_id("image", label, self.nodes.len() as u32);
+        let idx = self.raw_push(DocumentNode {
+            id,
+            content: NodeContent::Image {
+                description: alt.filter(|s| !s.is_empty()).map(str::to_string),
+                src: src.map(str::to_string),
+                image_index: None,
+            },
+            parent,
+            children: Vec::new(),
+            annotations: Vec::new(),
+            attributes: None,
+        });
+        if let Some(p) = parent {
+            self.add_child(p, idx);
+        }
+        idx
+    }
+
+    /// Record a code block.
+    ///
+    /// Returns the node index.
+    pub fn push_code(&mut self, text: &str, language: Option<&str>) -> u32 {
+        let parent = self.current_structural_parent();
+        let id = Self::generate_id("code", text, self.nodes.len() as u32);
+        let idx = self.raw_push(DocumentNode {
+            id,
+            content: NodeContent::Code {
+                text: text.to_string(),
+                language: language.map(str::to_string),
+            },
+            parent,
+            children: Vec::new(),
+            annotations: Vec::new(),
+            attributes: None,
+        });
+        if let Some(p) = parent {
+            self.add_child(p, idx);
+        }
+        idx
+    }
+
+    /// Open a blockquote container.
+    ///
+    /// Returns the node index; call [`push_quote_end`] to close it.
+    pub fn push_quote_start(&mut self) -> u32 {
+        let parent = self.current_structural_parent();
+        let id = Self::generate_id("quote", "blockquote", self.nodes.len() as u32);
+        let idx = self.raw_push(DocumentNode {
+            id,
+            content: NodeContent::Quote,
+            parent,
+            children: Vec::new(),
+            annotations: Vec::new(),
+            attributes: None,
+        });
+        if let Some(p) = parent {
+            self.add_child(p, idx);
+        }
+        self.container_stack.push(idx);
+        idx
+    }
+
+    /// Close the innermost open blockquote container.
+    pub fn push_quote_end(&mut self) {
+        self.container_stack.pop();
+    }
+
+    /// Record a raw block (e.g. preserved `<script>` or `<style>` content).
+    ///
+    /// Returns the node index.
+    pub fn push_raw_block(&mut self, format: &str, content: &str) -> u32 {
+        let parent = self.current_structural_parent();
+        let id = Self::generate_id("raw_block", format, self.nodes.len() as u32);
+        let idx = self.raw_push(DocumentNode {
+            id,
+            content: NodeContent::RawBlock {
+                format: format.to_string(),
+                content: content.to_string(),
+            },
+            parent,
+            children: Vec::new(),
+            annotations: Vec::new(),
+            attributes: None,
+        });
+        if let Some(p) = parent {
+            self.add_child(p, idx);
+        }
+        idx
+    }
+
+    /// Consume the collector and return the completed [`DocumentStructure`].
+    pub fn finish(self) -> DocumentStructure {
+        DocumentStructure {
+            nodes: self.nodes,
+            source_format: Some("html".to_string()),
+        }
+    }
+
+    // ── Private helpers ──────────────────────────────────────────────────────
+
+    /// The effective structural parent for a new node:
+    /// list stack > container stack > section stack > None.
+    fn current_structural_parent(&self) -> Option<u32> {
+        // List items: already handled explicitly in push_list_item.
+        // For non-list-item content, prefer the innermost container (blockquote),
+        // then innermost section group.
+        if let Some(&q) = self.container_stack.last() {
+            return Some(q);
+        }
+        if let Some(&(_, g)) = self.section_stack.last() {
+            return Some(g);
+        }
+        None
+    }
+
+    /// Append a node to the flat list and return its index.
+    fn raw_push(&mut self, node: DocumentNode) -> u32 {
+        let idx = self.nodes.len() as u32;
+        self.nodes.push(node);
+        idx
+    }
+
+    /// Record `child_idx` as a child of `parent_idx`.
+    fn add_child(&mut self, parent_idx: u32, child_idx: u32) {
+        if let Some(parent) = self.nodes.get_mut(parent_idx as usize) {
+            parent.children.push(child_idx);
+        }
+    }
+
+    /// Generate a deterministic node ID: `"{node_type}-{hash:016x}"`.
+    ///
+    /// Hashes `(node_type, text[..64], index)` with `DefaultHasher`.
+    fn generate_id(node_type: &str, text: &str, index: u32) -> String {
+        use std::collections::hash_map::DefaultHasher;
+        use std::hash::{Hash, Hasher};
+
+        let mut hasher = DefaultHasher::new();
+        node_type.hash(&mut hasher);
+        text[..text.len().min(64)].hash(&mut hasher);
+        index.hash(&mut hasher);
+        let digest = hasher.finish();
+        format!("{node_type}-{digest:016x}")
+    }
+}
+
+impl Default for StructureCollector {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_heading_creates_group_and_heading() {
+        let mut c = StructureCollector::new();
+        let heading_idx = c.push_heading(1, "Title", None);
+        // Group is at index 0, Heading at index 1.
+        assert_eq!(heading_idx, 1);
+        assert_eq!(c.nodes.len(), 2);
+
+        let group = &c.nodes[0];
+        matches!(
+            &group.content,
+            NodeContent::Group {
+                heading_level: Some(1),
+                ..
+            }
+        );
+        assert!(group.children.contains(&1));
+
+        let heading = &c.nodes[1];
+        assert!(matches!(&heading.content, NodeContent::Heading { level: 1, .. }));
+        assert_eq!(heading.parent, Some(0));
+    }
+
+    #[test]
+    fn test_heading_closes_deeper_groups() {
+        let mut c = StructureCollector::new();
+        c.push_heading(1, "H1", None);
+        c.push_heading(2, "H2", None);
+        // Now push another H1 — must close the H2 group.
+        c.push_heading(1, "H1b", None);
+        // After the second H1 there should be 2 open groups gone and 1 new one.
+        assert_eq!(c.section_stack.len(), 1);
+        let (level, _) = c.section_stack[0];
+        assert_eq!(level, 1);
+    }
+
+    #[test]
+    fn test_paragraph_parents_under_section() {
+        let mut c = StructureCollector::new();
+        c.push_heading(1, "Title", None);
+        let p_idx = c.push_paragraph("Some text");
+        let para = &c.nodes[p_idx as usize];
+        // Parent should be the group node (index 0).
+        assert_eq!(para.parent, Some(0));
+    }
+
+    #[test]
+    fn test_list_items_attach_to_list() {
+        let mut c = StructureCollector::new();
+        let list_idx = c.push_list_start(false);
+        let item_idx = c.push_list_item("Item 1");
+        c.push_list_end();
+        assert_eq!(c.nodes[item_idx as usize].parent, Some(list_idx));
+        let list = &c.nodes[list_idx as usize];
+        assert!(list.children.contains(&item_idx));
+    }
+
+    #[test]
+    fn test_quote_container() {
+        let mut c = StructureCollector::new();
+        let q_idx = c.push_quote_start();
+        let p_idx = c.push_paragraph("Quoted text");
+        c.push_quote_end();
+        assert_eq!(c.nodes[p_idx as usize].parent, Some(q_idx));
+    }
+
+    #[test]
+    fn test_finish_returns_document_structure() {
+        let mut c = StructureCollector::new();
+        c.push_heading(1, "Title", None);
+        c.push_paragraph("Text");
+        let doc = c.finish();
+        assert_eq!(doc.source_format, Some("html".to_string()));
+        assert_eq!(doc.nodes.len(), 3); // Group + Heading + Paragraph
+    }
+}

data/vendor/html-to-markdown-rs/src/types/tables.rs

@@ -0,0 +1,47 @@
+//! Structured table types aligned with kreuzberg's `TableGrid`.
+
+use serde::{Deserialize, Serialize};
+
+/// A structured table grid with cell-level data including spans.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TableGrid {
+    /// Number of rows.
+    pub rows: u32,
+    /// Number of columns.
+    pub cols: u32,
+    /// All cells in the table (may be fewer than rows*cols due to spans).
+    pub cells: Vec<GridCell>,
+}
+
+/// A single cell in a table grid.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct GridCell {
+    /// The text content of the cell.
+    pub content: String,
+    /// 0-indexed row position.
+    pub row: u32,
+    /// 0-indexed column position.
+    pub col: u32,
+    /// Number of rows this cell spans (default 1).
+    #[serde(default = "default_span")]
+    pub row_span: u32,
+    /// Number of columns this cell spans (default 1).
+    #[serde(default = "default_span")]
+    pub col_span: u32,
+    /// Whether this is a header cell (`<th>`).
+    #[serde(default)]
+    pub is_header: bool,
+}
+
+fn default_span() -> u32 {
+    1
+}
+
+/// A top-level extracted table with both structured data and markdown representation.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TableData {
+    /// The structured table grid.
+    pub grid: TableGrid,
+    /// The markdown rendering of this table.
+    pub markdown: String,
+}

data/vendor/html-to-markdown-rs/src/types/warnings.rs

@@ -0,0 +1,28 @@
+//! Processing warning types for non-fatal issues during conversion.
+
+use serde::{Deserialize, Serialize};
+
+/// A non-fatal warning generated during HTML processing.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ProcessingWarning {
+    /// Human-readable warning message.
+    pub message: String,
+    /// The category of warning.
+    pub kind: WarningKind,
+}
+
+/// Categories of processing warnings.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum WarningKind {
+    /// An image could not be extracted (e.g. invalid data URI, unsupported format).
+    ImageExtractionFailed,
+    /// The input encoding was not recognized; fell back to UTF-8.
+    EncodingFallback,
+    /// The input was truncated due to size limits.
+    TruncatedInput,
+    /// The HTML was malformed but processing continued with best effort.
+    MalformedHtml,
+    /// Sanitization was applied to remove potentially unsafe content.
+    SanitizationApplied,
+}

data/vendor/html-to-markdown-rs/src/visitor/mod.rs

@@ -35,13 +35,7 @@ mod default_impl;
 mod traits;
 mod types;
 
-#[cfg(feature = "async-visitor")]
-mod async_traits;
-
 // Re-export all public items from submodules
 pub use default_impl::VisitorHandle;
 pub use traits::HtmlVisitor;
 pub use types::{NodeContext, NodeType, VisitResult};
-
-#[cfg(feature = "async-visitor")]
-pub use async_traits::AsyncHtmlVisitor;

data/vendor/html-to-markdown-rs/src/visitor/traits.rs

@@ -1,7 +1,6 @@
 //! Visitor traits for HTML to Markdown conversion.
 //!
 //! This module contains the synchronous visitor trait.
-//! For async version, see `async_traits.rs`.
 
 use super::types::{NodeContext, VisitResult};
 

data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/callbacks/mod.rs

@@ -1,21 +1 @@
-//! Callback management, macros, and async-to-sync bridge.
-//!
-//! This module provides helper macros for common visitor patterns and an async-to-sync
-//! visitor bridge for integrating async visitors with synchronous converters.
-
-mod bridge;
-mod bridge_visitor;
-mod macros;
-
-#[cfg(feature = "async-visitor")]
-pub use bridge::AsyncToSyncVisitorBridge;
-#[cfg(feature = "async-visitor")]
-pub use macros::dispatch_async_visitor;
-
-/// Type alias for an async visitor handle (Arc-wrapped `Mutex` for interior mutability).
-///
-/// This allows async visitors to be passed around and shared while still being mutable.
-/// Uses Arc<Mutex<>> instead of Rc<RefCell<>> to enable Send across thread boundaries.
-/// The + Send + 'static bounds allow the visitor to be moved to other threads.
-#[cfg(feature = "async-visitor")]
-pub type AsyncVisitorHandle = std::sync::Arc<tokio::sync::Mutex<dyn crate::visitor::AsyncHtmlVisitor + Send + 'static>>;
+//! Callback management for visitor pattern.

data/vendor/html-to-markdown-rs/src/visitor_helpers/helpers/mod.rs

@@ -22,11 +22,6 @@ pub mod content;
 pub mod state;
 pub mod traversal;
 
-#[cfg(feature = "async-visitor")]
-pub use callbacks::AsyncVisitorHandle;
 pub use content::VisitorDispatch;
 pub use state::build_node_context;
 pub use traversal::dispatch_visitor;
-
-#[cfg(feature = "async-visitor")]
-pub use callbacks::{AsyncToSyncVisitorBridge, dispatch_async_visitor};