html-to-markdown 2.27.2 → 2.28.0

data/rust-vendor/getrandom/src/backends/windows.rs

@@ -49,9 +49,25 @@ const TRUE: BOOL = 1;
 #[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     let result = unsafe { ProcessPrng(dest.as_mut_ptr().cast::<u8>(), dest.len()) };
-    // `ProcessPrng` is documented to always return TRUE. All potential errors are handled
-    // during loading of `BCryptPrimitive.dll`. See the "Process base PRNG" section
-    // in the aforementioned Windows RNG whitepaper for more information.
-    debug_assert!(result == TRUE);
-    Ok(())
+    // On Windows 10 and later, `ProcessPrng` is documented to always return
+    // TRUE. All potential errors are handled during loading of
+    // `BCryptPrimitive.dll`. See the "Process base PRNG" section in the
+    // aforementioned Windows RNG whitepaper for more information.
+    //
+    // The Zig project found that Windows 8 implements `ProcessPrng` in a way
+    // that may fail and return a value other than `TRUE`. Although recent
+    // versions of the Rust toolchain do not support Windows 8, we cannot rule
+    // out this backend being used in an executable that will run on Windows 8
+    // (e.g. a fork of this crate backported to have an MSRV lower than 1.76,
+    // or a fork of the Rust toolchain to support older Windows versions, or
+    // other build hacks).
+    //
+    // Further, Wine's implementation of `ProcessPrng` CAN fail, in every
+    // version through Wine 11.2, and this may be the case for any other Windows
+    // emulation layers.
+    if result == TRUE {
+        Ok(())
+    } else {
+        Err(Error::UNEXPECTED)
+    }
 }

data/rust-vendor/getrandom/src/utils/lazy_bool.rs

@@ -0,0 +1,39 @@
+use core::sync::atomic::{AtomicU8, Ordering::Relaxed};
+
+/// Lazily caches a `bool` in an `AtomicU8`.
+///
+/// Initialization is intentionally unsynchronized: concurrent callers may race
+/// and run `init` more than once. Once a value is produced, it is cached and
+/// reused by subsequent calls.
+///
+/// Uses `Relaxed` ordering because this helper only publishes the cached
+/// value itself.
+pub(crate) struct LazyBool(AtomicU8);
+
+impl LazyBool {
+    const UNINIT: u8 = u8::MAX;
+
+    /// Create new `LazyBool`.
+    pub const fn new() -> Self {
+        Self(AtomicU8::new(Self::UNINIT))
+    }
+
+    /// Call the `init` closure and return the result after caching it.
+    #[cold]
+    fn cold_init(&self, init: impl FnOnce() -> bool) -> bool {
+        let val = u8::from(init());
+        self.0.store(val, Relaxed);
+        val != 0
+    }
+
+    /// Retrieve the cached value if it was already initialized or call the `init` closure
+    /// and return the result after caching it.
+    #[inline]
+    pub fn unsync_init(&self, init: impl FnOnce() -> bool) -> bool {
+        let val = self.0.load(Relaxed);
+        if val == Self::UNINIT {
+            return self.cold_init(init);
+        }
+        val != 0
+    }
+}

data/rust-vendor/getrandom/src/utils/lazy_ptr.rs

@@ -0,0 +1,57 @@
+use core::{
+    convert::Infallible,
+    ptr::{self, NonNull},
+    sync::atomic::{AtomicPtr, Ordering::Relaxed},
+};
+
+/// Lazily caches a non-null pointer in an `AtomicPtr`.
+///
+/// Initialization is intentionally unsynchronized: concurrent callers may race
+/// and run `init` more than once. Once a value is produced, it is cached and
+/// reused by subsequent calls.
+///
+/// For fallible initialization (`try_unsync_init`), only successful values are
+/// cached; errors are returned to the caller and are not cached.
+///
+/// Uses `Ordering::Relaxed` because this helper only publishes the cached
+/// pointer value. Callers must not rely on this mechanism to synchronize
+/// unrelated memory side effects performed by `init`.
+pub(crate) struct LazyPtr<T>(AtomicPtr<T>);
+
+impl<T> LazyPtr<T> {
+    /// Create new `LazyPtr`.
+    pub const fn new() -> Self {
+        Self(AtomicPtr::new(ptr::null_mut()))
+    }
+
+    /// Call the `init` closure and return the result after caching it in the case of success.
+    #[cold]
+    fn cold_init<E>(&self, init: impl FnOnce() -> Result<NonNull<T>, E>) -> Result<NonNull<T>, E> {
+        let val = init()?;
+        self.0.store(val.as_ptr(), Relaxed);
+        Ok(val)
+    }
+
+    /// Retrieve the cached value if it was already initialized or call the potentially fallible
+    /// `init` closure and return the result after caching it in the case of success.
+    #[inline]
+    pub fn try_unsync_init<E>(
+        &self,
+        init: impl FnOnce() -> Result<NonNull<T>, E>,
+    ) -> Result<NonNull<T>, E> {
+        let p = self.0.load(Relaxed);
+        match NonNull::new(p) {
+            Some(val) => Ok(val),
+            None => self.cold_init(init),
+        }
+    }
+
+    /// Retrieve the cached value if it was already initialized or call the `init` closure
+    /// and return the result after caching it.
+    #[inline]
+    #[allow(dead_code, reason = "Some modules use only `try_unsync_init`")]
+    pub fn unsync_init(&self, init: impl FnOnce() -> NonNull<T>) -> NonNull<T> {
+        let Ok(p): Result<_, Infallible> = self.try_unsync_init(|| Ok(init()));
+        p
+    }
+}

data/rust-vendor/html-to-markdown-rs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "html-to-markdown-rs"
-version = "2.27.2"
+version = "2.28.0"
 edition = "2024"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
 license = "MIT"
@@ -40,7 +40,7 @@ serde = { version = "1.0", features = ["derive"], optional = true }
 serde_json = { version = "1.0", optional = true }
 async-trait = { version = "0.1", optional = true }
 futures = { version = "0.3", optional = true }
-tokio = { version = "1.49", features = ["rt-multi-thread", "sync"], optional = true }
+tokio = { version = "1.50", features = ["rt-multi-thread", "sync"], optional = true }
 
 [dev-dependencies]
 serde = { version = "1.0", features = ["derive"] }

data/rust-vendor/html-to-markdown-rs/README.md

@@ -148,6 +148,35 @@ for (i, img) in extraction.inline_images.iter().enumerate() {
 }
 ```
 
+## Table Extraction
+
+Extract structured table data alongside the Markdown conversion. Each table found in the HTML is returned with its cell contents, header row flags, and rendered Markdown output.
+
+Requires the `visitor` feature.
+
+```rust
+use html_to_markdown_rs::convert_with_tables;
+
+let html = r#"
+<table>
+    <tr><th>Name</th><th>Age</th></tr>
+    <tr><td>Alice</td><td>30</td></tr>
+    <tr><td>Bob</td><td>25</td></tr>
+</table>
+"#;
+
+let result = convert_with_tables(html, None, None)?;
+
+println!("{}", result.content);
+for table in &result.tables {
+    println!("Table with {} rows:", table.cells.len());
+    for (i, row) in table.cells.iter().enumerate() {
+        let prefix = if table.is_header_row[i] { "Header" } else { "Row" };
+        println!("  {}: {:?}", prefix, row);
+    }
+}
+```
+
 ## Other Language Bindings
 
 This is the core Rust library. For other languages:

data/rust-vendor/html-to-markdown-rs/src/convert_api.rs

@@ -681,3 +681,371 @@ pub fn metadata_config_from_json(json: &str) -> Result<MetadataConfig> {
     let update: crate::MetadataConfigUpdate = parse_json(json)?;
     Ok(MetadataConfig::from(update))
 }
+
+// ============================================================================
+// Table Extraction API (requires visitor feature)
+// ============================================================================
+
+/// Extracted table data from HTML conversion.
+///
+/// Each instance represents a single `<table>` element found during conversion.
+/// Tables are collected in document order.
+#[cfg(feature = "visitor")]
+#[derive(Debug, Clone)]
+#[cfg_attr(
+    any(feature = "serde", feature = "metadata"),
+    derive(serde::Serialize, serde::Deserialize)
+)]
+pub struct TableData {
+    /// Table cells organized as rows x columns. Cell contents are already
+    /// converted to the target output format (markdown/djot/plain).
+    pub cells: Vec<Vec<String>>,
+    /// Complete rendered table in the target output format.
+    pub markdown: String,
+    /// Per-row flag indicating whether the row was inside `<thead>`.
+    pub is_header_row: Vec<bool>,
+}
+
+/// Result of HTML-to-markdown conversion with extracted table data.
+#[cfg(feature = "visitor")]
+#[derive(Debug, Clone)]
+#[cfg_attr(
+    any(feature = "serde", feature = "metadata"),
+    derive(serde::Serialize, serde::Deserialize)
+)]
+pub struct ConversionWithTables {
+    /// Converted markdown/djot/plain text content.
+    pub content: String,
+    /// Extended metadata (if metadata extraction was requested).
+    #[cfg(feature = "metadata")]
+    pub metadata: Option<ExtendedMetadata>,
+    /// All tables found in the HTML, in document order.
+    pub tables: Vec<TableData>,
+}
+
+#[cfg(feature = "visitor")]
+#[derive(Debug)]
+struct TableCollector {
+    tables: Vec<TableData>,
+    current_rows: Vec<Vec<String>>,
+    current_is_header: Vec<bool>,
+}
+
+#[cfg(feature = "visitor")]
+impl TableCollector {
+    fn new() -> Self {
+        Self {
+            tables: Vec::new(),
+            current_rows: Vec::new(),
+            current_is_header: Vec::new(),
+        }
+    }
+}
+
+#[cfg(feature = "visitor")]
+impl visitor::HtmlVisitor for TableCollector {
+    fn visit_table_start(&mut self, _ctx: &visitor::NodeContext) -> visitor::VisitResult {
+        self.current_rows.clear();
+        self.current_is_header.clear();
+        visitor::VisitResult::Continue
+    }
+
+    fn visit_table_row(
+        &mut self,
+        _ctx: &visitor::NodeContext,
+        cells: &[String],
+        is_header: bool,
+    ) -> visitor::VisitResult {
+        self.current_rows.push(cells.to_vec());
+        self.current_is_header.push(is_header);
+        visitor::VisitResult::Continue
+    }
+
+    fn visit_table_end(&mut self, _ctx: &visitor::NodeContext, output: &str) -> visitor::VisitResult {
+        if !self.current_rows.is_empty() {
+            self.tables.push(TableData {
+                cells: std::mem::take(&mut self.current_rows),
+                markdown: output.to_string(),
+                is_header_row: std::mem::take(&mut self.current_is_header),
+            });
+        }
+        visitor::VisitResult::Continue
+    }
+}
+
+/// Convert HTML to markdown/djot/plain text with structured table extraction.
+///
+/// Combines conversion, optional metadata extraction, and table data collection
+/// in a single DOM walk. Each table found in the HTML is returned with its
+/// cell contents (already converted to the target format) and rendered output.
+///
+/// # Arguments
+///
+/// * `html` - The HTML string to convert
+/// * `options` - Optional conversion options (defaults to `ConversionOptions::default()`)
+/// * `metadata_cfg` - Optional metadata extraction configuration (requires `metadata` feature)
+///
+/// # Example
+///
+/// ```ignore
+/// use html_to_markdown_rs::convert_with_tables;
+///
+/// let html = r#"<table><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr></table>"#;
+/// let result = convert_with_tables(html, None, None).unwrap();
+/// assert_eq!(result.tables.len(), 1);
+/// assert_eq!(result.tables[0].cells[0], vec!["Name", "Age"]);
+/// ```
+///
+/// # Errors
+///
+/// Returns an error if HTML parsing fails or if the input contains invalid UTF-8.
+#[cfg(feature = "visitor")]
+pub fn convert_with_tables(
+    html: &str,
+    options: Option<ConversionOptions>,
+    #[cfg(feature = "metadata")] metadata_cfg: Option<MetadataConfig>,
+    #[cfg(not(feature = "metadata"))] _metadata_cfg: Option<()>,
+) -> Result<ConversionWithTables> {
+    use std::cell::RefCell;
+    use std::rc::Rc;
+
+    let collector = Rc::new(RefCell::new(TableCollector::new()));
+    let visitor_handle: visitor::VisitorHandle = Rc::clone(&collector) as visitor::VisitorHandle;
+
+    #[cfg(feature = "metadata")]
+    let result = {
+        let metadata_config = metadata_cfg.unwrap_or_default();
+        let (content, metadata) = convert_with_metadata(html, options, metadata_config, Some(visitor_handle))?;
+        let tables = Rc::try_unwrap(collector)
+            .map_err(|_| ConversionError::Other("failed to recover table collector state".into()))?
+            .into_inner()
+            .tables;
+        ConversionWithTables {
+            content,
+            metadata: Some(metadata),
+            tables,
+        }
+    };
+
+    #[cfg(not(feature = "metadata"))]
+    let result = {
+        let content = convert_with_visitor(html, options, Some(visitor_handle))?;
+        let tables = Rc::try_unwrap(collector)
+            .map_err(|_| ConversionError::Other("failed to recover table collector state".into()))?
+            .into_inner()
+            .tables;
+        ConversionWithTables { content, tables }
+    };
+
+    Ok(result)
+}
+
+#[cfg(test)]
+#[cfg(feature = "visitor")]
+mod table_extraction_tests {
+    use super::*;
+
+    fn tables_from_html(html: &str) -> ConversionWithTables {
+        convert_with_tables(
+            html,
+            None,
+            #[cfg(feature = "metadata")]
+            None,
+            #[cfg(not(feature = "metadata"))]
+            None,
+        )
+        .unwrap()
+    }
+
+    #[test]
+    fn test_convert_with_tables_basic() {
+        let html = r#"<table><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr></table>"#;
+        let result = tables_from_html(html);
+        assert_eq!(result.tables.len(), 1);
+        assert_eq!(result.tables[0].cells.len(), 2);
+        assert_eq!(result.tables[0].cells[0], vec!["Name", "Age"]);
+        assert_eq!(result.tables[0].cells[1], vec!["Alice", "30"]);
+        assert!(result.tables[0].is_header_row[0]);
+        assert!(!result.tables[0].is_header_row[1]);
+        assert!(result.tables[0].markdown.contains('|'));
+    }
+
+    #[test]
+    fn test_convert_with_tables_nested() {
+        let html = r#"
+        <table>
+            <tr><th>Category</th><th>Details</th></tr>
+            <tr>
+                <td>Project Alpha</td>
+                <td>
+                    <table>
+                        <tr><th>Task</th><th>Status</th></tr>
+                        <tr><td>001</td><td>Done</td></tr>
+                    </table>
+                </td>
+            </tr>
+        </table>"#;
+        let result = tables_from_html(html);
+        assert!(
+            result.tables.len() >= 2,
+            "Expected at least 2 tables (outer + nested), got {}",
+            result.tables.len()
+        );
+    }
+
+    #[test]
+    fn test_convert_with_tables_no_tables() {
+        let html = "<p>No tables here</p>";
+        let result = tables_from_html(html);
+        assert!(result.tables.is_empty());
+        assert!(result.content.contains("No tables here"));
+    }
+
+    #[test]
+    fn test_convert_with_tables_empty_table() {
+        let result = tables_from_html("<table></table>");
+        assert!(result.tables.is_empty(), "Empty table should not produce TableData");
+    }
+
+    #[test]
+    fn test_convert_with_tables_headers_only() {
+        let html = r#"<table><thead><tr><th>A</th><th>B</th></tr></thead></table>"#;
+        let result = tables_from_html(html);
+        assert_eq!(result.tables.len(), 1);
+        assert!(result.tables[0].is_header_row[0]);
+        assert_eq!(result.tables[0].cells[0], vec!["A", "B"]);
+    }
+
+    #[test]
+    fn test_convert_with_tables_thead_tbody_tfoot() {
+        let html = r#"
+        <table>
+            <thead><tr><th>H1</th></tr></thead>
+            <tbody><tr><td>B1</td></tr></tbody>
+            <tfoot><tr><td>F1</td></tr></tfoot>
+        </table>"#;
+        let result = tables_from_html(html);
+        assert_eq!(result.tables.len(), 1);
+        let t = &result.tables[0];
+        assert!(t.is_header_row[0], "thead row should be header");
+        assert!(!t.is_header_row[1], "tbody row should not be header");
+        assert_eq!(t.cells[0], vec!["H1"]);
+        assert_eq!(t.cells[1], vec!["B1"]);
+    }
+
+    #[test]
+    fn test_convert_with_tables_multiple_separate() {
+        let html = r#"
+        <table><tr><td>T1</td></tr></table>
+        <p>Between tables</p>
+        <table><tr><td>T2</td></tr></table>"#;
+        let result = tables_from_html(html);
+        assert_eq!(result.tables.len(), 2, "Should find 2 separate tables");
+    }
+
+    #[test]
+    fn test_convert_with_tables_special_chars() {
+        let html = r#"<table><tr><td>a | b</td><td>c*d</td></tr></table>"#;
+        let result = tables_from_html(html);
+        assert_eq!(result.tables.len(), 1);
+        assert!(!result.tables[0].cells[0].is_empty());
+    }
+
+    #[test]
+    fn test_convert_with_tables_single_cell() {
+        let html = r#"<table><tr><td>Only cell</td></tr></table>"#;
+        let result = tables_from_html(html);
+        assert_eq!(result.tables.len(), 1);
+        assert_eq!(result.tables[0].cells.len(), 1);
+        assert_eq!(result.tables[0].cells[0], vec!["Only cell"]);
+    }
+
+    #[test]
+    fn test_convert_with_tables_content_preserved() {
+        let html = r#"<p>Before</p><table><tr><td>Cell</td></tr></table><p>After</p>"#;
+        let result = tables_from_html(html);
+        assert!(result.content.contains("Before"));
+        assert!(result.content.contains("After"));
+        assert!(result.content.contains('|'), "Markdown table should appear in content");
+    }
+
+    #[test]
+    fn test_convert_with_tables_with_options() {
+        let options = ConversionOptions {
+            heading_style: crate::options::HeadingStyle::Underlined,
+            ..ConversionOptions::default()
+        };
+        let html = r#"<h1>Title</h1><table><tr><td>Cell</td></tr></table>"#;
+        let result = convert_with_tables(
+            html,
+            Some(options),
+            #[cfg(feature = "metadata")]
+            None,
+            #[cfg(not(feature = "metadata"))]
+            None,
+        )
+        .unwrap();
+        assert_eq!(result.tables.len(), 1);
+        assert!(result.content.contains("Title"));
+    }
+
+    #[test]
+    fn test_convert_with_tables_plain_text_format() {
+        let options = ConversionOptions {
+            output_format: crate::options::OutputFormat::Plain,
+            ..ConversionOptions::default()
+        };
+        let html = r#"<table><tr><th>Name</th></tr><tr><td>Alice</td></tr></table>"#;
+        let result = convert_with_tables(
+            html,
+            Some(options),
+            #[cfg(feature = "metadata")]
+            None,
+            #[cfg(not(feature = "metadata"))]
+            None,
+        )
+        .unwrap();
+        assert!(
+            !result.tables.is_empty(),
+            "Tables should be populated even with plain text output format"
+        );
+        assert_eq!(result.tables[0].cells[0], vec!["Name"]);
+    }
+
+    #[cfg(feature = "metadata")]
+    #[test]
+    fn test_convert_with_tables_metadata_integration() {
+        let html = r#"<html lang="en"><head><title>Test</title></head><body>
+            <table><tr><th>Col</th></tr><tr><td>Val</td></tr></table>
+        </body></html>"#;
+        let config = MetadataConfig::default();
+        let result = convert_with_tables(html, None, Some(config)).unwrap();
+        assert_eq!(result.tables.len(), 1);
+        let meta = result.metadata.as_ref().expect("metadata should be present");
+        assert_eq!(meta.document.language, Some("en".to_string()));
+    }
+
+    #[cfg(feature = "metadata")]
+    #[test]
+    fn test_convert_with_tables_plain_text_metadata() {
+        let options = ConversionOptions {
+            output_format: crate::options::OutputFormat::Plain,
+            ..ConversionOptions::default()
+        };
+        let html = r#"<html lang="fr"><body>
+            <table><tr><td>Cell</td></tr></table>
+        </body></html>"#;
+        let config = MetadataConfig::default();
+        let result = convert_with_tables(html, Some(options), Some(config)).unwrap();
+        assert!(
+            !result.tables.is_empty(),
+            "Tables should be populated in plain text mode"
+        );
+        let meta = result.metadata.as_ref().expect("metadata should be present");
+        assert_eq!(
+            meta.document.language,
+            Some("fr".to_string()),
+            "Metadata should be populated in plain text mode"
+        );
+    }
+}

data/rust-vendor/html-to-markdown-rs/src/converter/main.rs

@@ -136,11 +136,9 @@ pub(crate) fn convert_html_impl(
         }
     }
 
-    // Fast path for plain text output: skip the full conversion pipeline
-    if options.output_format == OutputFormat::Plain {
-        let plain = extract_plain_text(&dom, parser, options);
-        return Ok(plain);
-    }
+    // Plain text output: run the full pipeline (for metadata + visitor callbacks),
+    // then return plain text instead of markdown.
+    let is_plain_text = options.output_format == OutputFormat::Plain;
 
     let wants_frontmatter = options.extract_metadata && !options.convert_as_inline;
     #[cfg(feature = "metadata")]
@@ -230,6 +228,13 @@ pub(crate) fn convert_html_impl(
         return Err(crate::error::ConversionError::Visitor(err.clone()));
     }
 
+    // If plain text was requested, discard the markdown output and return plain text.
+    // The full pipeline was still run above so that metadata + visitor callbacks fire.
+    if is_plain_text {
+        let plain = extract_plain_text(&dom, parser, options);
+        return Ok(plain);
+    }
+
     trim_line_end_whitespace(&mut output);
     let trimmed = output.trim_end_matches('\n');
     if trimmed.is_empty() {

data/rust-vendor/html-to-markdown-rs/src/converter/text_node.rs

@@ -188,7 +188,8 @@ pub fn process_text_node(
             // the current block's content, not from a previous block's closing.
             // Without this distinction, the second paragraph after a "\n\n" boundary
             // would incorrectly suppress the trailing space before inline elements.
-            let current_block_output = &output[ctx.block_content_start..];
+            let safe_start = ctx.block_content_start.min(output.len());
+            let current_block_output = &output[safe_start..];
             let at_paragraph_break = current_block_output.ends_with("\n\n");
             if !at_paragraph_break {
                 if has_double_newline {

data/rust-vendor/html-to-markdown-rs/src/lib.rs

@@ -98,6 +98,9 @@ pub use convert_api::convert_with_metadata;
 #[cfg(feature = "visitor")]
 pub use convert_api::convert_with_visitor;
 
+#[cfg(feature = "visitor")]
+pub use convert_api::{ConversionWithTables, TableData, convert_with_tables};
+
 #[cfg(feature = "async-visitor")]
 pub use convert_api::convert_with_async_visitor;
 

data/rust-vendor/html-to-markdown-rs/src/prelude.rs

@@ -19,5 +19,8 @@ pub use crate::convert_with_metadata;
 #[cfg(feature = "visitor")]
 pub use crate::convert_with_visitor;
 
+#[cfg(feature = "visitor")]
+pub use crate::{ConversionWithTables, TableData, convert_with_tables};
+
 #[cfg(feature = "async-visitor")]
 pub use crate::convert_with_async_visitor;