html-to-markdown 3.4.0.pre.rc.44 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d5a2e4bc34ff80125912031e6e942772c52769430f5520f8ab312222cdf099cb
-  data.tar.gz: e3985c9fa10a4507043670283f7965c2a04a92eb6073f917e48f1514b4dac428
+  metadata.gz: e1296a51b6ca0f757a95487c567e9d1e0d2acda80a46656cd14002322a4b850b
+  data.tar.gz: dca253ecef2f4818aa9e39e56d48ca5bb9ad35f92759580ee893d17eb455d494
 SHA512:
-  metadata.gz: 4a179c7450759417b24669b8bda93462f707af97183634740a687205b7a16b2ce5169daa1369f402b17a3e9ae07a124603a6c2d3ce3f7dc5c2b1832fd42559f8
-  data.tar.gz: b56307b497e4794a1dc08b9dee9d5fd61dc0df09fa9ea40a819b7b39b8b89daa658798c0b909c2030ab9472b4b72186319f4a73a2cb6fbb194f274e783943a04
+  metadata.gz: 989e6b723c3670894b9e87ad87c1d752c27e3917f5cc60f0ba8f40175ee97c35ef4a2a40da53ab77908c6febb44abc0fe66064470494e73be3a5aae61d8498ae
+  data.tar.gz: 6275bc085c86f521114dbfd860e856dfdeac43ae92b755d81960e22e11a6d73ec4f25c89c837c37072d5500fd22af92f9507ace1673337708cac1a5376db926c

data/ext/html_to_markdown_rb/Cargo.toml

@@ -1,7 +1,7 @@
 
 [package]
 name = "html-to-markdown-rb"
-version = "3.4.0-rc.44"
+version = "3.4.0"
 edition = "2024"
 license = "MIT"
 [workspace]

data/lib/html_to_markdown/native.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:1bd3c99db21c2ca078644f1e220656297c784bcca532f13d3b7efed3f45dbb0b
+# alef:hash:b54e7bb2ab55cc6c25c9cac0e62ec66c35fd2d1956ef9ba5e3dc9e7ba5e666a5
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # Issues & docs: https://github.com/kreuzberg-dev/alef
@@ -23,7 +23,8 @@ end
 class Hash
   # Support internally-tagged enum accessors like format.excel, format.email, etc.
   # Also support direct field access like format.sheet_count
-  def method_missing(method_name, *args, &)
+  # rubocop:disable Metrics/CyclomaticComplexity
+  def method_missing(method_name, *args, &block)
     # Try symbol key first (how Magnus converts JSON keys)
     return self[method_name] if key?(method_name)
 
@@ -31,22 +32,25 @@ class Hash
     return self[method_name.to_s] if key?(method_name.to_s)
 
     # Check if this hash has a 'format_type' field (indicating an internally-tagged enum)
-    format_type = self[:format_type] || self['format_type']
+    format_type = self[:'format_type'] || self['format_type']
     return super unless format_type
 
     # If the method name matches the format_type (snake_case), extract and return the variant's wrapped data
     # Internally-tagged enums store variant data in the '_0' field (from alef's struct variant conversion)
     # This allows format.excel to return the ExcelMetadata hash with sheet_count, sheet_names, etc.
     snake_case_method = method_name.to_s.downcase
-    return self[:_0] || self['_0'] || self if snake_case_method == format_type.to_s.downcase
+    if snake_case_method == format_type.to_s.downcase
+      return self[:'_0'] || self['_0'] || self
+    end
 
     super
   end
+  # rubocop:enable Metrics/CyclomaticComplexity
 
   def respond_to_missing?(method_name, include_private = false)
     return true if key?(method_name) || key?(method_name.to_s)
 
-    format_type = self[:format_type] || self['format_type']
+    format_type = self[:'format_type'] || self['format_type']
     return false unless format_type
 
     snake_case_method = method_name.to_s.downcase

data/lib/html_to_markdown/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:4ce3ba64cbea5c72b082111f57a04e219e555fda52859a9673edcec555031706
+# alef:hash:9c58cf63849e82246f03b4fcc3996c264d47f2b2c27e0e8ba6b93eb4a84cb279
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # Issues & docs: https://github.com/kreuzberg-dev/alef
 # frozen_string_literal: true
 
 module HtmlToMarkdown
-  VERSION = '3.4.0.pre.rc.44'
+  VERSION = '3.4.0'
 end

data/vendor/Cargo.toml

@@ -3,7 +3,7 @@ members = ["html-to-markdown-rs"]
 resolver = "2"
 
 [workspace.package]
-version = "3.4.0-rc.44"
+version = "3.4.0"
 edition = "2024"
 rust-version = "1.85"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]

data/vendor/html-to-markdown-rs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "html-to-markdown-rs"
-version = "3.4.0-rc.44"
+version = "3.4.0"
 edition = "2024"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
 license = "MIT"

data/vendor/html-to-markdown-rs/src/converter/block/table/builder.rs

@@ -6,7 +6,7 @@
 use std::borrow::Cow;
 
 use super::cell::{collect_table_cells, get_colspan};
-use super::cells::{append_layout_row, convert_table_row};
+use super::cells::{append_layout_row, collect_row_cell_widths, convert_table_row};
 use super::scanner::scan_table;
 use super::utils::{is_tag_name, normalized_tag_name};
 #[cfg(feature = "visitor")]
@@ -216,6 +216,50 @@ pub fn handle_table(
         let mut rowspan_tracker = vec![None; total_cols];
         let mut row_cells = Vec::new();
 
+        // Pre-pass: compute per-column max content widths for aligned padding.
+        // Uses a rowspan tracker so spanned columns are skipped just as they
+        // are in the render pass, keeping column indices correctly aligned.
+        let col_widths = {
+            let mut widths: Vec<usize> = Vec::new();
+            let mut prepass_rowspan: Vec<Option<usize>> = Vec::new();
+            let children = tag.children();
+            for child_handle in children.top().iter() {
+                if let Some(tl::Node::Tag(child_tag)) = child_handle.get(parser) {
+                    let tag_name = normalized_tag_name(child_tag.name().as_utf8_str());
+                    match tag_name.as_ref() {
+                        "thead" | "tbody" | "tfoot" => {
+                            for row_handle in child_tag.children().top().iter() {
+                                if is_tag_name(row_handle, parser, dom_ctx, "tr") {
+                                    collect_row_cell_widths(
+                                        row_handle,
+                                        parser,
+                                        options,
+                                        ctx,
+                                        dom_ctx,
+                                        &mut widths,
+                                        &mut prepass_rowspan,
+                                    );
+                                }
+                            }
+                        }
+                        "tr" | "row" => {
+                            collect_row_cell_widths(
+                                child_handle,
+                                parser,
+                                options,
+                                ctx,
+                                dom_ctx,
+                                &mut widths,
+                                &mut prepass_rowspan,
+                            );
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            widths
+        };
+
         let children = tag.children();
         {
             for child_handle in children.top().iter() {
@@ -282,6 +326,7 @@ pub fn handle_table(
                                                 dom_ctx,
                                                 depth + 1,
                                                 is_header_section,
+                                                &col_widths,
                                             );
                                             row_index += 1;
                                         }
@@ -312,6 +357,7 @@ pub fn handle_table(
                                 dom_ctx,
                                 depth + 1,
                                 row_index == 0,
+                                &col_widths,
                             );
                             row_index += 1;
                         }

data/vendor/html-to-markdown-rs/src/converter/block/table/cell.rs

@@ -97,10 +97,86 @@ pub fn collect_table_cells(
     }
 }
 
+/// Extract the text content of a table cell for column width calculation.
+///
+/// Returns the same text that would appear in the rendered cell, without
+/// the surrounding pipe delimiters. Used in the first pass to compute
+/// maximum column widths before rendering with padding.
+///
+/// # Arguments
+/// * `node_handle` - Handle to the cell element
+/// * `parser` - HTML parser instance
+/// * `options` - Conversion options
+/// * `ctx` - Conversion context
+/// * `dom_ctx` - DOM context
+#[allow(clippy::trivially_copy_pass_by_ref)]
+pub fn cell_text_content(
+    node_handle: &tl::NodeHandle,
+    parser: &tl::Parser,
+    options: &crate::options::ConversionOptions,
+    ctx: &super::super::super::Context,
+    dom_ctx: &super::super::super::DomContext,
+) -> String {
+    let mut text = String::with_capacity(64);
+
+    let cell_ctx = super::super::super::Context {
+        in_table_cell: true,
+        ..ctx.clone()
+    };
+
+    if let Some(tl::Node::Tag(tag)) = node_handle.get(parser) {
+        let children = tag.children();
+        let has_tag_child = children
+            .top()
+            .iter()
+            .any(|child_handle| matches!(child_handle.get(parser), Some(tl::Node::Tag(_))));
+
+        if has_tag_child {
+            for child_handle in children.top().iter() {
+                super::super::super::walk_node(child_handle, parser, &mut text, options, &cell_ctx, 0, dom_ctx);
+            }
+        } else {
+            let raw = dom_ctx.text_content(*node_handle, parser);
+            let normalized = if options.whitespace_mode == crate::options::WhitespaceMode::Normalized {
+                crate::text::normalize_whitespace_cow(raw.as_str())
+            } else {
+                Cow::Borrowed(raw.as_str())
+            };
+            let escaped = escape_cell_text(normalized.as_ref(), options);
+            text = escaped;
+        }
+    }
+
+    let text = text.trim();
+    if options.br_in_tables {
+        text.to_string()
+    } else if text.contains('\n') {
+        text.replace('\n', " ")
+    } else {
+        text.to_string()
+    }
+}
+
+/// Escape text for use inside a table cell.
+///
+/// Always escapes `*` and `_` (to prevent unintended emphasis inside cells),
+/// applies `escape_misc` / `escape_ascii` per options, and escapes `|` (pipe)
+/// when `escape_misc` is not already handling it.
+fn escape_cell_text(text: &str, options: &crate::options::ConversionOptions) -> String {
+    // Always escape * and _ in table cells to prevent unintended emphasis.
+    let escaped = crate::text::escape(text, options.escape_misc, true, true, options.escape_ascii);
+    if options.escape_misc {
+        escaped.into_owned()
+    } else {
+        escaped.replace('|', r"\|")
+    }
+}
+
 /// Convert a table cell (td or th) to Markdown format.
 ///
 /// Processes cell content and renders it with pipe delimiters for Markdown tables.
 /// Handles colspan by adding extra pipes, and escapes pipes in cell content.
+/// Always escapes `*` and `_` to prevent unintended emphasis inside cells.
 ///
 /// # Arguments
 /// * `node_handle` - Handle to the cell element
@@ -110,6 +186,7 @@ pub fn collect_table_cells(
 /// * `ctx` - Conversion context (visitor, etc)
 /// * `_tag_name` - Tag name (for consistency, not used)
 /// * `dom_ctx` - DOM context for content extraction
+/// * `col_width` - Optional target width for padding (None = no padding)
 #[allow(clippy::trivially_copy_pass_by_ref)]
 pub fn convert_table_cell(
     node_handle: &tl::NodeHandle,
@@ -119,6 +196,7 @@ pub fn convert_table_cell(
     ctx: &super::super::super::Context,
     _tag_name: &str,
     dom_ctx: &super::super::super::DomContext,
+    col_width: Option<usize>,
 ) {
     let mut text = String::with_capacity(128);
 
@@ -145,18 +223,7 @@ pub fn convert_table_cell(
             } else {
                 Cow::Borrowed(raw.as_str())
             };
-            let escaped = crate::text::escape(
-                normalized.as_ref(),
-                options.escape_misc,
-                options.escape_asterisks,
-                options.escape_underscores,
-                options.escape_ascii,
-            );
-            if options.escape_misc {
-                text = escaped.into_owned();
-            } else {
-                text = escaped.replace('|', r"\|");
-            }
+            text = escape_cell_text(normalized.as_ref(), options);
         }
     }
 
@@ -175,6 +242,14 @@ pub fn convert_table_cell(
 
     output.push(' ');
     output.push_str(&text);
+    if let Some(width) = col_width {
+        let text_len = text.chars().count();
+        if text_len < width {
+            for _ in 0..(width - text_len) {
+                output.push(' ');
+            }
+        }
+    }
     for _ in 0..colspan {
         output.push_str(" |");
     }

data/vendor/html-to-markdown-rs/src/converter/block/table/cells.rs

@@ -10,7 +10,7 @@ use crate::converter::utility::content::collect_tag_attributes;
 use crate::converter::utility::content::normalized_tag_name;
 use std::borrow::Cow;
 
-use super::cell::{collect_table_cells, convert_table_cell, get_colspan_rowspan};
+use super::cell::{cell_text_content, collect_table_cells, convert_table_cell, get_colspan_rowspan};
 
 /// Maximum allowed table columns to prevent unbounded memory usage.
 const MAX_TABLE_COLS: usize = 1000;
@@ -87,6 +87,75 @@ pub fn append_layout_row(
     }
 }
 
+/// Collect the rendered text content of every cell in a row for width calculation.
+///
+/// `rowspan_tracker` mirrors the tracker in `convert_table_row` so that spanned
+/// columns are skipped in the width pre-pass just as they are skipped in rendering.
+/// Pass a shared tracker across all row calls to correctly handle multi-row spans.
+#[allow(clippy::trivially_copy_pass_by_ref)]
+pub fn collect_row_cell_widths(
+    node_handle: &tl::NodeHandle,
+    parser: &tl::Parser,
+    options: &crate::options::ConversionOptions,
+    ctx: &super::super::super::Context,
+    dom_ctx: &super::super::super::DomContext,
+    col_widths: &mut Vec<usize>,
+    rowspan_tracker: &mut Vec<Option<usize>>,
+) {
+    let mut cells = Vec::new();
+    collect_table_cells(node_handle, parser, dom_ctx, &mut cells);
+
+    let mut col = 0usize;
+    let mut cell_iter = cells.iter();
+
+    loop {
+        // Skip columns that are filled by a rowspan from a previous row.
+        while col < rowspan_tracker.len() {
+            if let Some(Some(remaining)) = rowspan_tracker.get_mut(col) {
+                if *remaining > 0 {
+                    *remaining -= 1;
+                    if *remaining == 0 {
+                        rowspan_tracker[col] = None;
+                    }
+                    col += 1;
+                    continue;
+                }
+            }
+            break;
+        }
+
+        let Some(cell_handle) = cell_iter.next() else {
+            break;
+        };
+
+        let text = cell_text_content(cell_handle, parser, options, ctx, dom_ctx);
+        let width = text.chars().count();
+
+        // Grow the widths vec if needed.
+        if col >= col_widths.len() {
+            col_widths.resize(col + 1, 0);
+        }
+        if width > col_widths[col] {
+            col_widths[col] = width;
+        }
+
+        let (colspan, rowspan) = get_colspan_rowspan(cell_handle, parser);
+
+        // Record rowspan for future rows.
+        if rowspan > 1 {
+            if col >= rowspan_tracker.len() {
+                rowspan_tracker.resize(col + 1, None);
+            }
+            rowspan_tracker[col] = Some(rowspan - 1);
+        }
+
+        col = col.saturating_add(colspan);
+    }
+}
+
+/// Minimum separator dash count per column (matches `---`).
+const MIN_SEPARATOR_DASHES: usize = 3;
+
 /// Convert a table row (tr) to Markdown format.
 ///
 /// Processes all cells in a row, handling colspan and rowspan for proper
@@ -107,6 +176,7 @@ pub fn append_layout_row(
 /// * `dom_ctx` - DOM context
 /// * `depth` - Nesting depth
 /// * `is_header` - Whether this is a header row
+/// * `col_widths` - Per-column max content widths for padding (empty = no padding)
 #[allow(clippy::too_many_arguments)]
 #[cfg_attr(not(feature = "visitor"), allow(unused_variables))]
 #[allow(clippy::trivially_copy_pass_by_ref)]
@@ -124,6 +194,7 @@ pub fn convert_table_row(
     dom_ctx: &super::super::super::DomContext,
     depth: usize,
     is_header: bool,
+    col_widths: &[usize],
 ) {
     let mut row_text = String::with_capacity(256);
     let mut cells = Vec::new();
@@ -203,7 +274,13 @@ pub fn convert_table_row(
             if col_index < total_cols {
                 if let Some(Some(remaining_rows)) = rowspan_tracker.get_mut(col_index) {
                     if *remaining_rows > 0 {
+                        let width = col_widths.get(col_index).copied();
                         row_text.push(' ');
+                        if let Some(w) = width {
+                            for _ in 0..w {
+                                row_text.push(' ');
+                            }
+                        }
                         row_text.push_str(" |");
                         *remaining_rows -= 1;
                         if *remaining_rows == 0 {
@@ -216,7 +293,8 @@ pub fn convert_table_row(
             }
 
             if let Some(cell_handle) = cell_iter.next() {
-                convert_table_cell(cell_handle, parser, &mut row_text, options, ctx, "", dom_ctx);
+                let col_width = col_widths.get(col_index).copied();
+                convert_table_cell(cell_handle, parser, &mut row_text, options, ctx, "", dom_ctx, col_width);
 
                 let (colspan, rowspan) = get_colspan_rowspan(cell_handle, parser);
 
@@ -230,8 +308,9 @@ pub fn convert_table_row(
             }
         }
     } else {
-        for cell_handle in &cells {
-            convert_table_cell(cell_handle, parser, &mut row_text, options, ctx, "", dom_ctx);
+        for (cell_idx, cell_handle) in cells.iter().enumerate() {
+            let col_width = col_widths.get(cell_idx).copied();
+            convert_table_cell(cell_handle, parser, &mut row_text, options, ctx, "", dom_ctx, col_width);
         }
     }
 
@@ -247,7 +326,10 @@ pub fn convert_table_row(
             if i > 0 {
                 output.push_str(" | ");
             }
-            output.push_str("---");
+            let dash_count = col_widths.get(i).copied().unwrap_or(0).max(MIN_SEPARATOR_DASHES);
+            for _ in 0..dash_count {
+                output.push('-');
+            }
         }
         output.push_str(" |\n");
     }

data/vendor/html-to-markdown-rs/src/converter/block/table/mod.rs

@@ -117,11 +117,11 @@ pub fn handle_table_with_context(
         let indented = layout::indent_table_for_list(&table_output, ctx.list_depth, options);
         output.push_str(&indented);
     } else {
-        if !output.ends_with("\n\n") {
-            if output.is_empty() || !output.ends_with('\n') {
-                output.push_str("\n\n");
-            } else {
+        if !output.is_empty() && !output.ends_with("\n\n") {
+            if output.ends_with('\n') {
                 output.push('\n');
+            } else {
+                output.push_str("\n\n");
             }
         }
         output.push_str(&table_output);

data/vendor/html-to-markdown-rs/src/converter/handlers/image.rs

@@ -253,7 +253,25 @@ fn format_image_markdown(
     buf.push_str("![");
     buf.push_str(alt);
     buf.push_str("](");
-    buf.push_str(src);
+
+    if src.is_empty() {
+        buf.push_str("<>");
+    } else if src.contains(' ') || src.contains('\n') {
+        buf.push('<');
+        buf.push_str(src);
+        buf.push('>');
+    } else {
+        let open_count = src.chars().filter(|&c| c == '(').count();
+        let close_count = src.chars().filter(|&c| c == ')').count();
+
+        if open_count == close_count {
+            buf.push_str(src);
+        } else {
+            let escaped_src = src.replace('(', "\\(").replace(')', "\\)");
+            buf.push_str(&escaped_src);
+        }
+    }
+
     if let Some(title_text) = title {
         buf.push_str(" \"");
         buf.push_str(title_text);

data/vendor/html-to-markdown-rs/src/converter/main.rs

@@ -23,7 +23,8 @@ use crate::converter::preprocessing_helpers::{has_inline_block_misnest, should_d
 use crate::converter::utility::caching::build_dom_context;
 use crate::converter::utility::content::normalized_tag_name;
 use crate::converter::utility::preprocessing::{
-    normalize_bogus_comment_endings, preprocess_html, strip_hidden_elements, strip_script_and_style_tags,
+    normalize_bogus_comment_endings, normalize_split_closing_tags, preprocess_html, strip_hidden_elements,
+    strip_script_and_style_tags,
 };
 use crate::converter::utility::serialization::serialize_tag_to_html;
 use crate::options::OutputFormat;
@@ -66,6 +67,10 @@ pub fn convert_html_impl(
     // Normalise bogus HTML comment endings (`--->`, `---->`, …) that cause the
     // `tl` parser to silently discard all document content that follows them.
     let stripped = normalize_bogus_comment_endings(&stripped);
+    // Normalise closing tags whose `>` is on a subsequent line (JSX-style `</a\n>`).
+    // The `tl` parser does not handle such end-tags and leaves the element unclosed,
+    // causing all subsequent siblings to be absorbed as children.
+    let stripped = normalize_split_closing_tags(&stripped);
     let mut preprocessed = preprocess_html(&stripped).into_owned();
     let mut preprocessed_len = preprocessed.len();
 
@@ -74,6 +79,7 @@ pub fn convert_html_impl(
             let stripped = strip_script_and_style_tags(&repaired_html);
             let stripped = strip_hidden_elements(&stripped);
             let stripped = normalize_bogus_comment_endings(&stripped);
+            let stripped = normalize_split_closing_tags(&stripped);
             let repaired = preprocess_html(&stripped).into_owned();
             preprocessed = repaired;
             preprocessed_len = preprocessed.len();
@@ -88,6 +94,7 @@ pub fn convert_html_impl(
             let stripped = strip_script_and_style_tags(&repaired_html);
             let stripped = strip_hidden_elements(&stripped);
             let stripped = normalize_bogus_comment_endings(&stripped);
+            let stripped = normalize_split_closing_tags(&stripped);
             preprocessed = preprocess_html(&stripped).into_owned();
             preprocessed_len = preprocessed.len();
             continue;
@@ -109,6 +116,7 @@ pub fn convert_html_impl(
             let stripped = strip_script_and_style_tags(&repaired_html);
             let stripped = strip_hidden_elements(&stripped);
             let stripped = normalize_bogus_comment_endings(&stripped);
+            let stripped = normalize_split_closing_tags(&stripped);
             preprocessed = preprocess_html(&stripped).into_owned();
             preprocessed_len = preprocessed.len();
             // Re-parse with repaired HTML