html-to-markdown 3.0.2 → 3.2.0

data/html-to-markdown-rb.gemspec

@@ -87,7 +87,7 @@ Gem::Specification.new do |spec|
   spec.files = files
   spec.extra_rdoc_files = ['README.md']
 
-  spec.extensions = ['ext/html-to-markdown-rb/extconf.rb']
+  spec.extensions = ['ext/html_to_markdown_rb/extconf.rb']
 
   spec.add_dependency 'rb_sys', '>= 0.9', '< 1.0'
   spec.metadata['rubygems_mfa_required'] = 'true'

data/lib/html_to_markdown/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HtmlToMarkdown
-  VERSION = '3.0.2'
+  VERSION = '3.2.0'
 end

data/lib/html_to_markdown.rb

@@ -3,28 +3,38 @@
 require_relative 'html_to_markdown/version'
 require 'html_to_markdown_rb'
 
+# High-performance HTML to Markdown conversion.
+#
+# @example Simple conversion
+#   HtmlToMarkdown.convert('<h1>Hello</h1>') # => "# Hello\n\n"
+#
+# @example With options
+#   HtmlToMarkdown.convert('<h1>Hello</h1>', heading_style: 'atx')
 module HtmlToMarkdown
-  autoload :CLI, 'html_to_markdown/cli'
-  autoload :CLIProxy, 'html_to_markdown/cli_proxy'
-
-  class << self
-    alias native_convert convert
-  end
-
-  module_function
-
-  # Convert HTML to Markdown, returning a Hash with:
-  #   - :content [String, nil] the converted Markdown output
-  #   - :document [nil] document structure (not yet exposed)
-  #   - :metadata [Hash, nil] extracted HTML metadata
-  #   - :tables [Array<Hash>] extracted tables with :grid and :markdown
-  #   - :images [Array<Hash>] extracted inline images
-  #   - :warnings [Array<Hash>] processing warnings
+  # Convert HTML to Markdown.
   #
-  # @param html [String] HTML string to convert
-  # @param options [Hash, nil] optional conversion options
-  # @return [Hash] conversion result
-  def convert(html, options = nil)
-    native_convert(html.to_s, options)
+  # @param html [String] The HTML content to convert.
+  # @param options [Hash] Optional conversion options.
+  #   Supported keys (all optional):
+  #   - :heading_style       - 'atx', 'atx_closed', 'setext', 'underlined'
+  #   - :code_block_style    - 'backticks', 'tildes', 'indented'
+  #   - :escape_asterisks    - Boolean
+  #   - :escape_underscores  - Boolean
+  #   - :escape_misc         - Boolean
+  #   - :escape_ascii        - Boolean
+  #   - :strip_newlines      - Boolean
+  #   - :keep_inline_images_in - Array of tag names
+  #   - :strip_tags          - Array of tag names to strip
+  #   - :preserve_tags       - Array of tag names to preserve verbatim
+  #   (and more, matching ConversionOptions fields)
+  # @return [String] The converted Markdown content.
+  def self.convert(html, options = {})
+    opts = if options.nil? || options.empty?
+             nil
+           else
+             HtmlToMarkdownRs::ConversionOptions.new(options)
+           end
+    result = HtmlToMarkdownRs.convert(html, opts)
+    result.content || ''
   end
 end

data/{ext/html-to-markdown-rb/native/extconf.rb → lib/html_to_markdown_rs.rb}

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-require_relative '../extconf'
+require 'html_to_markdown_rb'

data/sig/html_to_markdown.rbs

@@ -1,3 +1,16 @@
+# Native extension module (Magnus/rb-sys)
+module HtmlToMarkdownRs
+  class ConversionOptions
+    def initialize: (Hash[Symbol, untyped]) -> void
+  end
+
+  class ConversionResult
+    def content: () -> String?
+  end
+
+  def self.convert: (String html, ConversionOptions? options) -> ConversionResult
+end
+
 # Type definitions for HtmlToMarkdown Ruby gem
 module HtmlToMarkdown
   VERSION: String
@@ -8,6 +21,7 @@ module HtmlToMarkdown
   type whitespace_mode = :normalized | :strict
   type newline_style = :spaces | :backslash
   type code_block_style = :indented | :backticks | :tildes
+  type link_style = :inline | :reference
   type output_format = :markdown | :djot
   type preprocessing_preset = :minimal | :standard | :aggressive
 
@@ -49,6 +63,7 @@ module HtmlToMarkdown
     debug?: bool,
     strip_tags?: Array[String],
     preserve_tags?: Array[String],
+    link_style?: link_style,
     output_format?: output_format,
     skip_images?: bool,
     include_document_structure?: bool,
@@ -126,12 +141,9 @@ module HtmlToMarkdown
 
   public
 
-  # Convert HTML to Markdown, returning a Hash with content, metadata, tables, images, and warnings.
+  # Convert HTML to Markdown, returning the markdown content string.
   #
   # Example:
   #   result = HtmlToMarkdown.convert(html)
-  def self.convert: (String html, ?conversion_options options) -> Hash[String, untyped]
-
-  # Instance method version (created by module_function)
-  def convert: (String html, ?conversion_options options) -> Hash[String, untyped]
+  def self.convert: (String html, ?conversion_options options) -> String
 end

data/vendor/Cargo.toml

@@ -3,7 +3,7 @@ members = ["html-to-markdown-rs"]
 resolver = "2"
 
 [workspace.package]
-version = "3.0.2"
+version = "3.2.0"
 edition = "2024"
 rust-version = "1.85"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
@@ -18,11 +18,11 @@ clap = { version = "4.6", features = ["derive"] }
 clap_complete = "4.6"
 clap_mangen = "0.3"
 encoding_rs = "0.8"
-ext-php-rs = "0.15.8"
+ext-php-rs = "0.15.10"
 html5ever = "0.39.0"
 once_cell = "1.21"
-pyo3 = { version = "0.28.2", features = ["abi3-py310"] }
-rayon = "1.11"
+pyo3 = { version = "0.28.3", features = ["abi3-py310"] }
+rayon = "1.12"
 regex = "1.12"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"

data/vendor/html-to-markdown-rs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "html-to-markdown-rs"
-version = "3.0.2"
+version = "3.2.0"
 edition = "2024"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
 license = "MIT"
@@ -39,7 +39,7 @@ image = { version = "0.25", default-features = false, features = [
     "bmp",
     "webp",
 ], optional = true }
-lru = "0.16"
+lru = "0.17"
 memchr = "2"
 once_cell = "1.21"
 regex = "1.12"

data/vendor/html-to-markdown-rs/examples/test_deser.rs

@@ -0,0 +1,12 @@
+#![allow(missing_docs)]
+use html_to_markdown_rs::ConversionOptions;
+
+fn main() {
+    let json = r#"{"headingStyle":"","listIndentType":"","listIndentWidth":2,"bullets":"-*+","strongEmSymbol":"*","escapeAsterisks":false,"escapeUnderscores":false,"escapeMisc":false,"escapeAscii":false,"codeLanguage":"","autolinks":true,"defaultTitle":false,"brInTables":false,"highlightStyle":"","extractMetadata":true,"whitespaceMode":"","stripNewlines":false,"wrap":false,"wrapWidth":80,"convertAsInline":false,"subSymbol":"","supSymbol":"","newlineStyle":"spaces","codeBlockStyle":"tildes","keepInlineImagesIn":null,"preprocessing":{"enabled":false,"preset":"","removeNavigation":false,"removeForms":false},"encoding":"utf-8","debug":false,"stripTags":null,"preserveTags":null,"skipImages":false,"linkStyle":"","outputFormat":"","includeDocumentStructure":false,"extractImages":false,"maxImageSize":5242880,"captureSvg":false,"inferDimensions":true}"#;
+
+    let opts: ConversionOptions = serde_json::from_str(json).unwrap();
+    println!("code_block_style: {:?}", opts.code_block_style);
+
+    let result = html_to_markdown_rs::convert("<pre><code>some code</code></pre>", Some(opts)).unwrap();
+    println!("result: {:?}", result.content);
+}

data/vendor/html-to-markdown-rs/src/converter/block/mod.rs

@@ -40,7 +40,7 @@ pub use super::{Context, DomContext};
 /// element was handled, `false` otherwise.
 ///
 /// # Usage in converter.rs
-/// ```ignore
+/// ```text
 /// if crate::converter::block::dispatch_block_handler(
 ///     &tag_name,
 ///     node_handle,

data/vendor/html-to-markdown-rs/src/converter/block/table/mod.rs

@@ -29,7 +29,7 @@ pub(crate) use caption::handle_caption;
 /// Dispatches table element handling to the main convert_table function.
 ///
 /// # Usage in converter.rs
-/// ```ignore
+/// ```text
 /// if "table" == tag_name {
 ///     crate::converter::block::table::handle_table(
 ///         node_handle,

data/vendor/html-to-markdown-rs/src/converter/context.rs

@@ -12,6 +12,7 @@ use std::rc::Rc;
 #[cfg(feature = "inline-images")]
 use crate::inline_images::InlineImageCollector;
 
+use crate::converter::reference_collector::ReferenceCollectorHandle;
 use crate::types::structure_collector::StructureCollectorHandle;
 
 /// Handle type for inline image collector when feature is enabled.
@@ -105,6 +106,8 @@ pub struct Context {
     ///
     /// Populated when `options.include_document_structure == true`.
     pub(crate) structure_collector: Option<StructureCollectorHandle>,
+    /// Optional reference collector for reference-style links.
+    pub(crate) reference_collector: Option<ReferenceCollectorHandle>,
 }
 
 impl Context {
@@ -122,6 +125,7 @@ impl Context {
         #[cfg(feature = "visitor")] visitor: Option<crate::visitor::VisitorHandle>,
         #[cfg(not(feature = "visitor"))] _visitor: Option<()>,
         structure_collector: Option<StructureCollectorHandle>,
+        reference_collector: Option<ReferenceCollectorHandle>,
     ) -> Self {
         #[cfg(feature = "metadata")]
         let (
@@ -186,6 +190,7 @@ impl Context {
             #[cfg(feature = "visitor")]
             visitor_error: Rc::new(RefCell::new(None)),
             structure_collector,
+            reference_collector,
         }
     }
 }

data/vendor/html-to-markdown-rs/src/converter/form/mod.rs

@@ -53,7 +53,7 @@ pub use elements::handle as handle_form_elements;
 ///
 /// # Example
 ///
-/// ```ignore
+/// ```text
 /// if dispatch_form_handler(tag_name, &node_handle, &parser, output, options, ctx, depth, dom_ctx) {
 ///     // Tag was handled
 /// } else {

data/vendor/html-to-markdown-rs/src/converter/handlers/graphic.rs

@@ -128,6 +128,8 @@ pub fn handle_graphic(
                 &alt,
                 title.as_deref(),
                 should_use_alt_text,
+                options.link_style,
+                ctx.reference_collector.as_ref(),
             )),
             VisitResult::Custom(custom) => Some(custom),
             VisitResult::Skip => None,
@@ -145,6 +147,8 @@ pub fn handle_graphic(
             &alt,
             title.as_deref(),
             should_use_alt_text,
+            options.link_style,
+            ctx.reference_collector.as_ref(),
         ))
     };
 
@@ -154,6 +158,8 @@ pub fn handle_graphic(
         &alt,
         title.as_deref(),
         should_use_alt_text,
+        options.link_style,
+        ctx.reference_collector.as_ref(),
     ));
 
     if !options.skip_images {
@@ -189,21 +195,39 @@ pub fn handle_graphic(
 ///
 /// If `use_alt_only` is true, returns just the alt text.
 /// Otherwise returns the full `![alt](src "title")` syntax.
-fn format_graphic_markdown(src: &str, alt: &str, title: Option<&str>, use_alt_only: bool) -> String {
+fn format_graphic_markdown(
+    src: &str,
+    alt: &str,
+    title: Option<&str>,
+    use_alt_only: bool,
+    link_style: crate::options::validation::LinkStyle,
+    reference_collector: Option<&crate::converter::reference_collector::ReferenceCollectorHandle>,
+) -> String {
     if use_alt_only {
-        alt.to_string()
-    } else {
-        let mut buf = String::with_capacity(src.len() + alt.len() + 10);
-        buf.push_str("![");
-        buf.push_str(alt);
-        buf.push_str("](");
-        buf.push_str(src);
-        if let Some(title_text) = title {
-            buf.push_str(" \"");
-            buf.push_str(title_text);
-            buf.push('"');
+        return alt.to_string();
+    }
+    if link_style == crate::options::validation::LinkStyle::Reference {
+        if let Some(collector) = reference_collector {
+            let ref_num = collector.borrow_mut().get_or_insert(src, title);
+            let mut buf = String::with_capacity(alt.len() + 10);
+            buf.push_str("![");
+            buf.push_str(alt);
+            buf.push_str("][");
+            buf.push_str(&ref_num.to_string());
+            buf.push(']');
+            return buf;
         }
-        buf.push(')');
-        buf
     }
+    let mut buf = String::with_capacity(src.len() + alt.len() + 10);
+    buf.push_str("![");
+    buf.push_str(alt);
+    buf.push_str("](");
+    buf.push_str(src);
+    if let Some(title_text) = title {
+        buf.push_str(" \"");
+        buf.push_str(title_text);
+        buf.push('"');
+    }
+    buf.push(')');
+    buf
 }

data/vendor/html-to-markdown-rs/src/converter/handlers/image.rs

@@ -146,7 +146,14 @@ pub fn handle_img(
             visitor.visit_image(&node_ctx, &src, &alt, title.as_deref())
         };
         match visit_result {
-            VisitResult::Continue => Some(format_image_markdown(&src, &alt, title.as_deref(), should_use_alt_text)),
+            VisitResult::Continue => Some(format_image_markdown(
+                &src,
+                &alt,
+                title.as_deref(),
+                should_use_alt_text,
+                options.link_style,
+                ctx.reference_collector.as_ref(),
+            )),
             VisitResult::Custom(custom) => Some(custom),
             VisitResult::Skip => None,
             VisitResult::Error(err) => {
@@ -158,11 +165,25 @@ pub fn handle_img(
             VisitResult::PreserveHtml => Some(serialize_node(node_handle, parser)),
         }
     } else {
-        Some(format_image_markdown(&src, &alt, title.as_deref(), should_use_alt_text))
+        Some(format_image_markdown(
+            &src,
+            &alt,
+            title.as_deref(),
+            should_use_alt_text,
+            options.link_style,
+            ctx.reference_collector.as_ref(),
+        ))
     };
 
     #[cfg(not(feature = "visitor"))]
-    let image_output = Some(format_image_markdown(&src, &alt, title.as_deref(), should_use_alt_text));
+    let image_output = Some(format_image_markdown(
+        &src,
+        &alt,
+        title.as_deref(),
+        should_use_alt_text,
+        options.link_style,
+        ctx.reference_collector.as_ref(),
+    ));
 
     // Only output image if skip_images is not enabled
     if !options.skip_images {
@@ -204,21 +225,39 @@ pub fn handle_img(
 ///
 /// If `use_alt_only` is true, returns just the alt text.
 /// Otherwise returns the full `![alt](src "title")` syntax.
-fn format_image_markdown(src: &str, alt: &str, title: Option<&str>, use_alt_only: bool) -> String {
+fn format_image_markdown(
+    src: &str,
+    alt: &str,
+    title: Option<&str>,
+    use_alt_only: bool,
+    link_style: crate::options::validation::LinkStyle,
+    reference_collector: Option<&crate::converter::reference_collector::ReferenceCollectorHandle>,
+) -> String {
     if use_alt_only {
-        alt.to_string()
-    } else {
-        let mut buf = String::with_capacity(src.len() + alt.len() + 10);
-        buf.push_str("![");
-        buf.push_str(alt);
-        buf.push_str("](");
-        buf.push_str(src);
-        if let Some(title_text) = title {
-            buf.push_str(" \"");
-            buf.push_str(title_text);
-            buf.push('"');
+        return alt.to_string();
+    }
+    if link_style == crate::options::validation::LinkStyle::Reference {
+        if let Some(collector) = reference_collector {
+            let ref_num = collector.borrow_mut().get_or_insert(src, title);
+            let mut buf = String::with_capacity(alt.len() + 10);
+            buf.push_str("![");
+            buf.push_str(alt);
+            buf.push_str("][");
+            buf.push_str(&ref_num.to_string());
+            buf.push(']');
+            return buf;
         }
-        buf.push(')');
-        buf
     }
+    let mut buf = String::with_capacity(src.len() + alt.len() + 10);
+    buf.push_str("![");
+    buf.push_str(alt);
+    buf.push_str("](");
+    buf.push_str(src);
+    if let Some(title_text) = title {
+        buf.push_str(" \"");
+        buf.push_str(title_text);
+        buf.push('"');
+    }
+    buf.push(')');
+    buf
 }

data/vendor/html-to-markdown-rs/src/converter/handlers/link.rs

@@ -115,6 +115,7 @@ pub fn handle_link(
                             title.as_deref(),
                             raw_text.as_str(),
                             options,
+                            ctx.reference_collector.as_ref(),
                         );
                         push_heading(output, ctx, options, heading_level, link_buffer.as_str());
                         return;
@@ -190,6 +191,13 @@ pub fn handle_link(
             label = href.clone();
         }
 
+        // Normalize Wikipedia-style back-reference links: <a href="#cite_ref-N">^</a>
+        // These produce `[^](#cite_ref-N)` which is confusing (looks like a footnote).
+        // Convert to `[↑](#cite_ref-N)` to avoid ambiguity with markdown footnote syntax.
+        if label == "^" && href.starts_with('#') {
+            label = "↑".to_string();
+        }
+
         let escaped_label = escape_link_label(&label);
 
         #[cfg(feature = "visitor")]
@@ -226,6 +234,7 @@ pub fn handle_link(
                         title.as_deref(),
                         label.as_str(),
                         options,
+                        ctx.reference_collector.as_ref(),
                     );
                     Some(buf)
                 }
@@ -248,6 +257,7 @@ pub fn handle_link(
                 title.as_deref(),
                 label.as_str(),
                 options,
+                ctx.reference_collector.as_ref(),
             );
             Some(buf)
         };
@@ -262,6 +272,7 @@ pub fn handle_link(
                 title.as_deref(),
                 label.as_str(),
                 options,
+                ctx.reference_collector.as_ref(),
             );
             Some(buf)
         };

data/vendor/html-to-markdown-rs/src/converter/inline/link.rs

@@ -145,6 +145,7 @@ pub(crate) fn handle(
                             title.as_deref(),
                             raw_text.as_str(),
                             options,
+                            ctx.reference_collector.as_ref(),
                         );
                         push_heading(output, ctx, options, heading_level, link_buffer.as_str());
                         return;
@@ -262,6 +263,7 @@ pub(crate) fn handle(
                         title.as_deref(),
                         label.as_str(),
                         options,
+                        ctx.reference_collector.as_ref(),
                     );
                     Some(buf)
                 }
@@ -284,6 +286,7 @@ pub(crate) fn handle(
                 title.as_deref(),
                 label.as_str(),
                 options,
+                ctx.reference_collector.as_ref(),
             );
             Some(buf)
         };
@@ -298,6 +301,7 @@ pub(crate) fn handle(
                 title.as_deref(),
                 label.as_str(),
                 options,
+                ctx.reference_collector.as_ref(),
             );
             Some(buf)
         };
@@ -363,7 +367,20 @@ pub(crate) fn append_markdown_link(
     title: Option<&str>,
     raw_text: &str,
     options: &ConversionOptions,
+    reference_collector: Option<&crate::converter::reference_collector::ReferenceCollectorHandle>,
 ) {
+    if options.link_style == crate::options::validation::LinkStyle::Reference && !href.is_empty() {
+        if let Some(collector) = reference_collector {
+            let ref_num = collector.borrow_mut().get_or_insert(href, title);
+            output.push('[');
+            output.push_str(label);
+            output.push_str("][");
+            output.push_str(&ref_num.to_string());
+            output.push(']');
+            return;
+        }
+    }
+
     output.push('[');
     output.push_str(label);
     output.push_str("](");

data/vendor/html-to-markdown-rs/src/converter/inline/mod.rs

@@ -64,7 +64,7 @@ pub use super::{Context, DomContext};
 ///
 /// # Usage in converter.rs
 ///
-/// ```ignore
+/// ```text
 /// if crate::converter::inline::dispatch_inline_handler(
 ///     &tag_name,
 ///     &node_handle,

data/vendor/html-to-markdown-rs/src/converter/list/item.rs

@@ -198,7 +198,11 @@ pub(crate) fn handle_li(
             } else {
                 let bullets: Vec<char> = options.bullets.chars().collect();
                 let bullet_index = if ctx.ul_depth > 0 { ctx.ul_depth - 1 } else { 0 };
-                let bullet = bullets.get(bullet_index % bullets.len()).copied().unwrap_or('*');
+                let bullet = if bullets.is_empty() {
+                    '*'
+                } else {
+                    bullets[bullet_index % bullets.len()]
+                };
                 output.push(bullet);
                 output.push(' ');
             }
@@ -265,7 +269,11 @@ pub(crate) fn handle_li(
             } else {
                 let bullets: Vec<char> = options.bullets.chars().collect();
                 let bullet_index = if ctx.ul_depth > 0 { ctx.ul_depth - 1 } else { 0 };
-                let bullet = bullets.get(bullet_index % bullets.len()).copied().unwrap_or('*');
+                let bullet = if bullets.is_empty() {
+                    '*'
+                } else {
+                    bullets[bullet_index % bullets.len()]
+                };
                 let bullet_str = bullet.to_string();
                 let text_start = last_line.find(bullet).map_or(0, |pos| pos + 1);
                 (bullet_str, last_line[text_start..].trim().to_string())

data/vendor/html-to-markdown-rs/src/converter/main.rs

@@ -196,6 +196,14 @@ pub(crate) fn convert_html_impl(
         }
     }
 
+    let reference_collector = if options.link_style == crate::options::LinkStyle::Reference {
+        Some(std::rc::Rc::new(std::cell::RefCell::new(
+            crate::converter::reference_collector::ReferenceCollector::new(),
+        )))
+    } else {
+        None
+    };
+
     #[cfg(all(feature = "metadata", feature = "visitor"))]
     let ctx = Context::new(
         options,
@@ -203,6 +211,7 @@ pub(crate) fn convert_html_impl(
         metadata_collector,
         visitor,
         structure_collector.as_ref().map(std::rc::Rc::clone),
+        reference_collector.as_ref().map(std::rc::Rc::clone),
     );
     #[cfg(all(feature = "metadata", not(feature = "visitor")))]
     let ctx = Context::new(
@@ -211,6 +220,7 @@ pub(crate) fn convert_html_impl(
         metadata_collector,
         _visitor,
         structure_collector.as_ref().map(std::rc::Rc::clone),
+        reference_collector.as_ref().map(std::rc::Rc::clone),
     );
     #[cfg(all(not(feature = "metadata"), feature = "visitor"))]
     let ctx = Context::new(
@@ -219,6 +229,7 @@ pub(crate) fn convert_html_impl(
         _metadata_collector,
         visitor,
         structure_collector.as_ref().map(std::rc::Rc::clone),
+        reference_collector.as_ref().map(std::rc::Rc::clone),
     );
     #[cfg(all(not(feature = "metadata"), not(feature = "visitor")))]
     let ctx = Context::new(
@@ -227,6 +238,7 @@ pub(crate) fn convert_html_impl(
         _metadata_collector,
         _visitor,
         structure_collector.as_ref().map(std::rc::Rc::clone),
+        reference_collector.as_ref().map(std::rc::Rc::clone),
     );
 
     for child_handle in dom.children() {
@@ -242,6 +254,19 @@ pub(crate) fn convert_html_impl(
     // reference to the same collector, and Rc::try_unwrap requires exactly one reference.
     drop(ctx);
 
+    // Append reference-style link definitions if any were collected
+    if let Some(rc) = reference_collector {
+        if let Ok(collector) = std::rc::Rc::try_unwrap(rc) {
+            let ref_section = collector.into_inner().finish();
+            if !ref_section.is_empty() {
+                let trimmed_len = output.trim_end_matches('\n').len();
+                output.truncate(trimmed_len);
+                output.push_str("\n\n");
+                output.push_str(&ref_section);
+            }
+        }
+    }
+
     // If plain text was requested, discard the markdown output and return plain text.
     // The full pipeline was still run above so that metadata + visitor callbacks fire.
     if is_plain_text {