RubyGems - html-to-markdown - Versions diffs - 2.29.0 → 3.0.0 - Mend

html-to-markdown 2.29.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (166) hide show

data/vendor/html-to-markdown-rs/Cargo.toml CHANGED Viewed

@@ -1,6 +1,6 @@
 [package]
 name = "html-to-markdown-rs"
-version = "2.29.0"
+version = "3.0.0"
 edition = "2024"
 authors = ["Na'aman Hirschfeld <naaman@kreuzberg.dev>"]
 license = "MIT"
@@ -14,25 +14,22 @@ keywords = ["html", "markdown", "converter", "astral-tl", "doc-processing"]
 categories = ["parsing", "text-processing", "web-programming"]
 [package.metadata.cargo-machete]
-ignored = ["once_cell", "futures"]
+ignored = ["once_cell", "ahash"]
 [lib]
 crate-type = ["rlib"]
 [features]
 default = ["metadata"]
-full = ["inline-images", "metadata", "visitor", "async-visitor", "serde"]
+full = ["inline-images", "metadata", "visitor", "serde"]
 inline-images = ["dep:image"]
 metadata = ["dep:serde", "dep:serde_json"]
 visitor = []
-async-visitor = ["visitor", "dep:async-trait", "dep:futures", "dep:tokio"]
 serde = ["dep:serde", "dep:serde_json"]
 [dependencies]
 ahash = { version = "0.8", features = ["std", "compile-time-rng"], default-features = false }
-async-trait = { version = "0.1", optional = true }
 base64 = "0.22"
-futures = { version = "0.3", optional = true }
 html-escape = "0.2.13"
 html5ever = "0.39.0"
 image = { version = "0.25", default-features = false, features = [
@@ -43,16 +40,13 @@ image = { version = "0.25", default-features = false, features = [
     "webp",
 ], optional = true }
 lru = "0.16"
+memchr = "2"
 once_cell = "1.21"
 regex = "1.12"
 serde = { version = "1.0", features = ["derive"], optional = true }
 serde_json = { version = "1.0", optional = true }
 thiserror = "2.0"
 tl = { package = "astral-tl", version = "0.7.11" }
-tokio = { version = "1.50", features = [
-    "rt-multi-thread",
-    "sync",
-], optional = true }
 [dev-dependencies]
 serde = { version = "1.0", features = ["derive"] }

data/vendor/html-to-markdown-rs/README.md CHANGED Viewed

@@ -18,42 +18,80 @@ Fast, reliable HTML to Markdown conversion with full CommonMark compliance. Buil
 ```toml
 [dependencies]
-html-to-markdown-rs = "2.3"
+html-to-markdown-rs = "3.0"
 ```
 ## Basic Usage
+`convert()` returns a structured `ConversionResult` with the converted text, metadata, tables, and more:
 ```rust
-use html_to_markdown_rs::{convert, ConversionOptions};
+use html_to_markdown_rs::convert;
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let html = r#"
-        <h1>Welcome</h1>
-        <p>This is <strong>fast</strong> conversion!</p>
-        <ul>
-            <li>Built with Rust</li>
-            <li>CommonMark compliant</li>
-        </ul>
+        <html lang="en">
+          <head><title>Welcome</title></head>
+          <body>
+            <h1>Welcome</h1>
+            <p>This is <strong>fast</strong> conversion!</p>
+            <ul>
+                <li>Built with Rust</li>
+                <li>CommonMark compliant</li>
+            </ul>
+          </body>
+        </html>
     "#;
-    let markdown = convert(html, None)?;
-    println!("{}", markdown);
+    let result = convert(html, None)?;
+    println!("{}", result.content.unwrap_or_default());
+    if let Some(metadata) = &result.metadata {
+        println!("Title: {:?}", metadata.document.title);
+        println!("Headers: {:?}", metadata.headers);
+    }
+    for table in &result.tables {
+        println!("Table with {} rows", table.cells.len());
+    }
     Ok(())
 }
 ```
 ## Error Handling
-Conversion returns a `Result<String, ConversionError>`. Inputs that look like binary data are rejected with
+Conversion returns a `Result<ConversionResult, ConversionError>`. Inputs that look like binary data are rejected with
 `ConversionError::InvalidInput` to prevent runaway allocations. Table `colspan`/`rowspan` values are also clamped
 internally to keep output sizes bounded.
 ## Configuration
+### Builder Pattern
+```rust
+use html_to_markdown_rs::{
+    convert, ConversionOptions, HeadingStyle, CodeBlockStyle,
+};
+let options = ConversionOptions::builder()
+    .heading_style(HeadingStyle::Atx)
+    .list_indent_width(2)
+    .bullets("-")
+    .autolinks(true)
+    .wrap(true)
+    .wrap_width(80)
+    .build();
+let result = convert(html, Some(options))?;
+println!("{}", result.content.unwrap_or_default());
+```
+### Struct Literal
 ```rust
 use html_to_markdown_rs::{
     convert, ConversionOptions, HeadingStyle, ListIndentType,
-    PreprocessingOptions, PreprocessingPreset,
 };
 let options = ConversionOptions {
@@ -69,12 +107,13 @@ let options = ConversionOptions {
     ..Default::default()
 };
-let markdown = convert(html, Some(options))?;
+let result = convert(html, Some(options))?;
+println!("{}", result.content.unwrap_or_default());
 ```
 ### Preserving HTML Tags
-The `preserve_tags` option allows you to keep specific HTML tags in their original form instead of converting them to Markdown. This is useful for complex elements like tables that may not convert well:
+The `preserve_tags` option allows you to keep specific HTML tags in their original form instead of converting them to Markdown:
 ```rust
 use html_to_markdown_rs::{convert, ConversionOptions};
@@ -93,18 +132,8 @@ let options = ConversionOptions {
     ..Default::default()
 };
-let markdown = convert(html, Some(options))?;
-// Result: "Before table\n\n<table class=\"data\">...</table>\n\nAfter table\n"
-```
-You can preserve multiple tag types and combine with `strip_tags`:
-```rust
-let options = ConversionOptions {
-    preserve_tags: vec!["table".to_string(), "form".to_string()],
-    strip_tags: vec!["script".to_string(), "style".to_string()],
-    ..Default::default()
-};
+let result = convert(html, Some(options))?;
+// result.content => "Before table\n\n<table class=\"data\">...</table>\n\nAfter table\n"
 ```
 ## Web Scraping with Preprocessing
@@ -118,44 +147,62 @@ options.preprocessing.preset = html_to_markdown_rs::PreprocessingPreset::Aggress
 options.preprocessing.remove_navigation = true;
 options.preprocessing.remove_forms = true;
-let markdown = convert(scraped_html, Some(options))?;
+let result = convert(scraped_html, Some(options))?;
+println!("{}", result.content.unwrap_or_default());
 ```
-## hOCR Table Extraction
+## Metadata Extraction
-```rust
-use html_to_markdown_rs::convert;
+Metadata is automatically included in the result. Configure which fields to extract via `MetadataConfig`:
-// hOCR documents (from Tesseract, etc.) are detected automatically.
-// Tables and spatial layout are reconstructed without additional options.
-let markdown = convert(hocr_html, None)?;
+```rust
+use html_to_markdown_rs::{convert, ConversionOptions, MetadataConfig};
+let options = ConversionOptions::builder()
+    .metadata_config(MetadataConfig {
+        extract_headers: true,
+        extract_links: true,
+        extract_images: false,
+        ..Default::default()
+    })
+    .build();
+let result = convert(html, Some(options))?;
+if let Some(metadata) = &result.metadata {
+    println!("Title: {:?}", metadata.document.title);
+    for header in &metadata.headers {
+        println!("H{}: {}", header.level, header.text);
+    }
+    for link in &metadata.links {
+        println!("Link: {} -> {}", link.text, link.href);
+    }
+}
 ```
-## Inline Image Extraction
+## Image Extraction
 ```rust
-use html_to_markdown_rs::{convert_with_inline_images, InlineImageConfig};
-let config = InlineImageConfig::new(5 * 1024 * 1024) // 5MB max
-    .with_infer_dimensions(true)
-    .with_filename_prefix("img_".to_string());
+use html_to_markdown_rs::{convert, ConversionOptions};
-let extraction = convert_with_inline_images(html, None, config)?;
+let options = ConversionOptions::builder()
+    .extract_images(true)
+    .max_image_size(5 * 1024 * 1024) // 5 MB max
+    .infer_dimensions(true)
+    .build();
-println!("{}", extraction.markdown);
-for (i, img) in extraction.inline_images.iter().enumerate() {
-    println!("Image {}: {} ({} bytes)", i, img.format, img.data.len());
+let result = convert(html, Some(options))?;
+println!("{}", result.content.unwrap_or_default());
+for img in &result.images {
+    println!("Image: {} ({} bytes)", img.src, img.data.as_ref().map_or(0, |d| d.len()));
 }
 ```
 ## Table Extraction
-Extract structured table data alongside the Markdown conversion. Each table found in the HTML is returned with its cell contents, header row flags, and rendered Markdown output.
-Requires the `visitor` feature.
+Structured table data is always included in `ConversionResult.tables`:
 ```rust
-use html_to_markdown_rs::convert_with_tables;
+use html_to_markdown_rs::convert;
 let html = r#"
 <table>
@@ -165,9 +212,9 @@ let html = r#"
 </table>
 "#;
-let result = convert_with_tables(html, None, None)?;
+let result = convert(html, None)?;
-println!("{}", result.content);
+println!("{}", result.content.unwrap_or_default());
 for table in &result.tables {
     println!("Table with {} rows:", table.cells.len());
     for (i, row) in table.cells.iter().enumerate() {
@@ -177,6 +224,34 @@ for table in &result.tables {
 }
 ```
+## Custom Visitors
+```rust
+use html_to_markdown_rs::{convert, ConversionOptions};
+use html_to_markdown_rs::visitor::{HtmlVisitor, NodeContext, VisitResult};
+struct NoImagesVisitor;
+impl HtmlVisitor for NoImagesVisitor {
+    fn visit_image(
+        &mut self,
+        _ctx: &NodeContext,
+        _src: &str,
+        _alt: &str,
+        _title: Option<&str>,
+    ) -> VisitResult {
+        VisitResult::Skip
+    }
+}
+let options = ConversionOptions::builder()
+    .visitor(Box::new(NoImagesVisitor))
+    .build();
+let result = convert(html, Some(options))?;
+println!("{}", result.content.unwrap_or_default());
+```
 ## Other Language Bindings
 This is the core Rust library. For other languages:
@@ -189,13 +264,14 @@ This is the core Rust library. For other languages:
 ## Documentation
-- [Full Documentation](https://github.com/kreuzberg-dev/html-to-markdown/blob/main/README.md)
+- [Full Documentation](https://docs.html-to-markdown.kreuzberg.dev)
 - [API Reference](https://docs.rs/html-to-markdown-rs)
+- [Migration Guide (v2 -> v3)](https://docs.html-to-markdown.kreuzberg.dev/migration/v3/)
 - [Contributing Guide](https://github.com/kreuzberg-dev/html-to-markdown/blob/main/CONTRIBUTING.md)
 ## Performance
-10-30x faster than pure Python/JavaScript implementations, delivering 150-210 MB/s throughput.
+10-30x faster than pure Python/JavaScript implementations, delivering 150-280 MB/s throughput.
 ## License

data/vendor/html-to-markdown-rs/examples/basic.rs CHANGED Viewed

@@ -1,6 +1,11 @@
 //! Example: Basic HTML to Markdown conversion
-use html_to_markdown_rs::convert;
+fn convert(
+    html: &str,
+    opts: Option<html_to_markdown_rs::ConversionOptions>,
+) -> html_to_markdown_rs::error::Result<String> {
+    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+}
 fn main() {
     let html = "<h1>Hello World</h1><p>This is a <strong>test</strong>.</p>";

data/vendor/html-to-markdown-rs/examples/table.rs CHANGED Viewed

@@ -1,6 +1,11 @@
 //! Example: Converting HTML tables to Markdown
-use html_to_markdown_rs::convert;
+fn convert(
+    html: &str,
+    opts: Option<html_to_markdown_rs::ConversionOptions>,
+) -> html_to_markdown_rs::error::Result<String> {
+    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+}
 fn main() {
     let html = r"<table>

data/vendor/html-to-markdown-rs/examples/test_escape.rs CHANGED Viewed

@@ -1,6 +1,11 @@
 //! Example: Testing HTML escape sequences and special characters
-use html_to_markdown_rs::convert;
+fn convert(
+    html: &str,
+    opts: Option<html_to_markdown_rs::ConversionOptions>,
+) -> html_to_markdown_rs::error::Result<String> {
+    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+}
 fn main() {
     let html = "<p>Use *wildcards* for search</p>";

data/vendor/html-to-markdown-rs/examples/test_inline_formatting.rs CHANGED Viewed

@@ -1,6 +1,12 @@
-//! Example: Testing inline formatting (bold, italic, code, etc.)
+#![allow(missing_docs)]
+fn convert(
+    html: &str,
+    opts: Option<html_to_markdown_rs::ConversionOptions>,
+) -> html_to_markdown_rs::error::Result<String> {
+    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+}
-use html_to_markdown_rs::{ConversionOptions, convert};
+use html_to_markdown_rs::ConversionOptions;
 fn main() {
     let html = "<p>This is <mark>highlighted</mark> text</p>";

data/vendor/html-to-markdown-rs/examples/test_lists.rs CHANGED Viewed

@@ -1,6 +1,11 @@
 //! Example: Testing HTML list conversion (ordered and unordered lists)
-use html_to_markdown_rs::convert;
+fn convert(
+    html: &str,
+    opts: Option<html_to_markdown_rs::ConversionOptions>,
+) -> html_to_markdown_rs::error::Result<String> {
+    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+}
 fn main() {
     let html = "<ol><li>First item</li><li>Second item</li><li>Third item</li></ol>";

data/vendor/html-to-markdown-rs/examples/test_semantic_tags.rs CHANGED Viewed

@@ -1,6 +1,11 @@
 //! Example: Testing HTML5 semantic tags (article, section, nav, etc.)
-use html_to_markdown_rs::convert;
+fn convert(
+    html: &str,
+    opts: Option<html_to_markdown_rs::ConversionOptions>,
+) -> html_to_markdown_rs::error::Result<String> {
+    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+}
 fn main() {
     let html = r"<article>

data/vendor/html-to-markdown-rs/examples/test_tables.rs CHANGED Viewed

@@ -1,6 +1,11 @@
 //! Example: Converting HTML tables to Markdown
-use html_to_markdown_rs::convert;
+fn convert(
+    html: &str,
+    opts: Option<html_to_markdown_rs::ConversionOptions>,
+) -> html_to_markdown_rs::error::Result<String> {
+    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+}
 fn main() {
     let html = r"<table>

data/vendor/html-to-markdown-rs/examples/test_task_lists.rs CHANGED Viewed

@@ -1,6 +1,11 @@
 //! Example: Testing task list conversion (checkboxes)
-use html_to_markdown_rs::convert;
+fn convert(
+    html: &str,
+    opts: Option<html_to_markdown_rs::ConversionOptions>,
+) -> html_to_markdown_rs::error::Result<String> {
+    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+}
 fn main() {
     let html = r#"<ul>

data/vendor/html-to-markdown-rs/examples/test_whitespace.rs CHANGED Viewed

@@ -1,6 +1,11 @@
 //! Example: Testing whitespace handling and normalization
-use html_to_markdown_rs::convert;
+fn convert(
+    html: &str,
+    opts: Option<html_to_markdown_rs::ConversionOptions>,
+) -> html_to_markdown_rs::error::Result<String> {
+    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
+}
 fn main() {
     let html = "<p>text    with    multiple    spaces</p>";