html-to-markdown 3.4.0 → 3.6.0.pre.rc.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1296a51b6ca0f757a95487c567e9d1e0d2acda80a46656cd14002322a4b850b
-  data.tar.gz: dca253ecef2f4818aa9e39e56d48ca5bb9ad35f92759580ee893d17eb455d494
+  metadata.gz: 99d668eec369d0e03a53d5ddf4192c1e896ad5940d669b598a73ed65b0329c82
+  data.tar.gz: 91e04cc771c7a7725a705258d4268eb16eda7541d7b591d480dfa53da1480015
 SHA512:
-  metadata.gz: 989e6b723c3670894b9e87ad87c1d752c27e3917f5cc60f0ba8f40175ee97c35ef4a2a40da53ab77908c6febb44abc0fe66064470494e73be3a5aae61d8498ae
-  data.tar.gz: 6275bc085c86f521114dbfd860e856dfdeac43ae92b755d81960e22e11a6d73ec4f25c89c837c37072d5500fd22af92f9507ace1673337708cac1a5376db926c
+  metadata.gz: 8b85249b540aab795dafc749a17b1bd00b1d58e4557cb4b524e833c0f1fc86c7b5a141956cf5a96a6fe8a14aaaea6aa44e9fd102d1e1337b2045dff829c4f302
+  data.tar.gz: d7c532c8de1ab8f7f5a0313f56c409d5f51ad4618552be841fe6a1e4b39b69a28d9c25788d423e52490130682a6c26f5d352b2db4d617fc7bb31ae2a9dc85c01

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright 2024-2025 Na'aman Hirschfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,347 @@
+# html-to-markdown
+
+<div align="center" style="display: flex; flex-wrap: wrap; gap: 8px; justify-content: center; margin: 20px 0;">
+  <a href="https://github.com/kreuzberg-dev/alef">
+    <img src="https://img.shields.io/badge/Bindings-alef%20%D7%90-007ec6" alt="Bindings">
+  </a>
+  <!-- Language Bindings -->
+  <a href="https://crates.io/crates/html-to-markdown-rs">
+    <img src="https://img.shields.io/crates/v/html-to-markdown-rs?label=Rust&color=007ec6" alt="Rust">
+  </a>
+  <a href="https://pypi.org/project/html-to-markdown/">
+    <img src="https://img.shields.io/pypi/v/html-to-markdown?label=Python&color=007ec6" alt="Python">
+  </a>
+  <a href="https://www.npmjs.com/package/@kreuzberg/html-to-markdown-node">
+    <img src="https://img.shields.io/npm/v/@kreuzberg/html-to-markdown-node?label=Node.js&color=007ec6" alt="Node.js">
+  </a>
+  <a href="https://www.npmjs.com/package/@kreuzberg/html-to-markdown-wasm">
+    <img src="https://img.shields.io/npm/v/@kreuzberg/html-to-markdown-wasm?label=WASM&color=007ec6" alt="WASM">
+  </a>
+  <a href="https://central.sonatype.com/artifact/dev.kreuzberg/html-to-markdown">
+    <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/html-to-markdown?label=Java&color=007ec6" alt="Java">
+  </a>
+  <a href="https://pkg.go.dev/github.com/kreuzberg-dev/html-to-markdown/packages/go/v3/htmltomarkdown">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/html-to-markdown?label=Go&color=007ec6&filter=v3*" alt="Go">
+  </a>
+  <a href="https://www.nuget.org/packages/KreuzbergDev.HtmlToMarkdown/">
+    <img src="https://img.shields.io/nuget/v/KreuzbergDev.HtmlToMarkdown?label=C%23&color=007ec6" alt="C#">
+  </a>
+  <a href="https://packagist.org/packages/kreuzberg-dev/html-to-markdown">
+    <img src="https://img.shields.io/packagist/v/kreuzberg-dev/html-to-markdown?label=PHP&color=007ec6" alt="PHP">
+  </a>
+  <a href="https://rubygems.org/gems/html-to-markdown">
+    <img src="https://img.shields.io/gem/v/html-to-markdown?label=Ruby&color=007ec6" alt="Ruby">
+  </a>
+  <a href="https://hex.pm/packages/html_to_markdown">
+    <img src="https://img.shields.io/hexpm/v/html_to_markdown?label=Elixir&color=007ec6" alt="Elixir">
+  </a>
+  <a href="https://kreuzberg-dev.r-universe.dev/htmltomarkdown">
+    <img src="https://img.shields.io/badge/R-htmltomarkdown-007ec6" alt="R">
+  </a>
+  <a href="https://pub.dev/packages/h2m">
+    <img src="https://img.shields.io/pub/v/h2m?label=Dart&color=007ec6" alt="Dart">
+  </a>
+  <a href="https://central.sonatype.com/artifact/dev.kreuzberg/html-to-markdown-android">
+    <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/html-to-markdown-android?label=Kotlin&color=007ec6" alt="Kotlin">
+  </a>
+  <a href="https://github.com/kreuzberg-dev/html-to-markdown/tree/main/packages/swift">
+    <img src="https://img.shields.io/badge/Swift-SPM-007ec6" alt="Swift">
+  </a>
+  <a href="https://github.com/kreuzberg-dev/html-to-markdown/tree/main/packages/zig">
+    <img src="https://img.shields.io/badge/Zig-package-007ec6" alt="Zig">
+  </a>
+  <a href="https://github.com/kreuzberg-dev/html-to-markdown/releases">
+    <img src="https://img.shields.io/badge/C-FFI-007ec6" alt="C FFI">
+  </a>
+
+  <!-- Project Info -->
+  <a href="https://github.com/kreuzberg-dev/html-to-markdown/blob/main/LICENSE">
+    <img src="https://img.shields.io/badge/License-MIT-007ec6" alt="License">
+  </a>
+  <a href="https://docs.html-to-markdown.kreuzberg.dev">
+    <img src="https://img.shields.io/badge/Docs-html--to--markdown-007ec6" alt="Documentation">
+  </a>
+</div>
+
+<div align="center" style="margin: 24px 0 0;">
+  <a href="https://kreuzberg.dev">
+    <img alt="html-to-markdown" src="https://github.com/user-attachments/assets/478a83da-237b-446b-b3a8-e564c13e00a8" />
+  </a>
+</div>
+
+<div align="center" style="display: flex; flex-wrap: wrap; gap: 12px; justify-content: center; margin: 28px 0 24px;">
+  <a href="https://discord.gg/xt9WY3GnKR">
+    <img height="22" src="https://img.shields.io/badge/Discord-Chat-007ec6?logo=discord&logoColor=white" alt="Join Discord">
+  </a>
+  <a href="https://docs.html-to-markdown.kreuzberg.dev/demo/">
+    <img height="22" src="https://img.shields.io/badge/Live%20Demo-Open-007ec6?logo=webassembly&logoColor=white" alt="Live Demo">
+  </a>
+</div>
+
+Blazing-fast HTML to Markdown conversion for Ruby, powered by the same Rust engine used by our Python, Node.js, WebAssembly, and PHP packages.
+Ship identical Markdown across every runtime while enjoying native extension performance with Magnus bindings.
+
+## What This Package Provides
+
+- **Same renderer as every binding** — output matches Rust, Python, Node.js, Ruby, PHP, Go, Java, .NET, Elixir, R, Dart, Swift, Zig, C FFI, and WASM.
+- **Structured conversion result** — Markdown plus metadata, links, headings, images, tables, and warnings where the binding exposes them.
+- **Production defaults** — HTML is parsed with the Rust core, sanitized by default, and rendered without runtime-specific Markdown drift.
+- **Ruby extension** — Magnus-backed native extension with idiomatic Ruby objects and no separate service process.
+
+## Installation
+
+```bash
+gem install html-to-markdown
+```
+
+Requires Ruby 3.2+ with Magnus native extension bindings. Published for Linux, macOS.
+
+## Performance Snapshot
+
+**Apple M4** · `convert()` · Real Wikipedia documents
+
+| Document | Size | Latency | Throughput |
+|----------|------|---------|------------|
+| Lists (Timeline) | 129KB | 0.71ms | 182 MB/s |
+| Tables (Countries) | 360KB | 2.15ms | 167 MB/s |
+| Mixed (Python wiki) | 656KB | 4.89ms | 134 MB/s |
+
+## Quick Start
+
+Basic conversion:
+
+```ruby
+require 'html_to_markdown'
+
+html = "<h1>Hello</h1><p>This is <strong>fast</strong>!</p>"
+result = HtmlToMarkdown.convert(html)
+markdown = result[:content]
+```
+
+With conversion options:
+
+```ruby
+require 'html_to_markdown'
+
+html = "<h1>Hello</h1><p>This is <strong>fast</strong>!</p>"
+result = HtmlToMarkdown.convert(html, heading_style: :atx, code_block_style: :fenced)
+markdown = result[:content]
+```
+
+## Architecture
+
+The converter routes each input through one of three tiers based on a fast prescan of the byte stream:
+
+1. **Tier-1 — single-pass byte scanner.** Handles 110+ HTML tags directly. Bails on any construct it cannot prove byte-equivalent to Tier-2.
+2. **Tier-2 — DOM walker.** Picks up Tier-1 bails and inputs the classifier rejected up front.
+3. **Tier-3 — standards-conformant parser.** Engaged for malformed HTML requiring full HTML5 repair.
+
+The dispatcher is invisible to the caller. Output is byte-identical across tiers — enforced by a 116-snapshot oracle.
+
+## Capabilities
+
+- **16 languages, one Rust core.** Rust, Python, Node.js, WASM, Java, Go, C#, PHP, Ruby, Elixir, R, Dart, Kotlin (Android), Swift, Zig, C ABI.
+- **CommonMark-compatible Markdown** with GFM-style tables.
+- **Djot output**: set `output_format = "djot"` (see Djot Output Format section below).
+- **Real-HTML robust**: unclosed tags, CDATA, custom elements, malformed entities, nested tables, mixed encodings handled without losing content.
+- **Metadata extraction**, **visitor API**, **inline images**, **configurable preprocessing presets**.
+- **Per-group regression gates in CI**: every PR runs the bench harness against per-group thresholds.
+
+## API Reference
+
+### Core Function
+
+**`convert(html, options: nil, visitor: nil) -> ConversionResult`**
+
+Converts HTML to Markdown. Returns a `ConversionResult` hash with all results in a single call.
+
+```ruby
+require 'html_to_markdown'
+
+result = HtmlToMarkdown.convert(html)
+markdown = result[:content]       # Converted Markdown string
+metadata = result[:metadata]      # Metadata (when extract_metadata: true)
+tables   = result[:tables]        # Structured table data (when extract_tables: true)
+document = result[:document]      # Document-level info
+images   = result[:images]        # Extracted images
+warnings = result[:warnings]      # Any conversion warnings
+```
+
+### Options
+
+**`ConversionOptions`** – Key configuration fields:
+
+- `heading_style`: Heading format (`"underlined"` | `"atx"` | `"atx_closed"`) — default: `"underlined"`
+- `list_indent_width`: Spaces per indent level — default: `2`
+- `bullets`: Bullet characters cycle — default: `"*+-"`
+- `wrap`: Enable text wrapping — default: `false`
+- `wrap_width`: Wrap at column — default: `80`
+- `code_language`: Default fenced code block language — default: none
+- `extract_metadata`: Enable metadata extraction into `result.metadata` — default: `false`
+- `extract_tables`: Enable structured table extraction into `result.tables` — default: `false`
+- `output_format`: Output markup format (`"markdown"` | `"djot"` | `"plain"`) — default: `"markdown"`
+
+## Djot Output Format
+
+The library supports converting HTML to [Djot](https://djot.net/), a lightweight markup language similar to Markdown but with a different syntax for some elements. Set `output_format` to `"djot"` to use this format.
+
+### Syntax Differences
+
+| Element        | Markdown   | Djot       |
+| -------------- | ---------- | ---------- |
+| Strong         | `**text**` | `*text*`   |
+| Emphasis       | `*text*`   | `_text_`   |
+| Strikethrough  | `~~text~~` | `{-text-}` |
+| Inserted/Added | N/A        | `{+text+}` |
+| Highlighted    | N/A        | `{=text=}` |
+| Subscript      | N/A        | `~text~`   |
+| Superscript    | N/A        | `^text^`   |
+
+### Example Usage
+
+```ruby
+require 'html_to_markdown'
+
+html = "<p>This is <strong>bold</strong> and <em>italic</em> text.</p>"
+
+# Default Markdown output
+markdown = HtmlToMarkdown.convert(html)
+# Result: "This is **bold** and *italic* text."
+
+# Djot output
+djot = HtmlToMarkdown.convert(html, output_format: 'djot')
+# Result: "This is *bold* and _italic_ text."
+```
+
+Djot's extended syntax allows you to express more semantic meaning in lightweight text, making it useful for documents that require strikethrough, insertion tracking, or mathematical notation.
+
+## Plain Text Output
+
+Set `output_format` to `"plain"` to strip all markup and return only visible text. This bypasses the Markdown conversion pipeline entirely for maximum speed.
+
+```ruby
+require 'html_to_markdown'
+
+html = "<h1>Title</h1><p>This is <strong>bold</strong> and <em>italic</em> text.</p>"
+
+plain = HtmlToMarkdown.convert(html, output_format: 'plain')
+# Result: "Title\n\nThis is bold and italic text."
+```
+
+Plain text mode is useful for search indexing, text extraction, and feeding content to LLMs.
+
+## Metadata Extraction
+
+The metadata extraction feature enables comprehensive document analysis during conversion. Extract document properties, headers, links, images, and structured data in a single pass — all via the standard `convert()` function.
+
+**Use Cases:**
+
+- **SEO analysis** – Extract title, description, Open Graph tags, Twitter cards
+- **Table of contents generation** – Build structured outlines from heading hierarchy
+- **Content migration** – Document all external links and resources
+- **Accessibility audits** – Check for images without alt text, empty links, invalid heading hierarchy
+- **Link validation** – Classify and validate anchor, internal, external, email, and phone links
+
+**Zero Overhead When Disabled:** Metadata extraction adds negligible overhead and happens during the HTML parsing pass. Pass `extract_metadata: true` in `ConversionOptions` to enable it; the result is available at `result.metadata`.
+
+### Example: Quick Start
+
+```ruby
+require 'html_to_markdown'
+
+html = '<h1>Article</h1><img src="test.jpg" alt="test">'
+result = HtmlToMarkdown.convert(html, extract_metadata: true)
+
+puts result[:content]                             # Converted Markdown
+puts result[:metadata][:document][:title]         # Document title
+puts result[:metadata][:headers]                  # All h1-h6 elements
+puts result[:metadata][:links]                    # All hyperlinks
+puts result[:metadata][:images]                   # All images with alt text
+puts result[:metadata][:structured_data]          # JSON-LD, Microdata, RDFa
+```
+
+## Visitor Pattern
+
+The visitor pattern enables custom HTML→Markdown conversion logic by providing callbacks for specific HTML elements during traversal. Pass a visitor as the third argument to `convert()`.
+
+**Use Cases:**
+
+- **Custom Markdown dialects** – Convert to Obsidian, Notion, or other flavors
+- **Content filtering** – Remove tracking pixels, ads, or unwanted elements
+- **URL rewriting** – Rewrite CDN URLs, add query parameters, validate links
+- **Accessibility validation** – Check alt text, heading hierarchy, link text
+- **Analytics** – Track element usage, link destinations, image sources
+
+**Supported Visitor Methods:** 40+ callbacks for text, inline elements, links, images, headings, lists, blocks, and tables.
+
+### Example: Quick Start
+
+```ruby
+require 'html_to_markdown'
+
+class MyVisitor
+  def visit_link(ctx, href, text, title = nil)
+    # Rewrite CDN URLs
+    if href.start_with?('https://old-cdn.com')
+      href = href.sub('https://old-cdn.com', 'https://new-cdn.com')
+    end
+    { type: :custom, output: "[#{text}](#{href})" }
+  end
+
+  def visit_image(ctx, src, alt = nil, title = nil)
+    # Skip tracking pixels
+    src.include?('tracking') ? { type: :skip } : { type: :continue }
+  end
+end
+
+html = '<a href="https://old-cdn.com/file.pdf">Download</a>'
+result = HtmlToMarkdown.convert(html, visitor: MyVisitor.new)
+markdown = result[:content]
+```
+
+## Examples
+
+## Links
+
+- **GitHub:** [github.com/kreuzberg-dev/html-to-markdown](https://github.com/kreuzberg-dev/html-to-markdown)
+- **RubyGems:** [rubygems.org/gems/html-to-markdown](https://rubygems.org/gems/html-to-markdown)
+- **Discord:** [discord.gg/xt9WY3GnKR](https://discord.gg/xt9WY3GnKR)
+
+## Part of Kreuzberg.dev
+
+- [Kreuzberg](https://github.com/kreuzberg-dev/kreuzberg) — document intelligence: text, tables, metadata from 90+ formats with optional OCR.
+- [Kreuzberg Cloud](https://github.com/kreuzberg-dev/kreuzberg-cloud) — managed extraction API with SDKs, dashboards, and observability.
+- [kreuzcrawl](https://github.com/kreuzberg-dev/kreuzcrawl) — web crawling and scraping with HTML→Markdown and headless-Chrome fallback.
+- [liter-llm](https://github.com/kreuzberg-dev/liter-llm) — universal LLM API client with native bindings for 14 languages and 143 providers.
+- [tree-sitter-language-pack](https://github.com/kreuzberg-dev/tree-sitter-language-pack) — tree-sitter grammars and code-intelligence primitives.
+- [alef](https://github.com/kreuzberg-dev/alef) — the polyglot binding generator that produces every per-language binding across the 5 polyglot repos.
+- [Discord](https://discord.gg/xt9WY3GnKR) — community, roadmap, announcements.
+
+## Contributing
+
+We welcome contributions! Please see our [Contributing Guide](https://github.com/kreuzberg-dev/html-to-markdown/blob/main/CONTRIBUTING.md) for details on:
+
+- Setting up the development environment
+- Running tests locally
+- Submitting pull requests
+- Reporting issues
+
+All contributions must follow our code quality standards (enforced via pre-commit hooks):
+
+- Proper test coverage (Rust 95%+, language bindings 80%+)
+- Formatting and linting checks
+- Documentation for public APIs
+
+## License
+
+MIT License – see [LICENSE](https://github.com/kreuzberg-dev/html-to-markdown/blob/main/LICENSE). Copyright © Kreuzberg, Inc.
+
+## Support
+
+If you find this library useful, consider [sponsoring the project](https://github.com/sponsors/kreuzberg-dev).
+
+Have questions or run into issues? We're here to help:
+
+- **GitHub Issues:** [github.com/kreuzberg-dev/html-to-markdown/issues](https://github.com/kreuzberg-dev/html-to-markdown/issues)
+- **Issues:** [github.com/kreuzberg-dev/html-to-markdown/issues](https://github.com/kreuzberg-dev/html-to-markdown/issues)
+- **Discord Community:** [discord.gg/xt9WY3GnKR](https://discord.gg/xt9WY3GnKR)

data/Steepfile

@@ -1,6 +1,14 @@
 # frozen_string_literal: true
 
 target :lib do
-  signature 'sig'
-  check 'lib'
+  signature "sig"
+  check "lib"
+  # The generated `lib/html_to_markdown/native.rb` carries inline Sorbet
+  # `sig { ... }` blocks on tagged-enum variant Data classes. Sorbet's runtime
+  # provides those via `extend T::Sig`, but Steep does not understand the
+  # extension (it relies on RBS, not Sorbet sigs) and reports
+  # `Type `self` does not have method `sig`` on every block. RBS coverage
+  # for the same surface lives in `sig/types.rbs`, so we steer Steep to the
+  # RBS file by ignoring the .rb.
+  ignore "lib/html_to_markdown/native.rb"
 end

data/ext/html_to_markdown_rb/Cargo.toml

@@ -1,7 +1,7 @@
 
 [package]
 name = "html-to-markdown-rb"
-version = "3.4.0"
+version = "3.6.0-rc.23"
 edition = "2024"
 license = "MIT"
 [workspace]
@@ -11,12 +11,13 @@ exclude = ["native"]
 crate-type = ["cdylib"]
 
 [dependencies]
-html-to-markdown-rs = { path = "../../vendor/html-to-markdown-rs", features = [
+html-to-markdown-rs = { path = "../../../../crates/html-to-markdown", features = [
     "full",
     "metadata",
     "visitor",
     "serde",
     "inline-images",
+    "testkit",
 ] }
 magnus = "0.8"
 serde = { version = "1", features = ["derive"] }

data/ext/html_to_markdown_rb/extconf.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require 'mkmf'
-require 'rb_sys/mkmf'
+require "mkmf"
+require "rb_sys/mkmf"
 
-default_profile = ENV.fetch('CARGO_PROFILE', 'release')
+default_profile = ENV.fetch("CARGO_PROFILE", "release")
 
-create_rust_makefile('html_to_markdown_rb') do |config|
+create_rust_makefile("html_to_markdown_rb") do |config|
   config.profile = default_profile.to_sym
-  config.ext_dir = 'native'
+  config.ext_dir = "native"
 end