@kreuzberg/html-to-markdown-wasm 3.5.5 → 3.6.0-rc.10

package/README.md

@@ -126,6 +126,25 @@ const markdown = result.content;
 console.log(markdown);
 ```
 
+## Architecture
+
+The converter routes each input through one of three tiers based on a fast prescan of the byte stream:
+
+1. **Tier-1 — single-pass byte scanner.** Handles 110+ HTML tags directly. Bails on any construct it cannot prove byte-equivalent to Tier-2.
+2. **Tier-2 — DOM walker.** Picks up Tier-1 bails and inputs the classifier rejected up front.
+3. **Tier-3 — standards-conformant parser.** Engaged for malformed HTML requiring full HTML5 repair.
+
+The dispatcher is invisible to the caller. Output is byte-identical across tiers — enforced by a 116-snapshot oracle.
+
+## Capabilities
+
+- **16 languages, one Rust core.** Rust, Python, Node.js, WASM, Java, Go, C#, PHP, Ruby, Elixir, R, Dart, Kotlin (Android), Swift, Zig, C ABI.
+- **CommonMark-compatible Markdown** with GFM-style tables.
+- **Djot output**: set `output_format = "djot"` (see Djot Output Format section below).
+- **Real-HTML robust**: unclosed tags, CDATA, custom elements, malformed entities, nested tables, mixed encodings handled without losing content.
+- **Metadata extraction**, **visitor API**, **inline images**, **configurable preprocessing presets**.
+- **Per-group regression gates in CI**: every PR runs the bench harness against per-group thresholds.
+
 ## API Reference
 
 ### Core Function

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kreuzberg/html-to-markdown-wasm",
-  "version": "3.5.5",
+  "version": "3.6.0-rc.10",
   "private": false,
   "description": "High-performance HTML to Markdown converter",
   "license": "MIT",

package/pkg/bundler/README.md

@@ -126,6 +126,25 @@ const markdown = result.content;
 console.log(markdown);
 ```
 
+## Architecture
+
+The converter routes each input through one of three tiers based on a fast prescan of the byte stream:
+
+1. **Tier-1 — single-pass byte scanner.** Handles 110+ HTML tags directly. Bails on any construct it cannot prove byte-equivalent to Tier-2.
+2. **Tier-2 — DOM walker.** Picks up Tier-1 bails and inputs the classifier rejected up front.
+3. **Tier-3 — standards-conformant parser.** Engaged for malformed HTML requiring full HTML5 repair.
+
+The dispatcher is invisible to the caller. Output is byte-identical across tiers — enforced by a 116-snapshot oracle.
+
+## Capabilities
+
+- **16 languages, one Rust core.** Rust, Python, Node.js, WASM, Java, Go, C#, PHP, Ruby, Elixir, R, Dart, Kotlin (Android), Swift, Zig, C ABI.
+- **CommonMark-compatible Markdown** with GFM-style tables.
+- **Djot output**: set `output_format = "djot"` (see Djot Output Format section below).
+- **Real-HTML robust**: unclosed tags, CDATA, custom elements, malformed entities, nested tables, mixed encodings handled without losing content.
+- **Metadata extraction**, **visitor API**, **inline images**, **configurable preprocessing presets**.
+- **Per-group regression gates in CI**: every PR runs the bench harness against per-group thresholds.
+
 ## API Reference
 
 ### Core Function

package/pkg/bundler/html_to_markdown_wasm.d.ts

@@ -40,7 +40,7 @@ export class WasmConversionOptions {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmConversionOptions;
-    constructor(headingStyle?: WasmHeadingStyle | null, listIndentType?: WasmListIndentType | null, listIndentWidth?: number | null, bullets?: string | null, strongEmSymbol?: string | null, escapeAsterisks?: boolean | null, escapeUnderscores?: boolean | null, escapeMisc?: boolean | null, escapeAscii?: boolean | null, codeLanguage?: string | null, autolinks?: boolean | null, defaultTitle?: boolean | null, brInTables?: boolean | null, compactTables?: boolean | null, highlightStyle?: WasmHighlightStyle | null, extractMetadata?: boolean | null, whitespaceMode?: WasmWhitespaceMode | null, stripNewlines?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, convertAsInline?: boolean | null, subSymbol?: string | null, supSymbol?: string | null, newlineStyle?: WasmNewlineStyle | null, codeBlockStyle?: WasmCodeBlockStyle | null, keepInlineImagesIn?: string[] | null, preprocessing?: WasmPreprocessingOptions | null, encoding?: string | null, debug?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, skipImages?: boolean | null, linkStyle?: WasmLinkStyle | null, outputFormat?: WasmOutputFormat | null, includeDocumentStructure?: boolean | null, extractImages?: boolean | null, maxImageSize?: bigint | null, captureSvg?: boolean | null, inferDimensions?: boolean | null, excludeSelectors?: string[] | null, maxDepth?: number | null);
+    constructor(headingStyle?: WasmHeadingStyle | null, listIndentType?: WasmListIndentType | null, listIndentWidth?: number | null, bullets?: string | null, strongEmSymbol?: string | null, escapeAsterisks?: boolean | null, escapeUnderscores?: boolean | null, escapeMisc?: boolean | null, escapeAscii?: boolean | null, codeLanguage?: string | null, autolinks?: boolean | null, defaultTitle?: boolean | null, brInTables?: boolean | null, compactTables?: boolean | null, highlightStyle?: WasmHighlightStyle | null, extractMetadata?: boolean | null, whitespaceMode?: WasmWhitespaceMode | null, stripNewlines?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, convertAsInline?: boolean | null, subSymbol?: string | null, supSymbol?: string | null, newlineStyle?: WasmNewlineStyle | null, codeBlockStyle?: WasmCodeBlockStyle | null, keepInlineImagesIn?: string[] | null, preprocessing?: WasmPreprocessingOptions | null, encoding?: string | null, debug?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, skipImages?: boolean | null, urlEscapeStyle?: WasmUrlEscapeStyle | null, linkStyle?: WasmLinkStyle | null, outputFormat?: WasmOutputFormat | null, includeDocumentStructure?: boolean | null, extractImages?: boolean | null, maxImageSize?: bigint | null, captureSvg?: boolean | null, inferDimensions?: boolean | null, excludeSelectors?: string[] | null, tierStrategy?: WasmTierStrategy | null, maxDepth?: number | null);
     autolinks: boolean;
     brInTables: boolean;
     bullets: string;
@@ -87,6 +87,10 @@ export class WasmConversionOptions {
     strongEmSymbol: string;
     subSymbol: string;
     supSymbol: string;
+    get tierStrategy(): string;
+    set tierStrategy(value: WasmTierStrategy);
+    get urlEscapeStyle(): string;
+    set urlEscapeStyle(value: WasmUrlEscapeStyle);
     get visitor(): WasmVisitorHandle | undefined;
     set visitor(value: WasmVisitorHandle | null | undefined);
     get whitespaceMode(): string;
@@ -105,7 +109,7 @@ export class WasmConversionOptionsUpdate {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmConversionOptionsUpdate;
-    constructor(headingStyle?: WasmHeadingStyle | null, listIndentType?: WasmListIndentType | null, listIndentWidth?: number | null, bullets?: string | null, strongEmSymbol?: string | null, escapeAsterisks?: boolean | null, escapeUnderscores?: boolean | null, escapeMisc?: boolean | null, escapeAscii?: boolean | null, codeLanguage?: string | null, autolinks?: boolean | null, defaultTitle?: boolean | null, brInTables?: boolean | null, compactTables?: boolean | null, highlightStyle?: WasmHighlightStyle | null, extractMetadata?: boolean | null, whitespaceMode?: WasmWhitespaceMode | null, stripNewlines?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, convertAsInline?: boolean | null, subSymbol?: string | null, supSymbol?: string | null, newlineStyle?: WasmNewlineStyle | null, codeBlockStyle?: WasmCodeBlockStyle | null, keepInlineImagesIn?: string[] | null, preprocessing?: WasmPreprocessingOptionsUpdate | null, encoding?: string | null, debug?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, skipImages?: boolean | null, linkStyle?: WasmLinkStyle | null, outputFormat?: WasmOutputFormat | null, includeDocumentStructure?: boolean | null, extractImages?: boolean | null, maxImageSize?: bigint | null, captureSvg?: boolean | null, inferDimensions?: boolean | null, maxDepth?: number | null, excludeSelectors?: string[] | null);
+    constructor(headingStyle?: WasmHeadingStyle | null, listIndentType?: WasmListIndentType | null, listIndentWidth?: number | null, bullets?: string | null, strongEmSymbol?: string | null, escapeAsterisks?: boolean | null, escapeUnderscores?: boolean | null, escapeMisc?: boolean | null, escapeAscii?: boolean | null, codeLanguage?: string | null, autolinks?: boolean | null, defaultTitle?: boolean | null, brInTables?: boolean | null, compactTables?: boolean | null, highlightStyle?: WasmHighlightStyle | null, extractMetadata?: boolean | null, whitespaceMode?: WasmWhitespaceMode | null, stripNewlines?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, convertAsInline?: boolean | null, subSymbol?: string | null, supSymbol?: string | null, newlineStyle?: WasmNewlineStyle | null, codeBlockStyle?: WasmCodeBlockStyle | null, keepInlineImagesIn?: string[] | null, preprocessing?: WasmPreprocessingOptionsUpdate | null, encoding?: string | null, debug?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, skipImages?: boolean | null, urlEscapeStyle?: WasmUrlEscapeStyle | null, linkStyle?: WasmLinkStyle | null, outputFormat?: WasmOutputFormat | null, includeDocumentStructure?: boolean | null, extractImages?: boolean | null, maxImageSize?: bigint | null, captureSvg?: boolean | null, inferDimensions?: boolean | null, maxDepth?: number | null, excludeSelectors?: string[] | null, tierStrategy?: WasmTierStrategy | null);
     get autolinks(): boolean | undefined;
     set autolinks(value: boolean | null | undefined);
     get brInTables(): boolean | undefined;
@@ -182,6 +186,10 @@ export class WasmConversionOptionsUpdate {
     set subSymbol(value: string | null | undefined);
     get supSymbol(): string | undefined;
     set supSymbol(value: string | null | undefined);
+    get tierStrategy(): string | undefined;
+    set tierStrategy(value: WasmTierStrategy | null | undefined);
+    get urlEscapeStyle(): string | undefined;
+    set urlEscapeStyle(value: WasmUrlEscapeStyle | null | undefined);
     get visitor(): WasmVisitorHandle | undefined;
     set visitor(value: WasmVisitorHandle | null | undefined);
     get whitespaceMode(): string | undefined;
@@ -217,7 +225,6 @@ export class WasmConversionResult {
     set content(value: string | null | undefined);
     get document(): WasmDocumentStructure | undefined;
     set document(value: WasmDocumentStructure | null | undefined);
-    images: string[];
     metadata: WasmHtmlMetadata;
     tables: WasmTableData[];
     warnings: WasmProcessingWarning[];
@@ -379,6 +386,23 @@ export class WasmHtmlMetadata {
     structuredData: WasmStructuredData[];
 }
 
+/**
+ * Image dimensions in pixels.
+ *
+ * Binding-safe replacement for `(u32, u32)` tuples, which degrade to
+ * `Vec<Vec<String>>` when sanitized for cross-language binding generation.
+ * Used by both `ImageMetadata` and
+ * `InlineImage`.
+ */
+export class WasmImageDimensions {
+    free(): void;
+    [Symbol.dispose](): void;
+    static default(): WasmImageDimensions;
+    constructor(width: number, height: number);
+    height: number;
+    width: number;
+}
+
 /**
  * Image metadata with source and dimensions.
  *
@@ -391,12 +415,12 @@ export class WasmImageMetadata {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmImageMetadata;
-    constructor(src: string, imageType: WasmImageType, attributes: any, alt?: string | null, title?: string | null, dimensions?: Uint32Array | null);
+    constructor(src: string, imageType: WasmImageType, attributes: any, alt?: string | null, title?: string | null, dimensions?: WasmImageDimensions | null);
     get alt(): string | undefined;
     set alt(value: string | null | undefined);
     attributes: any;
-    get dimensions(): Uint32Array | undefined;
-    set dimensions(value: Uint32Array | null | undefined);
+    get dimensions(): WasmImageDimensions | undefined;
+    set dimensions(value: WasmImageDimensions | null | undefined);
     get imageType(): string;
     set imageType(value: WasmImageType);
     src: string;
@@ -473,6 +497,22 @@ export enum WasmListIndentType {
     Tabs = 1,
 }
 
+/**
+ * A single key-value metadata entry from `<head>` meta tags.
+ *
+ * Binding-safe replacement for `(String, String)` tuples used in
+ * `NodeContent.MetadataBlock`. Tuple pairs cannot be represented
+ * across language boundaries without lossy degradation.
+ */
+export class WasmMetadataEntry {
+    free(): void;
+    [Symbol.dispose](): void;
+    static default(): WasmMetadataEntry;
+    constructor(key: string, value: string);
+    key: string;
+    value: string;
+}
+
 /**
  * Line break syntax in Markdown output.
  *
@@ -499,8 +539,8 @@ export class WasmNodeContent {
     set definition(value: string | null | undefined);
     get description(): string | undefined;
     set description(value: string | null | undefined);
-    get entries(): any | undefined;
-    set entries(value: any | null | undefined);
+    get entries(): WasmMetadataEntry[] | undefined;
+    set entries(value: WasmMetadataEntry[] | null | undefined);
     get format(): string | undefined;
     set format(value: string | null | undefined);
     get grid(): WasmTableGrid | undefined;
@@ -826,6 +866,34 @@ export enum WasmTextDirection {
     Auto = 2,
 }
 
+/**
+ * Controls which conversion tier is used.
+ */
+export enum WasmTierStrategy {
+    Auto = 0,
+    Tier2 = 1,
+    Tier1 = 2,
+}
+
+/**
+ * URL encoding strategy for link and image destinations.
+ *
+ * Controls how special characters in URL destinations are handled when they
+ * require escaping to produce valid Markdown.
+ *
+ * The `Angle` variant (default) wraps the destination in angle brackets:
+ * `[text](<url with spaces>)`. This is the CommonMark-specified escape hatch
+ * but breaks when the URL itself contains `>`.
+ *
+ * The `Percent` variant percent-encodes every character that is not an RFC 3986
+ * unreserved character or `/`, producing a destination safe for all Markdown
+ * parsers: `[text](url%20with%20spaces)`.
+ */
+export enum WasmUrlEscapeStyle {
+    Angle = 0,
+    Percent = 1,
+}
+
 /**
  * Result of a visitor callback.
  *
@@ -884,7 +952,12 @@ export enum WasmWhitespaceMode {
  * # Arguments
  *
  * * `html` — the HTML string to convert.
- * * `options` — optional conversion options. Defaults to `ConversionOptions.default`.
+ * * `options` — conversion options. The parameter bound is
+ *   `impl Into<Option<ConversionOptions>>`, so any of the following call shapes are accepted:
+ *   - `convert(html, ConversionOptions.default())` — bare options.
+ *   - `convert(html, opts)` — bare options.
+ *   - `convert(html, Some(opts))` — explicit `Option`.
+ *   - `convert(html, None)` — fall back to `ConversionOptions.default`.
  *
  * # Example
  *

package/pkg/bundler/html_to_markdown_wasm.js

@@ -5,5 +5,5 @@ import { __wbg_set_wasm } from "./html_to_markdown_wasm_bg.js";
 __wbg_set_wasm(wasm);
 
 export {
-    WasmAnnotationKind, WasmCodeBlockStyle, WasmConversionOptions, WasmConversionOptionsUpdate, WasmConversionResult, WasmDocumentMetadata, WasmDocumentNode, WasmDocumentStructure, WasmGridCell, WasmHeaderMetadata, WasmHeadingStyle, WasmHighlightStyle, WasmHtmlMetadata, WasmImageMetadata, WasmImageType, WasmLinkMetadata, WasmLinkStyle, WasmLinkType, WasmListIndentType, WasmNewlineStyle, WasmNodeContent, WasmNodeContext, WasmNodeType, WasmOutputFormat, WasmPreprocessingOptions, WasmPreprocessingOptionsUpdate, WasmPreprocessingPreset, WasmProcessingWarning, WasmStructuredData, WasmStructuredDataType, WasmTableData, WasmTableGrid, WasmTextAnnotation, WasmTextDirection, WasmVisitResult, WasmVisitorHandle, WasmWarningKind, WasmWhitespaceMode, convert
+    WasmAnnotationKind, WasmCodeBlockStyle, WasmConversionOptions, WasmConversionOptionsUpdate, WasmConversionResult, WasmDocumentMetadata, WasmDocumentNode, WasmDocumentStructure, WasmGridCell, WasmHeaderMetadata, WasmHeadingStyle, WasmHighlightStyle, WasmHtmlMetadata, WasmImageDimensions, WasmImageMetadata, WasmImageType, WasmLinkMetadata, WasmLinkStyle, WasmLinkType, WasmListIndentType, WasmMetadataEntry, WasmNewlineStyle, WasmNodeContent, WasmNodeContext, WasmNodeType, WasmOutputFormat, WasmPreprocessingOptions, WasmPreprocessingOptionsUpdate, WasmPreprocessingPreset, WasmProcessingWarning, WasmStructuredData, WasmStructuredDataType, WasmTableData, WasmTableGrid, WasmTextAnnotation, WasmTextDirection, WasmTierStrategy, WasmUrlEscapeStyle, WasmVisitResult, WasmVisitorHandle, WasmWarningKind, WasmWhitespaceMode, convert
 } from "./html_to_markdown_wasm_bg.js";