mdream 0.1.1

package/dist/shared/mdream.a2AvjJLp.d.mts

@@ -0,0 +1,218 @@
+/**
+ * Plugin interface for extending HTML to Markdown conversion
+ */
+interface Plugin {
+    /**
+     * Process a node before it's handled by the parser
+     */
+    beforeNodeProcess?: (event: NodeEvent, state: MdreamRuntimeState) => undefined | void | {
+        skip: boolean;
+    };
+    /**
+     * Hook that runs when entering a node
+     * @returns String to add to the output, or PluginHookResult with content
+     */
+    onNodeEnter?: (node: ElementNode, state: MdreamRuntimeState) => string | undefined | void;
+    /**
+     * Hook that runs when exiting a node
+     * @param event - The node event
+     * @param state - The current runtime state
+     * @returns String to add to the output, or PluginHookResult with content
+     */
+    onNodeExit?: (node: ElementNode, state: MdreamRuntimeState) => string | undefined | void;
+    /**
+     * Process attributes for a node
+     * @param node - The node to process attributes for
+     * @param state - The current runtime state
+     */
+    processAttributes?: (node: ElementNode, state: MdreamRuntimeState) => void;
+    /**
+     * Process a text node before it's added to the output
+     * @param node - The text node to process
+     * @param state - The current runtime state
+     * @returns Legacy format or PluginHookResult with textContent and skipNode
+     */
+    processTextNode?: (node: TextNode, state: MdreamRuntimeState) => undefined | void | {
+        content: string;
+        skip: boolean;
+    };
+}
+/**
+ * Plugin creation options for controlling plugin behavior
+ */
+interface PluginCreationOptions {
+    /**
+     * Order in which plugins are executed
+     * Lower numbers run first
+     */
+    order?: number;
+    /**
+     * Priority for region conflict resolution
+     * Higher numbers take precedence over lower
+     */
+    priority?: number;
+}
+interface HTMLToMarkdownOptions {
+    /**
+     * Origin URL for resolving relative image paths and internal links.
+     * Important when converting HTML with relative paths from a specific website.
+     */
+    origin?: string;
+    /**
+     * Plugins to extend HTML to Markdown conversion
+     */
+    plugins?: Plugin[];
+}
+declare const ELEMENT_NODE = 1;
+declare const TEXT_NODE = 3;
+interface ElementNode extends Node {
+    /** Element tag name (for ELEMENT_NODE) */
+    name: string;
+    /** HTML attributes (for ELEMENT_NODE) */
+    attributes: Record<string, string>;
+    /** Custom data added by plugins */
+    context?: Record<string, any>;
+    /** ID of the tag for fast handler lookup */
+    tagId?: number;
+    /** Map of tag names to their nesting count (using Uint8Array for performance) */
+    depthMap: Uint8Array;
+}
+interface TextNode extends Node {
+    /** Text content (for TEXT_NODE) */
+    value: string;
+    /** Custom data added by plugins */
+    context?: Record<string, any>;
+}
+/**
+ * Base DOM node interface
+ * Optimized for streaming HTML parsing with minimal memory footprint
+ */
+interface Node {
+    /** Node type (ELEMENT_NODE or TEXT_NODE) */
+    type: number;
+    /** Current nesting depth in the DOM tree */
+    depth: number;
+    /** Node exclusion and filtering now handled by plugins */
+    /** Index of this node within its parent's children */
+    index: number;
+    /** Current walk index for child traversal during streaming */
+    currentWalkIndex?: number;
+    /** Count of text child nodes - used for whitespace handling */
+    childTextNodeIndex?: number;
+    /** Whether node contains whitespace - used for whitespace optimization */
+    containsWhitespace?: boolean;
+    /** Cached reference to tag handler for performance */
+    tagHandler?: TagHandler;
+    /** Parent node */
+    parent?: ElementNode | null;
+    /** Custom data added by plugins */
+    context?: Record<string, any>;
+    /** Region ID for buffer region tracking */
+    regionId?: number;
+}
+/**
+ * Buffer region for tracking content inclusion/exclusion
+ */
+interface BufferRegion {
+    /** Unique identifier */
+    id: number;
+    /** Inclusion state */
+    include: boolean;
+}
+/**
+ * State interface for HTML parsing and processing
+ * Contains parsing state that's maintained during HTML traversal
+ */
+interface MdreamProcessingState {
+    /** Map of tag names to their current nesting depth - uses TypedArray for performance */
+    depthMap: Uint8Array;
+    /** Current overall nesting depth */
+    depth: number;
+    /** Currently processing element node */
+    currentNode?: ElementNode | null;
+    /** Node filtering and exclusion is now handled by plugins */
+    /** Whether current content contains HTML entities that need decoding */
+    hasEncodedHtmlEntity?: boolean;
+    /** Whether the last processed character was whitespace - for collapsing whitespace */
+    lastCharWasWhitespace?: boolean;
+    /** Whether the last processed buffer has whitespace - optimization flag */
+    textBufferContainsWhitespace?: boolean;
+    /** Whether the last processed buffer contains non-whitespace characters */
+    textBufferContainsNonWhitespace?: boolean;
+    /** Whether a tag was just closed - affects whitespace handling */
+    justClosedTag?: boolean;
+    /** Whether the next text node is the first in its element - for whitespace trimming */
+    isFirstTextInElement?: boolean;
+    /** Reference to the last processed text node - for context tracking */
+    lastTextNode?: Node;
+    /** Plugin instances array for efficient iteration */
+    plugins?: Plugin[];
+    /** Configuration options for conversion */
+    options?: HTMLToMarkdownOptions;
+}
+/**
+ * Runtime state for markdown generation
+ * Extended state that includes output tracking and options
+ */
+interface MdreamRuntimeState extends Partial<MdreamProcessingState> {
+    /** Number of newlines at end of most recent output */
+    lastNewLines?: number;
+    /** Configuration options for conversion */
+    options?: HTMLToMarkdownOptions;
+    /** Table processing state - specialized for Markdown tables */
+    tableRenderedTable?: boolean;
+    tableCurrentRowCells?: number;
+    tableColumnAlignments?: string[];
+    /** Plugin instances array for efficient iteration */
+    plugins?: Plugin[];
+    /** Map of region IDs to buffer regions for O(1) lookups */
+    regionToggles: Map<number, boolean>;
+    /** Content buffers for regions */
+    regionContentBuffers: Map<number, string[]>;
+    /** Performance cache for last content to avoid iteration */
+    lastContentCache?: string;
+    /** Reference to the last processed node */
+    lastNode?: Node;
+    context?: Record<string, any>;
+}
+type NodeEventEnter = 0;
+type NodeEventExit = 1;
+/**
+ * Node event for DOM traversal
+ * Used in the event-based traversal system for streaming processing
+ */
+interface NodeEvent {
+    /** Event type - enter (start tag) or exit (end tag) */
+    type: NodeEventEnter | NodeEventExit;
+    /** The node being processed */
+    node: Node;
+}
+/**
+ * Handler context for markdown conversion
+ * Passed to tag handler functions for converting specific elements
+ */
+interface HandlerContext {
+    /** Current node being processed */
+    node: ElementNode;
+    /** Parent node (if any) */
+    parent?: ElementNode;
+    /** Runtime state */
+    state: MdreamRuntimeState;
+}
+/**
+ * Tag handler interface for HTML elements
+ * Used by plugins to extend or customize tag handling
+ */
+interface TagHandler {
+    enter?: (context: HandlerContext) => string | undefined | void;
+    exit?: (context: HandlerContext) => string | undefined | void;
+    isSelfClosing?: boolean;
+    isNonNesting?: boolean;
+    collapsesInnerWhiteSpace?: boolean;
+    isInline?: boolean;
+    spacing?: readonly [number, number];
+    excludesTextNodes?: boolean;
+}
+
+export { ELEMENT_NODE as E, TEXT_NODE as T };
+export type { BufferRegion as B, HTMLToMarkdownOptions as H, MdreamProcessingState as M, Node as N, Plugin as P, PluginCreationOptions as a, ElementNode as b, TextNode as c, MdreamRuntimeState as d, NodeEvent as e, HandlerContext as f, TagHandler as g };

package/dist/shared/mdream.a2AvjJLp.d.ts

@@ -0,0 +1,218 @@
+/**
+ * Plugin interface for extending HTML to Markdown conversion
+ */
+interface Plugin {
+    /**
+     * Process a node before it's handled by the parser
+     */
+    beforeNodeProcess?: (event: NodeEvent, state: MdreamRuntimeState) => undefined | void | {
+        skip: boolean;
+    };
+    /**
+     * Hook that runs when entering a node
+     * @returns String to add to the output, or PluginHookResult with content
+     */
+    onNodeEnter?: (node: ElementNode, state: MdreamRuntimeState) => string | undefined | void;
+    /**
+     * Hook that runs when exiting a node
+     * @param event - The node event
+     * @param state - The current runtime state
+     * @returns String to add to the output, or PluginHookResult with content
+     */
+    onNodeExit?: (node: ElementNode, state: MdreamRuntimeState) => string | undefined | void;
+    /**
+     * Process attributes for a node
+     * @param node - The node to process attributes for
+     * @param state - The current runtime state
+     */
+    processAttributes?: (node: ElementNode, state: MdreamRuntimeState) => void;
+    /**
+     * Process a text node before it's added to the output
+     * @param node - The text node to process
+     * @param state - The current runtime state
+     * @returns Legacy format or PluginHookResult with textContent and skipNode
+     */
+    processTextNode?: (node: TextNode, state: MdreamRuntimeState) => undefined | void | {
+        content: string;
+        skip: boolean;
+    };
+}
+/**
+ * Plugin creation options for controlling plugin behavior
+ */
+interface PluginCreationOptions {
+    /**
+     * Order in which plugins are executed
+     * Lower numbers run first
+     */
+    order?: number;
+    /**
+     * Priority for region conflict resolution
+     * Higher numbers take precedence over lower
+     */
+    priority?: number;
+}
+interface HTMLToMarkdownOptions {
+    /**
+     * Origin URL for resolving relative image paths and internal links.
+     * Important when converting HTML with relative paths from a specific website.
+     */
+    origin?: string;
+    /**
+     * Plugins to extend HTML to Markdown conversion
+     */
+    plugins?: Plugin[];
+}
+declare const ELEMENT_NODE = 1;
+declare const TEXT_NODE = 3;
+interface ElementNode extends Node {
+    /** Element tag name (for ELEMENT_NODE) */
+    name: string;
+    /** HTML attributes (for ELEMENT_NODE) */
+    attributes: Record<string, string>;
+    /** Custom data added by plugins */
+    context?: Record<string, any>;
+    /** ID of the tag for fast handler lookup */
+    tagId?: number;
+    /** Map of tag names to their nesting count (using Uint8Array for performance) */
+    depthMap: Uint8Array;
+}
+interface TextNode extends Node {
+    /** Text content (for TEXT_NODE) */
+    value: string;
+    /** Custom data added by plugins */
+    context?: Record<string, any>;
+}
+/**
+ * Base DOM node interface
+ * Optimized for streaming HTML parsing with minimal memory footprint
+ */
+interface Node {
+    /** Node type (ELEMENT_NODE or TEXT_NODE) */
+    type: number;
+    /** Current nesting depth in the DOM tree */
+    depth: number;
+    /** Node exclusion and filtering now handled by plugins */
+    /** Index of this node within its parent's children */
+    index: number;
+    /** Current walk index for child traversal during streaming */
+    currentWalkIndex?: number;
+    /** Count of text child nodes - used for whitespace handling */
+    childTextNodeIndex?: number;
+    /** Whether node contains whitespace - used for whitespace optimization */
+    containsWhitespace?: boolean;
+    /** Cached reference to tag handler for performance */
+    tagHandler?: TagHandler;
+    /** Parent node */
+    parent?: ElementNode | null;
+    /** Custom data added by plugins */
+    context?: Record<string, any>;
+    /** Region ID for buffer region tracking */
+    regionId?: number;
+}
+/**
+ * Buffer region for tracking content inclusion/exclusion
+ */
+interface BufferRegion {
+    /** Unique identifier */
+    id: number;
+    /** Inclusion state */
+    include: boolean;
+}
+/**
+ * State interface for HTML parsing and processing
+ * Contains parsing state that's maintained during HTML traversal
+ */
+interface MdreamProcessingState {
+    /** Map of tag names to their current nesting depth - uses TypedArray for performance */
+    depthMap: Uint8Array;
+    /** Current overall nesting depth */
+    depth: number;
+    /** Currently processing element node */
+    currentNode?: ElementNode | null;
+    /** Node filtering and exclusion is now handled by plugins */
+    /** Whether current content contains HTML entities that need decoding */
+    hasEncodedHtmlEntity?: boolean;
+    /** Whether the last processed character was whitespace - for collapsing whitespace */
+    lastCharWasWhitespace?: boolean;
+    /** Whether the last processed buffer has whitespace - optimization flag */
+    textBufferContainsWhitespace?: boolean;
+    /** Whether the last processed buffer contains non-whitespace characters */
+    textBufferContainsNonWhitespace?: boolean;
+    /** Whether a tag was just closed - affects whitespace handling */
+    justClosedTag?: boolean;
+    /** Whether the next text node is the first in its element - for whitespace trimming */
+    isFirstTextInElement?: boolean;
+    /** Reference to the last processed text node - for context tracking */
+    lastTextNode?: Node;
+    /** Plugin instances array for efficient iteration */
+    plugins?: Plugin[];
+    /** Configuration options for conversion */
+    options?: HTMLToMarkdownOptions;
+}
+/**
+ * Runtime state for markdown generation
+ * Extended state that includes output tracking and options
+ */
+interface MdreamRuntimeState extends Partial<MdreamProcessingState> {
+    /** Number of newlines at end of most recent output */
+    lastNewLines?: number;
+    /** Configuration options for conversion */
+    options?: HTMLToMarkdownOptions;
+    /** Table processing state - specialized for Markdown tables */
+    tableRenderedTable?: boolean;
+    tableCurrentRowCells?: number;
+    tableColumnAlignments?: string[];
+    /** Plugin instances array for efficient iteration */
+    plugins?: Plugin[];
+    /** Map of region IDs to buffer regions for O(1) lookups */
+    regionToggles: Map<number, boolean>;
+    /** Content buffers for regions */
+    regionContentBuffers: Map<number, string[]>;
+    /** Performance cache for last content to avoid iteration */
+    lastContentCache?: string;
+    /** Reference to the last processed node */
+    lastNode?: Node;
+    context?: Record<string, any>;
+}
+type NodeEventEnter = 0;
+type NodeEventExit = 1;
+/**
+ * Node event for DOM traversal
+ * Used in the event-based traversal system for streaming processing
+ */
+interface NodeEvent {
+    /** Event type - enter (start tag) or exit (end tag) */
+    type: NodeEventEnter | NodeEventExit;
+    /** The node being processed */
+    node: Node;
+}
+/**
+ * Handler context for markdown conversion
+ * Passed to tag handler functions for converting specific elements
+ */
+interface HandlerContext {
+    /** Current node being processed */
+    node: ElementNode;
+    /** Parent node (if any) */
+    parent?: ElementNode;
+    /** Runtime state */
+    state: MdreamRuntimeState;
+}
+/**
+ * Tag handler interface for HTML elements
+ * Used by plugins to extend or customize tag handling
+ */
+interface TagHandler {
+    enter?: (context: HandlerContext) => string | undefined | void;
+    exit?: (context: HandlerContext) => string | undefined | void;
+    isSelfClosing?: boolean;
+    isNonNesting?: boolean;
+    collapsesInnerWhiteSpace?: boolean;
+    isInline?: boolean;
+    spacing?: readonly [number, number];
+    excludesTextNodes?: boolean;
+}
+
+export { ELEMENT_NODE as E, TEXT_NODE as T };
+export type { BufferRegion as B, HTMLToMarkdownOptions as H, MdreamProcessingState as M, Node as N, Plugin as P, PluginCreationOptions as a, ElementNode as b, TextNode as c, MdreamRuntimeState as d, NodeEvent as e, HandlerContext as f, TagHandler as g };

package/dist/shared/mdream.cpEmpxyh.mjs

@@ -0,0 +1,105 @@
+import { aa as ELEMENT_NODE, b as TAG_HEAD, ab as collectNodeContent, ac as TAG_TITLE, ad as TAG_META } from './mdream.-hdaPj9a.mjs';
+
+function createPlugin(plugin) {
+  return plugin;
+}
+
+function frontmatterPlugin(options = {}) {
+  const additionalFields = options.additionalFields || {};
+  const metaFields = /* @__PURE__ */ new Set([
+    "description",
+    "keywords",
+    "author",
+    "date",
+    "og:title",
+    "og:description",
+    "twitter:title",
+    "twitter:description",
+    ...options.metaFields || []
+  ]);
+  const frontmatter = { ...additionalFields, meta: {} };
+  let inHead = false;
+  const formatValue = options.formatValue || ((name, value) => {
+    value = value.replace(/"/g, '\\"');
+    if (value.includes("\n") || value.includes(":") || value.includes("#") || value.includes(" ")) {
+      return `"${value}"`;
+    }
+    return value;
+  });
+  return createPlugin({
+    onNodeEnter(node) {
+      if (node.tagId === TAG_HEAD) {
+        inHead = true;
+        return;
+      }
+      if (inHead && node.type === ELEMENT_NODE && node.tagId === TAG_TITLE) {
+        return;
+      }
+      if (inHead && node.type === ELEMENT_NODE && node.tagId === TAG_META) {
+        const elementNode = node;
+        const { name, property, content } = elementNode.attributes || {};
+        const metaName = property || name;
+        if (metaName && content && metaFields.has(metaName)) {
+          frontmatter.meta[metaName.includes(":") ? `"${metaName}"` : metaName] = formatValue(metaName, content);
+        }
+        return void 0;
+      }
+    },
+    onNodeExit(node, state) {
+      if (node.type === ELEMENT_NODE && node.tagId === TAG_HEAD) {
+        inHead = false;
+        if (Object.keys(frontmatter).length > 0) {
+          const frontmatterContent = generateFrontmatter();
+          collectNodeContent({ regionId: 0 }, frontmatterContent, state);
+        }
+      }
+      return void 0;
+    },
+    processTextNode(node) {
+      if (!inHead) {
+        return;
+      }
+      const parent = node.parent;
+      if (parent && parent.tagId === TAG_TITLE && node.value) {
+        frontmatter.title = formatValue("title", node.value.trim());
+        return { content: "", skip: true };
+      }
+    }
+  });
+  function generateFrontmatter() {
+    if (Object.keys(frontmatter).length === 0) {
+      return "";
+    }
+    let yamlLines = [];
+    const entries = Object.entries(frontmatter).sort(([a], [b]) => {
+      if (a === "title")
+        return -1;
+      if (b === "title")
+        return 1;
+      if (a === "description")
+        return -1;
+      if (b === "description")
+        return 1;
+      return a.localeCompare(b);
+    });
+    for (const [key, value] of entries) {
+      if (key === "meta" && Object.keys(value).length > 0) {
+        yamlLines.push("meta:");
+        const metaEntries = Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([metaKey, metaValue]) => `  ${metaKey}: ${metaValue}`);
+        yamlLines.push(...metaEntries);
+      } else if (key !== "meta" || Object.keys(value).length > 0) {
+        yamlLines.push(`${key}: ${value}`);
+      }
+    }
+    if (Object.keys(frontmatter.meta).length === 0) {
+      yamlLines = yamlLines.filter((line) => !line.startsWith("meta:"));
+    }
+    return `---
+${yamlLines.join("\n")}
+---
+
+`;
+  }
+}
+
+export { createPlugin as c, frontmatterPlugin as f };

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "mdream",
+  "type": "module",
+  "version": "0.1.1",
+  "description": "Ultra-performant JavaScript HTML to Markdown converter optimized for LLMs.",
+  "author": {
+    "name": "Harlan Wilton",
+    "email": "harlan@harlanzw.com",
+    "url": "https://harlanzw.com/"
+  },
+  "license": "MIT",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs"
+    },
+    "./cli": "./dist/cli.mjs",
+    "./plugins": "./dist/plugins.mjs",
+    "./preset/minimal": "./dist/preset/minimal.mjs"
+  },
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "bin": "./bin/mdream.mjs",
+  "files": [
+    "bin",
+    "dist"
+  ],
+  "dependencies": {
+    "cac": "^6.7.14"
+  },
+  "devDependencies": {
+    "@antfu/eslint-config": "^4.13.2",
+    "@types/node": "^22.15.29",
+    "bumpp": "^10.1.1",
+    "crawlee": "^3.13.5",
+    "eslint": "^9.28.0",
+    "llm-cost": "^1.0.5",
+    "playwright": "^1.52.0",
+    "typescript": "5.8.3",
+    "unbuild": "^3.5.0",
+    "vitest": "^3.1.4"
+  },
+  "scripts": {
+    "flame": "pnpm build && unbuild bench/bundle && clinic flame -- node bench/bundle/dist/string.mjs 10",
+    "bench:build": "pnpm build && unbuild bench/bundle",
+    "bench:stream": "pnpm build && unbuild bench/bundle && hyperfine --runs 100 'node bench/bundle/dist/stream.mjs' --warmup 3",
+    "bench:await": "pnpm build && unbuild bench/bundle && hyperfine --runs 100 'node bench/bundle/dist/await.mjs' --warmup 3",
+    "bench:string": "pnpm build && unbuild bench/bundle && hyperfine --runs 100 'node bench/bundle/dist/string.mjs' --warmup 3",
+    "ts": "node  --experimental-strip-types",
+    "test:live": "curl -s https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax | node ./bin/mdream.mjs --origin https://docs.github.com | tee test/github-markdown.md",
+    "test:images": "cat test/fixtures/test-origin.html | node ./bin/mdream.mjs --origin https://docs.github.com",
+    "test:github:live": "curl -s https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax | node ./bin/mdream.mjs --origin https://docs.github.com | tee test/github-markdown.md",
+    "test:github:file": "cat test/fixtures/github-markdown-complete.html | node ./bin/mdream.mjs --origin https://docs.github.com | tee test/github-markdown.md",
+    "test:wiki:file": "pnpm build && cat test/fixtures/wikipedia-largest.html | node ./bin/mdream.mjs --origin https://en.wikipedia.org | tee test/wiki-markdown.md",
+    "test:wiki-small:file": "cat test/fixtures/wikipedia-small.html | node ./bin/mdream.mjs --origin https://en.wikipedia.org | tee test/wiki-markdown.md",
+    "build": "unbuild",
+    "typecheck": "tsc --noEmit src/index.ts",
+    "dev:prepare": "unbuild --stub",
+    "test": "vitest test",
+    "release": "pnpm build && bumpp && pnpm -r publish"
+  }
+}