nuxt-ai-ready 2.5.0 → 2.5.2

package/node_modules/@mdream/js/dist/parse.d.mts

@@ -0,0 +1,90 @@
+import { ElementNode, Node, NodeEvent, TagHandler, TransformPlugin } from "./_chunks/types.mjs";
+/**
+ * Commit end-of-input state: flush trailing buffered text and close any open
+ * elements. The streaming parser keeps trailing text and unclosed elements
+ * pending (a later chunk might continue them); at true EOF they must be
+ * committed so trailing content is not dropped (e.g. `<p>a<p>b`).
+ *
+ * `leftover` is the residual returned by the final `parseHtmlStream`. Trailing
+ * text is emitted; a residual that is an incomplete start tag is dropped,
+ * matching the browser tokenizer's EOF-in-tag behaviour. A leading `<` alone
+ * does not make it a tag: `<3` is text. The text-buffer flags set while the
+ * trailing text was scanned persist on `state`, so `processTextBuffer` commits
+ * it as if the next tag had triggered the flush.
+ */
+export declare function finalizeParse(leftover: string, state: ParseState, handleEvent: (event: NodeEvent) => void): void;
+export interface ParseOptions {
+  resolvedPlugins?: TransformPlugin[];
+}
+export interface ParseState {
+  /** Map of tag names to their current nesting depth - uses TypedArray for performance */
+  depthMap: Uint16Array;
+  /** Current overall nesting depth */
+  depth: number;
+  /** Root tag currently flattened beyond the rich element depth. */
+  flattenedTagName?: string;
+  /** Resolved built-in identity of the flattened root. */
+  flattenedTagId?: number;
+  /** Same-name nesting below the flattened root. */
+  flattenedTagDepth?: number;
+  /** Exact output-excluded scope nested inside the flattened subtree. */
+  flattenedOpaqueTagName?: string;
+  flattenedOpaqueDepth?: number;
+  /** Raw-text tag currently open inside the flattened subtree. */
+  flattenedRawTagName?: string;
+  /** Whether flattened raw text must be excluded from output. */
+  flattenedRawExcludesText?: boolean;
+  /** Currently processing element node */
+  currentNode?: ElementNode | null;
+  /** Whether current content contains HTML entities that need decoding */
+  hasEncodedHtmlEntity?: boolean;
+  /** Whether the last processed character was whitespace - for collapsing whitespace */
+  lastCharWasWhitespace?: boolean;
+  /** Whether the last processed buffer has whitespace - optimization flag */
+  textBufferContainsWhitespace?: boolean;
+  /** Whether the last processed buffer contains non-whitespace characters */
+  textBufferContainsNonWhitespace?: boolean;
+  /** Whether a tag was just closed - affects whitespace handling */
+  justClosedTag?: boolean;
+  /** Whether the next text node is the first in its element - for whitespace trimming */
+  isFirstTextInElement?: boolean;
+  /** Persisted HTML tokenizer state while streaming a built-in script body. */
+  scriptDataState?: number;
+  /** Consumed script chunks retained for extraction/plugin text events. */
+  scriptTextChunks?: string[];
+  /** Reference to the last processed text node - for context tracking */
+  lastTextNode?: Node;
+  /** @deprecated No longer read or written. Retained for source compatibility. */
+  inSingleQuote?: boolean;
+  /** @deprecated No longer read or written. Retained for source compatibility. */
+  inDoubleQuote?: boolean;
+  /** @deprecated No longer read or written. Retained for source compatibility. */
+  inBacktick?: boolean;
+  /** @deprecated Rawtext parsing is no longer quote-aware. Retained for source compatibility. */
+  inRawTextQuoteAware?: boolean;
+  /** @deprecated No longer read or written. Retained for source compatibility. */
+  lastCharWasBackslash?: boolean;
+  /** Resolved plugin instances for event processing */
+  resolvedPlugins?: TransformPlugin[];
+  /** Tag override handlers built from declarative tagOverrides config */
+  tagOverrideHandlers?: Map<string, TagHandler>;
+  /** Whether emitted text should skip Markdown-only escaping */
+  plainText?: boolean;
+}
+export interface ParseResult {
+  events: NodeEvent[];
+  remainingHtml: string;
+}
+/**
+ * Pure HTML parser that emits DOM events
+ * Completely decoupled from markdown generation
+ */
+export declare function parseHtml(html: string, options?: ParseOptions): ParseResult;
+/**
+ * Streaming HTML parser - calls onEvent for each DOM event
+ */
+export declare function parseHtmlStream(html: string, state: ParseState, onEvent: (event: NodeEvent) => void): string;
+/**
+ * Parse HTML attributes string into key-value object
+ */
+export declare function parseAttributes(attrStr: string): Record<string, string>;

package/node_modules/@mdream/js/dist/parse.mjs

@@ -0,0 +1,2 @@
+import { finalizeParse, parseAttributes, parseHtml, parseHtmlStream } from "./_chunks/parse.mjs";
+export { finalizeParse, parseAttributes, parseHtml, parseHtmlStream };

package/node_modules/@mdream/js/dist/plugins.d.mts

@@ -0,0 +1,85 @@
+import { ElementNode, ExtractedElement as ExtractedElement$1, MdreamRuntimeState, TransformPlugin } from "./_chunks/types.mjs";
+import { createPlugin } from "./_chunks/index.mjs";
+interface ExtractedElement extends ElementNode {
+  textContent: string;
+}
+type ExtractionCallback = (element: ExtractedElement, state: MdreamRuntimeState) => void;
+/** Extract matching elements through the composable JavaScript plugin interface. */
+export declare function extractionPlugin(selectors: Record<string, ExtractionCallback>): TransformPlugin;
+/**
+ * Extraction collector for `plugins.extraction` config.
+ * Collects results during processing; callbacks are called post-conversion
+ * to match Rust engine behavior.
+ */
+export declare function extractionCollectorPlugin(config: Record<string, (element: ExtractedElement$1) => void>): {
+  plugin: TransformPlugin;
+  getResults: () => ExtractedElement$1[];
+  callHandlers: () => void;
+};
+/**
+ * Plugin that filters nodes based on CSS selectors.
+ * Allows including or excluding nodes based on selectors.
+ *
+ * @example
+ * // Include only heading elements and their children
+ * withQuerySelectorPlugin({ include: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'] })
+ *
+ * @example
+ * // Exclude navigation, sidebar, and footer
+ * withQuerySelectorPlugin({ exclude: ['nav', '#sidebar', '.footer'] })
+ */
+export declare function filterPlugin(options?: {
+  /** CSS selectors, tag names, or TAG_* constants for elements to include (all others will be excluded) */
+  include?: (string | number)[];
+  /** CSS selectors, tag names, or TAG_* constants for elements to exclude */
+  exclude?: (string | number)[];
+  /** Whether to also process the children of matching elements */
+  processChildren?: boolean;
+  keepAbsolute?: boolean;
+}): TransformPlugin;
+interface FrontmatterPluginOptions {
+  /** Additional frontmatter fields to include */
+  additionalFields?: Record<string, string>;
+  /** Meta tag names to extract (beyond the standard ones) */
+  metaFields?: string[];
+}
+/**
+ * A plugin that manages frontmatter generation from HTML head elements
+ * Extracts metadata from meta tags and title and generates YAML frontmatter
+ */
+export declare function frontmatterPlugin(options?: FrontmatterPluginOptions): TransformPlugin;
+/**
+ * Plugin that isolates main content using the following priority order:
+ * 1. If an explicit <main> element exists (within 5 depth levels), use its content exclusively
+ * 2. Otherwise, find content between the first header tag (h1-h6) and first footer
+ * 3. If footer is within 5 levels of nesting from the header, use it as the end boundary
+ * 4. Exclude all content before the start marker and after the end marker
+ *
+ * @example
+ * ```html
+ * <body>
+ *   <nav>Navigation (excluded)</nav>
+ *   <main>
+ *     <h1>Main Title (included)</h1>
+ *     <p>Main content (included)</p>
+ *   </main>
+ *   <footer>Footer (excluded)</footer>
+ * </body>
+ * ```
+ *
+ * @example
+ * ```html
+ * <body>
+ *   <nav>Navigation (excluded)</nav>
+ *   <h1>Main Title (included)</h1>
+ *   <p>Main content (included)</p>
+ *   <footer>Footer (excluded)</footer>
+ * </body>
+ * ```
+ */
+export declare function isolateMainPlugin(): TransformPlugin;
+/**
+ * Creates a plugin that adds Tailwind class processing
+ */
+export declare function tailwindPlugin(): TransformPlugin;
+export { type ExtractedElement, createPlugin };

package/node_modules/@mdream/js/dist/plugins.mjs

@@ -0,0 +1,2 @@
+import { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./_chunks/plugins.mjs";
+export { createPlugin, extractionCollectorPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };

package/node_modules/@mdream/js/dist/preset/minimal.d.mts

@@ -0,0 +1,2 @@
+import { withMinimalPreset } from "../_chunks/minimal.mjs";
+export { withMinimalPreset };

package/node_modules/@mdream/js/dist/preset/minimal.mjs

@@ -0,0 +1,33 @@
+import "../_chunks/const.mjs";
+const MINIMAL_EXCLUDE = [
+	40,
+	68,
+	103,
+	58,
+	47,
+	88,
+	73,
+	59,
+	66,
+	65,
+	43,
+	41
+];
+function withMinimalPreset(options = {}) {
+	const filter = options.plugins?.filter;
+	return {
+		clean: options.clean !== void 0 ? options.clean : true,
+		...options,
+		plugins: {
+			frontmatter: true,
+			isolateMain: true,
+			tailwind: true,
+			...options.plugins,
+			filter: filter === false ? false : {
+				...filter,
+				exclude: [...MINIMAL_EXCLUDE, ...filter?.exclude ?? []]
+			}
+		}
+	};
+}
+export { withMinimalPreset };

package/node_modules/@mdream/js/dist/splitter.d.mts

@@ -0,0 +1,18 @@
+import { MarkdownChunk, SplitterOptions } from "./_chunks/types.mjs";
+/**
+ * Convert HTML to Markdown and split into chunks in single pass.
+ * Yields chunks during HTML event processing for better memory efficiency.
+ *
+ * **JavaScript engine only** — uses the JS engine's internal processing pipeline.
+ * Not compatible with the Rust engine.
+ */
+export declare function htmlToMarkdownSplitChunksStream(html: string, options?: SplitterOptions): Generator<MarkdownChunk, void, undefined>;
+/**
+ * Convert HTML to Markdown and split into chunks in single pass.
+ * Chunks are created during HTML event processing.
+ *
+ * **JavaScript engine only** — uses the JS engine's internal processing pipeline.
+ * Not compatible with the Rust engine.
+ */
+export declare function htmlToMarkdownSplitChunks(html: string, options?: SplitterOptions): MarkdownChunk[];
+export type { MarkdownChunk, SplitterOptions };

package/node_modules/@mdream/js/dist/splitter.mjs

@@ -0,0 +1,194 @@
+import "./_chunks/const.mjs";
+import { parseHtmlStream } from "./_chunks/parse.mjs";
+import { createMarkdownProcessor, processPluginsForEvent } from "./_chunks/markdown-processor.mjs";
+import { resolvePlugins } from "./_chunks/resolve-plugins.mjs";
+const MARKDOWN_HEADER_LINE_RE = /^#{1,6}\s+/;
+const NEWLINE_RE = /\n/g;
+const DEFAULT_HEADERS_TO_SPLIT_ON = [
+	8,
+	9,
+	10,
+	11,
+	12
+];
+function createOptions(options) {
+	return {
+		headersToSplitOn: options.headersToSplitOn ?? DEFAULT_HEADERS_TO_SPLIT_ON,
+		returnEachLine: options.returnEachLine ?? false,
+		stripHeaders: options.stripHeaders ?? true,
+		chunkSize: options.chunkSize ?? 1e3,
+		chunkOverlap: options.chunkOverlap ?? 200,
+		lengthFunction: options.lengthFunction ?? ((text) => text.length),
+		keepSeparator: options.keepSeparator ?? false,
+		resolvedPlugins: resolvePlugins(options).plugins
+	};
+}
+function shouldSplitOnHeader(tagId, options) {
+	return options.headersToSplitOn.includes(tagId);
+}
+function getCurrentMarkdown(state) {
+	return state.buffer.join("").trimStart();
+}
+function* htmlToMarkdownSplitChunksStream(html, options = {}) {
+	const opts = createOptions(options);
+	if (opts.chunkOverlap >= opts.chunkSize) throw new Error("chunkOverlap must be less than chunkSize");
+	let currentChunkCodeLanguage = "";
+	const processor = createMarkdownProcessor(options, opts.resolvedPlugins);
+	processor.state.onCodeFenceOpen = (language) => {
+		if (language && !currentChunkCodeLanguage) currentChunkCodeLanguage = language;
+	};
+	const headerHierarchy = /* @__PURE__ */ new Map();
+	const seenSplitHeaders = /* @__PURE__ */ new Set();
+	let collectingHeaderText = false;
+	let currentHeaderTagId = null;
+	let currentHeaderText = "";
+	let lineNumber = 1;
+	let lastChunkEndPosition = 0;
+	let lastSplitPosition = 0;
+	function* flushChunk(endPosition, applyOverlap = false) {
+		const currentMd = getCurrentMarkdown(processor.state);
+		const chunkEnd = endPosition ?? currentMd.length;
+		const originalChunkContent = currentMd.slice(lastChunkEndPosition, chunkEnd);
+		if (!originalChunkContent.trim()) {
+			lastChunkEndPosition = chunkEnd;
+			return;
+		}
+		let chunkContent = originalChunkContent;
+		if (opts.stripHeaders) {
+			chunkContent = chunkContent.split("\n").filter((line) => !MARKDOWN_HEADER_LINE_RE.test(line)).join("\n").trim();
+			if (!chunkContent) {
+				lastChunkEndPosition = chunkEnd;
+				return;
+			}
+		}
+		const chunk = {
+			content: chunkContent.trimEnd(),
+			metadata: { loc: { lines: {
+				from: lineNumber,
+				to: lineNumber + (originalChunkContent.match(NEWLINE_RE) || []).length
+			} } }
+		};
+		if (headerHierarchy.size > 0) {
+			chunk.metadata.headers = {};
+			for (const [tagId, text] of headerHierarchy.entries()) {
+				const level = `h${tagId - 7 + 1}`;
+				chunk.metadata.headers[level] = text;
+			}
+		}
+		if (currentChunkCodeLanguage) chunk.metadata.code = currentChunkCodeLanguage;
+		yield chunk;
+		currentChunkCodeLanguage = "";
+		lastSplitPosition = chunkEnd;
+		if (applyOverlap && opts.chunkOverlap > 0) {
+			const maxOverlap = Math.max(0, originalChunkContent.length - 1);
+			lastChunkEndPosition = chunkEnd - Math.min(opts.chunkOverlap, maxOverlap);
+		} else lastChunkEndPosition = chunkEnd;
+		lineNumber += (originalChunkContent.match(NEWLINE_RE) || []).length;
+	}
+	const parseState = {
+		depthMap: processor.state.depthMap,
+		depth: 0,
+		resolvedPlugins: opts.resolvedPlugins
+	};
+	const eventBuffer = [];
+	const processResolvedEvent = opts.resolvedPlugins?.length ? (event) => processPluginsForEvent(event, opts.resolvedPlugins, processor.state, processor.processEvent) : processor.processEvent;
+	parseHtmlStream(html, parseState, (event) => {
+		eventBuffer.push(event);
+	});
+	for (const event of eventBuffer) {
+		const { type: eventType, node } = event;
+		const eventDepthMap = node.type === 1 ? node.depthMap : node.parent?.depthMap;
+		if (eventDepthMap) processor.state.depthMap.set(eventDepthMap);
+		else processor.state.depthMap.fill(0);
+		if (eventType === 1 && node.type === 1) {
+			const tagId = node.tagId;
+			if (tagId !== void 0 && tagId >= 0 && tagId < processor.state.depthMap.length) processor.state.depthMap[tagId] = Math.max(0, processor.state.depthMap[tagId] - 1);
+		}
+		if (node.type === 1) {
+			const tagId = node.tagId;
+			if (tagId && tagId >= 7 && tagId <= 12) {
+				if (eventType === 0) {
+					collectingHeaderText = true;
+					currentHeaderTagId = tagId;
+					currentHeaderText = "";
+					if (shouldSplitOnHeader(tagId, opts)) {
+						if (seenSplitHeaders.has(tagId)) {
+							yield* flushChunk();
+							for (let i = tagId; i <= 12; i++) headerHierarchy.delete(i);
+						}
+						seenSplitHeaders.add(tagId);
+					}
+				} else if (eventType === 1 && currentHeaderTagId === tagId) {
+					headerHierarchy.set(tagId, currentHeaderText.trim());
+					collectingHeaderText = false;
+					currentHeaderTagId = null;
+				}
+			}
+			if (tagId === 13 && eventType === 0) yield* flushChunk();
+		}
+		if (collectingHeaderText && node.type === 2) currentHeaderText += node.value;
+		processResolvedEvent(event);
+		if (!opts.returnEachLine) {
+			const currentMd = getCurrentMarkdown(processor.state);
+			if (opts.lengthFunction(currentMd.slice(lastChunkEndPosition)) > opts.chunkSize) {
+				const idealSplitPos = lastChunkEndPosition + opts.chunkSize;
+				const separators = [
+					"\n\n",
+					"```\n",
+					"\n",
+					" "
+				];
+				let splitPosition = -1;
+				for (const sep of separators) {
+					const idx = currentMd.lastIndexOf(sep, idealSplitPos);
+					const candidateSplitPos = idx + sep.length;
+					if (idx >= 0) {
+						const beforeSplit = currentMd.slice(0, candidateSplitPos);
+						let backtickCount = 0;
+						let pos = beforeSplit.indexOf("```", 0);
+						while (pos !== -1) {
+							backtickCount++;
+							pos = beforeSplit.indexOf("```", pos + 3);
+						}
+						if (backtickCount % 2 === 1) continue;
+					}
+					if (idx >= 0 && candidateSplitPos > lastSplitPosition) {
+						splitPosition = candidateSplitPos;
+						break;
+					}
+				}
+				if (splitPosition === -1 || splitPosition <= lastChunkEndPosition) splitPosition = currentMd.length;
+				yield* flushChunk(splitPosition, true);
+			}
+		}
+	}
+	yield* flushChunk();
+}
+function htmlToMarkdownSplitChunks(html, options = {}) {
+	const opts = createOptions(options);
+	const chunks = [];
+	for (const chunk of htmlToMarkdownSplitChunksStream(html, options)) chunks.push(chunk);
+	if (opts.returnEachLine && chunks.length > 0) {
+		const lineChunks = [];
+		for (const chunk of chunks) {
+			const lines = chunk.content.split("\n");
+			const chunkStartLine = chunk.metadata.loc?.lines.from || 1;
+			for (let i = 0; i < lines.length; i++) {
+				const line = lines[i];
+				if (line && line.trim()) lineChunks.push({
+					content: line,
+					metadata: {
+						...chunk.metadata,
+						loc: { lines: {
+							from: chunkStartLine + i,
+							to: chunkStartLine + i
+						} }
+					}
+				});
+			}
+		}
+		return lineChunks;
+	}
+	return chunks;
+}
+export { htmlToMarkdownSplitChunks, htmlToMarkdownSplitChunksStream };

package/node_modules/@mdream/js/package.json

@@ -0,0 +1,103 @@
+{
+  "name": "@mdream/js",
+  "type": "module",
+  "version": "2.0.0-beta.1",
+  "description": "JavaScript HTML-to-Markdown engine for mdream. Escape hatch for hooks and edge runtimes.",
+  "author": {
+    "name": "Harlan Wilton",
+    "email": "harlan@harlanzw.com",
+    "url": "https://harlanzw.com/"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/harlan-zw/mdream",
+    "directory": "packages/js"
+  },
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": {
+        "types": "./dist/index.d.mts",
+        "default": "./dist/index.mjs"
+      },
+      "default": "./dist/index.mjs"
+    },
+    "./core": {
+      "types": "./dist/core.d.mts",
+      "import": {
+        "types": "./dist/core.d.mts",
+        "default": "./dist/core.mjs"
+      },
+      "default": "./dist/core.mjs"
+    },
+    "./negotiate": {
+      "types": "./dist/negotiate.d.mts",
+      "import": {
+        "types": "./dist/negotiate.d.mts",
+        "default": "./dist/negotiate.mjs"
+      },
+      "default": "./dist/negotiate.mjs"
+    },
+    "./plugins": {
+      "types": "./dist/plugins.d.mts",
+      "import": {
+        "types": "./dist/plugins.d.mts",
+        "default": "./dist/plugins.mjs"
+      },
+      "default": "./dist/plugins.mjs"
+    },
+    "./preset/minimal": {
+      "types": "./dist/preset/minimal.d.mts",
+      "import": {
+        "types": "./dist/preset/minimal.d.mts",
+        "default": "./dist/preset/minimal.mjs"
+      },
+      "default": "./dist/preset/minimal.mjs"
+    },
+    "./splitter": {
+      "types": "./dist/splitter.d.mts",
+      "import": {
+        "types": "./dist/splitter.d.mts",
+        "default": "./dist/splitter.mjs"
+      },
+      "default": "./dist/splitter.mjs"
+    },
+    "./parse": {
+      "types": "./dist/parse.d.mts",
+      "import": {
+        "types": "./dist/parse.d.mts",
+        "default": "./dist/parse.mjs"
+      },
+      "default": "./dist/parse.mjs"
+    },
+    "./llms-txt": {
+      "types": "./dist/llms-txt.d.mts",
+      "import": {
+        "types": "./dist/llms-txt.d.mts",
+        "default": "./dist/llms-txt.mjs"
+      },
+      "default": "./dist/llms-txt.mjs"
+    }
+  },
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "bin": {
+    "mdream-js": "./bin/mdream.mjs"
+  },
+  "files": [
+    "bin",
+    "dist"
+  ],
+  "dependencies": {
+    "cac": "^7.0.0",
+    "pathe": "^2.0.3"
+  },
+  "scripts": {
+    "build": "obuild",
+    "typecheck": "tsc --noEmit",
+    "dev:prepare": "obuild --stub",
+    "test:attw": "attw --pack --profile esm-only"
+  }
+}

package/node_modules/cac/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) EGOIST <0x142857@gmail.com> (https://github.com/egoist)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.