mdream 0.15.3 → 0.17.0

package/dist/index.d.mts

@@ -1,9 +1,8 @@
-import { _ as TagHandler, a as HandlerContext, b as ExtractedElement, c as MdreamRuntimeState, d as Plugin, f as PluginContext, g as TEXT_NODE, h as SplitterOptions, i as HTMLToMarkdownOptions, l as Node, m as ReadabilityContext, n as ELEMENT_NODE, o as MarkdownChunk, p as PluginCreationOptions, r as ElementNode, s as MdreamProcessingState, t as BufferRegion, u as NodeEvent, v as TailwindContext, y as TextNode } from "./_chunks/types.mjs";
+import { _ as TailwindContext, a as MarkdownChunk, c as Node, d as PluginContext, f as PluginCreationOptions, g as TagHandler, h as TEXT_NODE, i as HandlerContext, l as NodeEvent, m as SplitterOptions, n as ElementNode, o as MdreamProcessingState, p as ReadabilityContext, r as HTMLToMarkdownOptions, s as MdreamRuntimeState, t as ELEMENT_NODE, u as Plugin, v as TextNode, y as ExtractedElement } from "./_chunks/types.mjs";
 import { t as createPlugin } from "./_chunks/plugin.mjs";
 import { ReadableStream } from "node:stream/web";
 
 //#region src/const.d.ts
-
 declare const TagIdMap: {
   readonly html: 0;
   readonly head: 1;
@@ -119,10 +118,8 @@ declare const TagIdMap: {
 interface MarkdownState {
   /** Configuration options for conversion */
   options?: HTMLToMarkdownOptions;
-  /** Map of region IDs to buffer regions for O(1) lookups */
-  regionToggles: Map<number, boolean>;
-  /** Content buffers for regions */
-  regionContentBuffers: Map<number, string[]>;
+  /** Content buffer for markdown output */
+  buffer: string[];
   /** Performance cache for last content to avoid iteration */
   lastContentCache?: string;
   /** Reference to the last processed node */
@@ -178,4 +175,4 @@ declare function streamHtmlToMarkdown(htmlStream: ReadableStream | null, options
 //#region src/index.d.ts
 declare function htmlToMarkdown(html: string, options?: HTMLToMarkdownOptions): string;
 //#endregion
-export { BufferRegion, ELEMENT_NODE, ElementNode, ExtractedElement, HTMLToMarkdownOptions, HandlerContext, MarkdownChunk, MarkdownProcessor, MdreamProcessingState, MdreamRuntimeState, Node, NodeEvent, Plugin, PluginContext, PluginCreationOptions, ReadabilityContext, SplitterOptions, TEXT_NODE, TagHandler, TagIdMap, TailwindContext, TextNode, createPlugin, htmlToMarkdown, parseHtml, streamHtmlToMarkdown };
+export { ELEMENT_NODE, ElementNode, ExtractedElement, HTMLToMarkdownOptions, HandlerContext, MarkdownChunk, MarkdownProcessor, MdreamProcessingState, MdreamRuntimeState, Node, NodeEvent, Plugin, PluginContext, PluginCreationOptions, ReadabilityContext, SplitterOptions, TEXT_NODE, TagHandler, TagIdMap, TailwindContext, TextNode, createPlugin, htmlToMarkdown, parseHtml, streamHtmlToMarkdown };

package/dist/index.mjs

@@ -1,6 +1,5 @@
-import { _n as TagIdMap } from "./_chunks/const.mjs";
+import { s as TagIdMap } from "./_chunks/const.mjs";
 import { i as parseHtml, t as MarkdownProcessor } from "./_chunks/markdown-processor.mjs";
 import { t as createPlugin } from "./_chunks/plugin.mjs";
 import { n as streamHtmlToMarkdown, t as htmlToMarkdown } from "./_chunks/src.mjs";
-
-export { MarkdownProcessor, TagIdMap, createPlugin, htmlToMarkdown, parseHtml, streamHtmlToMarkdown };
+export { MarkdownProcessor, TagIdMap, createPlugin, htmlToMarkdown, parseHtml, streamHtmlToMarkdown };

package/dist/llms-txt.mjs

@@ -1,5 +1,469 @@
+import "./_chunks/const.mjs";
 import "./_chunks/markdown-processor.mjs";
-import "./_chunks/src.mjs";
-import { n as generateLlmsTxtArtifacts, t as createLlmsTxtStream } from "./_chunks/llms-txt.mjs";
-
-export { createLlmsTxtStream, generateLlmsTxtArtifacts };
+import "./_chunks/plugin.mjs";
+import { t as htmlToMarkdown } from "./_chunks/src.mjs";
+import { t as extractionPlugin } from "./_chunks/extraction.mjs";
+import { mkdir, open, readFile } from "node:fs/promises";
+import { basename, dirname, join, relative, sep } from "pathe";
+import { glob } from "tinyglobby";
+//#region src/llms-txt.ts
+const FRONTMATTER_RE = /^---\n([\s\S]*?)\n---\n([\s\S]*)$/;
+const ANCHOR_INVALID_CHARS_RE = /[^a-z0-9]/g;
+const LEADING_SLASH_RE = /^\//;
+const TRAILING_SLASH_RE = /\/$/;
+/**
+* Extract metadata from HTML content using mdream's extraction plugin
+*/
+function extractMetadata(html, url) {
+	let title = "";
+	let description = "";
+	let keywords = "";
+	let author = "";
+	htmlToMarkdown(html, {
+		plugins: [extractionPlugin({
+			"title": (element) => {
+				if (!title && element.textContent) title = element.textContent.trim();
+			},
+			"meta[name=\"description\"]": (element) => {
+				if (!description && element.attributes?.content) description = element.attributes.content.trim();
+			},
+			"meta[property=\"og:description\"]": (element) => {
+				if (!description && element.attributes?.content) description = element.attributes.content.trim();
+			},
+			"meta[name=\"keywords\"]": (element) => {
+				if (!keywords && element.attributes?.content) keywords = element.attributes.content.trim();
+			},
+			"meta[name=\"author\"]": (element) => {
+				if (!author && element.attributes?.content) author = element.attributes.content.trim();
+			},
+			"meta[property=\"og:title\"]": (element) => {
+				if (!title && element.attributes?.content) title = element.attributes.content.trim();
+			}
+		})],
+		origin: url
+	});
+	return {
+		title: title || void 0,
+		description: description || void 0,
+		keywords: keywords || void 0,
+		author: author || void 0
+	};
+}
+/**
+* Convert file path to URL path
+*/
+function pathToUrl(filePath, baseDir) {
+	let url = relative(baseDir, filePath);
+	url = url.split(sep).join("/");
+	if (url.endsWith(".html")) url = url.slice(0, -5);
+	if (url.endsWith("/index")) url = url.slice(0, -6);
+	if (url === "index") return "/";
+	if (!url.startsWith("/")) url = `/${url}`;
+	return url;
+}
+/**
+* Process HTML files from glob patterns
+*/
+async function processHtmlFiles(patterns, origin) {
+	const allPatterns = Array.isArray(patterns) ? patterns : [patterns];
+	const allFiles = [];
+	for (const pattern of allPatterns) {
+		const files = await glob(pattern);
+		allFiles.push(...files);
+	}
+	const uniqueFiles = [...new Set(allFiles)];
+	const results = [];
+	const baseDir = uniqueFiles.length > 0 ? dirname(uniqueFiles[0]) : ".";
+	for (const filePath of uniqueFiles) try {
+		const html = await readFile(filePath, "utf-8");
+		const metadata = extractMetadata(html, origin || filePath);
+		const content = htmlToMarkdown(html, { origin });
+		const url = pathToUrl(filePath, baseDir);
+		results.push({
+			filePath,
+			title: metadata?.title || basename(filePath, ".html"),
+			content,
+			url,
+			metadata
+		});
+	} catch (error) {
+		console.error(`Error processing ${filePath}:`, error);
+	}
+	return results;
+}
+/**
+* Generate llms.txt content
+*/
+function generateLlmsTxtContent(files, options) {
+	const { siteName = "Site", description, origin = "", sections, notes } = options;
+	let content = `# ${siteName}\n\n`;
+	if (description) content += `> ${description}\n\n`;
+	if (origin) content += `Canonical Origin: ${origin}\n\n`;
+	if (sections) for (const section of sections) content += formatSection(section);
+	if (files.length > 0) {
+		content += `## Pages\n\n`;
+		for (const file of files) {
+			const desc = file.metadata?.description;
+			const descText = desc ? `: ${desc.substring(0, 100)}${desc.length > 100 ? "..." : ""}` : "";
+			if (file.filePath && options.outputDir && file.filePath.endsWith(".md")) {
+				const relativePath = relative(options.outputDir, file.filePath);
+				content += `- [${file.title}](${relativePath})${descText}\n`;
+			} else {
+				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
+				content += `- [${file.title}](${url})${descText}\n`;
+			}
+		}
+	}
+	if (notes) content += `\n${formatNotes(notes)}`;
+	return content;
+}
+/**
+* Parse frontmatter from markdown content
+*/
+function parseFrontmatter(content) {
+	const match = content.match(FRONTMATTER_RE);
+	if (!match) return {
+		frontmatter: null,
+		body: content
+	};
+	const frontmatterContent = match[1];
+	const body = match[2];
+	const frontmatter = {};
+	const lines = frontmatterContent.split("\n");
+	for (const line of lines) {
+		const colonIndex = line.indexOf(":");
+		if (colonIndex > 0) {
+			const key = line.substring(0, colonIndex).trim();
+			frontmatter[key] = line.substring(colonIndex + 1).trim();
+		}
+	}
+	return {
+		frontmatter,
+		body
+	};
+}
+/**
+* Serialize frontmatter object to YAML-like format
+*/
+function serializeFrontmatter(data) {
+	const lines = [];
+	for (const [key, value] of Object.entries(data)) if (value !== void 0 && value !== null) lines.push(`${key}: ${String(value)}`);
+	return lines.join("\n");
+}
+/**
+* Generate llms-full.txt content with complete page content
+*/
+function generateLlmsFullTxtContent(files, options) {
+	const { siteName = "Site", description, origin = "", sections, notes } = options;
+	let content = `# ${siteName}\n\n`;
+	if (description) content += `> ${description}\n\n`;
+	if (origin) content += `Canonical Origin: ${origin}\n\n`;
+	if (sections) for (const section of sections) content += formatSection(section);
+	if (files.length > 0) {
+		content += `## Table of Contents\n\n`;
+		for (const file of files) {
+			const anchor = file.title.toLowerCase().replace(ANCHOR_INVALID_CHARS_RE, "-");
+			content += `- [${file.title}](#${anchor})\n`;
+		}
+		content += `\n---\n\n`;
+		for (const file of files) {
+			const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
+			const { frontmatter, body } = parseFrontmatter(file.content);
+			const metadata = {
+				title: file.title,
+				url
+			};
+			if (file.filePath && options.outputDir) metadata.file = relative(options.outputDir, file.filePath);
+			else if (file.filePath) metadata.file = file.filePath;
+			if (file.metadata) {
+				if (file.metadata.description) metadata.description = file.metadata.description;
+				if (file.metadata.keywords) metadata.keywords = file.metadata.keywords;
+				if (file.metadata.author) metadata.author = file.metadata.author;
+			}
+			const frontmatterString = serializeFrontmatter(frontmatter ? {
+				...frontmatter,
+				...metadata
+			} : metadata);
+			let contentBody = frontmatter ? body : file.content;
+			const titleLine = contentBody.trim().split("\n")[0];
+			if (titleLine === file.title || titleLine === `# ${file.title}`) contentBody = contentBody.trim().split("\n").slice(1).join("\n").trimStart();
+			content += `---\n${frontmatterString}\n---\n\n${contentBody}\n\n---\n\n`;
+		}
+	}
+	if (notes) content += `\n${formatNotes(notes)}`;
+	return content;
+}
+/**
+* Generate individual markdown files structure
+*/
+function generateMarkdownFilesContent(files) {
+	const markdownFiles = [];
+	for (const file of files) {
+		const mdPath = `md/${file.url === "/" ? "index" : file.url.replace(LEADING_SLASH_RE, "").replace(TRAILING_SLASH_RE, "")}.md`;
+		markdownFiles.push({
+			path: mdPath,
+			content: file.content
+		});
+	}
+	return markdownFiles;
+}
+/**
+* Main function to process files and generate llms.txt artifacts
+*/
+async function generateLlmsTxtArtifacts(options) {
+	let files;
+	if (options.files) files = options.files;
+	else if (options.patterns) files = await processHtmlFiles(options.patterns, options.origin);
+	else throw new Error("Either patterns or files must be provided");
+	const llmsTxt = generateLlmsTxtContent(files, options);
+	let llmsFullTxt;
+	if (options.generateFull) llmsFullTxt = generateLlmsFullTxtContent(files, options);
+	let markdownFiles;
+	if (options.generateMarkdown) markdownFiles = generateMarkdownFilesContent(files);
+	return {
+		llmsTxt,
+		llmsFullTxt,
+		markdownFiles,
+		processedFiles: files
+	};
+}
+/**
+* Format a section with title, description, and links
+*/
+function formatSection(section) {
+	let content = `## ${section.title}\n\n`;
+	if (section.description) {
+		const descriptions = Array.isArray(section.description) ? section.description : [section.description];
+		for (const desc of descriptions) content += `${desc}\n\n`;
+	}
+	if (section.links?.length) {
+		for (const link of section.links) {
+			const desc = link.description ? `: ${link.description}` : "";
+			content += `- [${link.title}](${link.href})${desc}\n`;
+		}
+		content += "\n";
+	}
+	return content;
+}
+/**
+* Format notes section
+*/
+function formatNotes(notes) {
+	const noteLines = Array.isArray(notes) ? notes : [notes];
+	let content = "";
+	for (const note of noteLines) content += `${note}\n\n`;
+	return content;
+}
+/**
+* Create a WritableStream that generates llms.txt artifacts by streaming pages to disk
+*
+* Writes llms.txt (and optionally llms-full.txt) incrementally as pages are written,
+* never keeping full content in memory. Creates outputDir recursively if needed.
+*
+* @example
+* ```typescript
+* const stream = createLlmsTxtStream({
+*   siteName: 'My Docs',
+*   description: 'Documentation site',
+*   origin: 'https://example.com',
+*   generateFull: true,
+*   outputDir: './dist',
+*   sections: [
+*     {
+*       title: 'Getting Started',
+*       description: 'Quick start guide',
+*       links: [
+*         { title: 'Installation', href: '/install', description: 'How to install' },
+*         { title: 'Quick Start', href: '/quickstart' },
+*       ],
+*     },
+*   ],
+*   notes: ['Generated by mdream', 'Last updated: 2024'],
+* })
+*
+* const writer = stream.getWriter()
+* await writer.write({
+*   title: 'Home',
+*   content: '# Welcome\n\nHome page content.',
+*   url: '/',
+* })
+* await writer.close()
+* ```
+*
+* @param options - Configuration options
+* @returns WritableStream that accepts ProcessedFile objects
+*/
+/**
+* Get group prefix for a URL (up to 2 segments)
+*/
+function getGroupPrefix(url, depth) {
+	const segments = url.split("/").filter(Boolean);
+	if (segments.length === 0) return "/";
+	if (depth === 1 || segments.length === 1) return `/${segments[0]}`;
+	return `/${segments[0]}/${segments[1]}`;
+}
+/**
+* Sort pages by URL path in hierarchical order (directory tree structure)
+* Groups by up to 2 segments, with root-level pages without nesting grouped together
+*/
+function sortPagesByPath(pages) {
+	const twoSegmentCount = /* @__PURE__ */ new Map();
+	for (const page of pages) {
+		const prefix = getGroupPrefix(page.url, 2);
+		twoSegmentCount.set(prefix, (twoSegmentCount.get(prefix) || 0) + 1);
+	}
+	const segmentHasNested = /* @__PURE__ */ new Map();
+	for (const page of pages) {
+		const segments = page.url.split("/").filter(Boolean);
+		const firstSegment = segments.length > 0 ? segments[0] : "";
+		if (!segmentHasNested.has(firstSegment)) segmentHasNested.set(firstSegment, false);
+		if (segments.length > 1) segmentHasNested.set(firstSegment, true);
+	}
+	return pages.sort((a, b) => {
+		const segmentsA = a.url.split("/").filter(Boolean);
+		const segmentsB = b.url.split("/").filter(Boolean);
+		const firstSegmentA = segmentsA.length > 0 ? segmentsA[0] : "";
+		const firstSegmentB = segmentsB.length > 0 ? segmentsB[0] : "";
+		const twoSegPrefixA = getGroupPrefix(a.url, 2);
+		const twoSegPrefixB = getGroupPrefix(b.url, 2);
+		const twoSegCountA = twoSegmentCount.get(twoSegPrefixA) || 0;
+		const twoSegCountB = twoSegmentCount.get(twoSegPrefixB) || 0;
+		let groupKeyA = twoSegCountA > 1 ? twoSegPrefixA : `/${firstSegmentA}`;
+		let groupKeyB = twoSegCountB > 1 ? twoSegPrefixB : `/${firstSegmentB}`;
+		const isRootLevelA = segmentsA.length <= 1;
+		const isRootLevelB = segmentsB.length <= 1;
+		const hasNestedA = segmentHasNested.get(firstSegmentA);
+		const hasNestedB = segmentHasNested.get(firstSegmentB);
+		if (isRootLevelA && !hasNestedA) groupKeyA = "";
+		if (isRootLevelB && !hasNestedB) groupKeyB = "";
+		if (groupKeyA === "" && groupKeyB !== "") return -1;
+		if (groupKeyA !== "" && groupKeyB === "") return 1;
+		if (groupKeyA !== groupKeyB) return groupKeyA.localeCompare(groupKeyB);
+		if (segmentsA.length === 0) return -1;
+		if (segmentsB.length === 0) return 1;
+		const minLen = Math.min(segmentsA.length, segmentsB.length);
+		for (let i = 0; i < minLen; i++) {
+			const cmp = segmentsA[i].localeCompare(segmentsB[i]);
+			if (cmp !== 0) return cmp;
+		}
+		return segmentsA.length - segmentsB.length;
+	});
+}
+function createLlmsTxtStream(options = {}) {
+	const { siteName = "Site", description, origin = "", generateFull, outputDir = process.cwd(), sections, notes } = options;
+	let llmsTxtHandle;
+	let llmsFullTxtHandle;
+	const bufferedPages = [];
+	return new WritableStream({
+		async start() {
+			await mkdir(outputDir, { recursive: true });
+			llmsTxtHandle = await open(join(outputDir, "llms.txt"), "w");
+			let header = `# ${siteName}\n\n`;
+			if (description) header += `> ${description}\n\n`;
+			if (origin) header += `Canonical Origin: ${origin}\n\n`;
+			if (sections) for (const section of sections) header += formatSection(section);
+			await llmsTxtHandle.write(header);
+			if (generateFull) {
+				llmsFullTxtHandle = await open(join(outputDir, "llms-full.txt"), "w");
+				let fullHeader = `# ${siteName}\n\n`;
+				if (description) fullHeader += `> ${description}\n\n`;
+				if (origin) fullHeader += `Canonical Origin: ${origin}\n\n`;
+				if (sections) for (const section of sections) fullHeader += formatSection(section);
+				await llmsFullTxtHandle.write(fullHeader);
+			}
+		},
+		async write(file) {
+			const desc = file.metadata?.description;
+			bufferedPages.push({
+				url: file.url,
+				title: file.title,
+				description: desc,
+				filePath: file.filePath
+			});
+			if (generateFull && llmsFullTxtHandle) {
+				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
+				const { frontmatter, body } = parseFrontmatter(file.content);
+				const metadata = {
+					title: file.title,
+					url
+				};
+				if (file.filePath) metadata.file = relative(outputDir, file.filePath);
+				if (file.metadata) {
+					if (file.metadata.description) metadata.description = file.metadata.description;
+					if (file.metadata.keywords) metadata.keywords = file.metadata.keywords;
+					if (file.metadata.author) metadata.author = file.metadata.author;
+				}
+				const frontmatterString = serializeFrontmatter(frontmatter ? {
+					...frontmatter,
+					...metadata
+				} : metadata);
+				let contentBody = frontmatter ? body : file.content;
+				const titleLine = contentBody.trim().split("\n")[0];
+				if (titleLine === file.title || titleLine === `# ${file.title}`) contentBody = contentBody.trim().split("\n").slice(1).join("\n").trimStart();
+				const fullChunk = `---\n${frontmatterString}\n---\n\n${contentBody}\n\n---\n\n`;
+				await llmsFullTxtHandle.write(fullChunk);
+			}
+		},
+		async close() {
+			const sortedPages = sortPagesByPath(bufferedPages);
+			const twoSegmentCount = /* @__PURE__ */ new Map();
+			for (const page of sortedPages) {
+				const prefix = getGroupPrefix(page.url, 2);
+				twoSegmentCount.set(prefix, (twoSegmentCount.get(prefix) || 0) + 1);
+			}
+			const segmentHasNested = /* @__PURE__ */ new Map();
+			for (const page of sortedPages) {
+				const segments = page.url.split("/").filter(Boolean);
+				const firstSegment = segments.length > 0 ? segments[0] : "";
+				if (!segmentHasNested.has(firstSegment)) segmentHasNested.set(firstSegment, false);
+				if (segments.length > 1) segmentHasNested.set(firstSegment, true);
+			}
+			await llmsTxtHandle?.write(`## Pages\n\n`);
+			let currentGroup = "";
+			let segmentGroupIndex = 0;
+			let urlsInCurrentGroup = 0;
+			for (let i = 0; i < sortedPages.length; i++) {
+				const page = sortedPages[i];
+				const segments = page.url.split("/").filter(Boolean);
+				const firstSegment = segments.length > 0 ? segments[0] : "";
+				const twoSegPrefix = getGroupPrefix(page.url, 2);
+				let groupKey = (twoSegmentCount.get(twoSegPrefix) || 0) > 1 ? twoSegPrefix : `/${firstSegment}`;
+				const isRootLevel = segments.length <= 1;
+				const hasNested = segmentHasNested.get(firstSegment);
+				if (isRootLevel && !hasNested) groupKey = "";
+				if (groupKey !== currentGroup) {
+					if (urlsInCurrentGroup > 0) {
+						if (segmentGroupIndex === 0 || segmentGroupIndex >= 1 && segmentGroupIndex <= 2 && urlsInCurrentGroup > 1) await llmsTxtHandle?.write("\n");
+					}
+					currentGroup = groupKey;
+					segmentGroupIndex++;
+					urlsInCurrentGroup = 0;
+				}
+				urlsInCurrentGroup++;
+				const descText = page.description ? `: ${page.description.substring(0, 160)}${page.description.length > 160 ? "..." : ""}` : "";
+				let chunk = "";
+				if (page.filePath && page.filePath.endsWith(".md")) {
+					const relativePath = relative(outputDir, page.filePath);
+					chunk = `- [${page.title}](${relativePath})${descText}\n`;
+				} else {
+					const url = page.url.startsWith("http://") || page.url.startsWith("https://") ? page.url : origin ? origin + page.url : page.url;
+					chunk = `- [${page.title}](${url})${descText}\n`;
+				}
+				await llmsTxtHandle?.write(chunk);
+			}
+			if (notes) {
+				const notesContent = formatNotes(notes);
+				await llmsTxtHandle?.write(`\n${notesContent}`);
+				if (generateFull && llmsFullTxtHandle) await llmsFullTxtHandle.write(`\n${notesContent}`);
+			}
+			await llmsTxtHandle?.close();
+			await llmsFullTxtHandle?.close();
+		},
+		async abort(_reason) {
+			await llmsTxtHandle?.close();
+			await llmsFullTxtHandle?.close();
+		}
+	});
+}
+//#endregion
+export { createLlmsTxtStream, generateLlmsTxtArtifacts };

package/dist/negotiate.d.mts

@@ -0,0 +1,26 @@
+//#region src/negotiate.d.ts
+interface AcceptEntry {
+  type: string;
+  q: number;
+  position: number;
+}
+/**
+ * Parse an HTTP Accept header into an ordered list of media types with quality values.
+ * Supports quality weights (q=0.9) and preserves original position for tie-breaking.
+ */
+declare function parseAcceptHeader(accept: string): AcceptEntry[];
+/**
+ * Determine if a client prefers markdown over HTML using proper content negotiation.
+ *
+ * Uses Accept header quality weights and position ordering:
+ * - If text/markdown or text/plain has higher quality than text/html → markdown
+ * - If same quality, earlier position in Accept header wins
+ * - Bare wildcard does NOT trigger markdown (prevents breaking OG crawlers)
+ * - sec-fetch-dest: document always returns false (browser navigation)
+ *
+ * @param acceptHeader - The HTTP Accept header value
+ * @param secFetchDest - The Sec-Fetch-Dest header value
+ */
+declare function shouldServeMarkdown(acceptHeader?: string, secFetchDest?: string): boolean;
+//#endregion
+export { parseAcceptHeader, shouldServeMarkdown };

package/dist/negotiate.mjs

@@ -0,0 +1,92 @@
+//#region src/negotiate.ts
+/**
+* Parse an HTTP Accept header into an ordered list of media types with quality values.
+* Supports quality weights (q=0.9) and preserves original position for tie-breaking.
+*/
+function parseAcceptHeader(accept) {
+	if (!accept) return [];
+	const entries = [];
+	const parts = accept.split(",");
+	for (let i = 0; i < parts.length; i++) {
+		const part = parts[i].trim();
+		if (!part) continue;
+		const semicolonIdx = part.indexOf(";");
+		let type;
+		let q = 1;
+		if (semicolonIdx === -1) type = part;
+		else {
+			type = part.slice(0, semicolonIdx).trim();
+			const paramStr = part.slice(semicolonIdx + 1);
+			const qIdx = paramStr.indexOf("q=");
+			if (qIdx !== -1) {
+				const qStart = qIdx + 2;
+				let qEnd = qStart;
+				while (qEnd < paramStr.length && paramStr.charCodeAt(qEnd) !== 59 && paramStr.charCodeAt(qEnd) !== 32) qEnd++;
+				q = +paramStr.slice(qStart, qEnd) || 0;
+			}
+		}
+		entries.push({
+			type,
+			q,
+			position: i
+		});
+	}
+	return entries;
+}
+/**
+* Determine if a client prefers markdown over HTML using proper content negotiation.
+*
+* Uses Accept header quality weights and position ordering:
+* - If text/markdown or text/plain has higher quality than text/html → markdown
+* - If same quality, earlier position in Accept header wins
+* - Bare wildcard does NOT trigger markdown (prevents breaking OG crawlers)
+* - sec-fetch-dest: document always returns false (browser navigation)
+*
+* @param acceptHeader - The HTTP Accept header value
+* @param secFetchDest - The Sec-Fetch-Dest header value
+*/
+function shouldServeMarkdown(acceptHeader, secFetchDest) {
+	if (secFetchDest === "document") return false;
+	const accept = acceptHeader || "";
+	if (!accept) return false;
+	const parts = accept.split(",");
+	let bestMdQ = -1;
+	let bestMdPos = -1;
+	let htmlQ = -1;
+	let htmlPos = -1;
+	for (let i = 0; i < parts.length; i++) {
+		const part = parts[i].trim();
+		if (!part) continue;
+		const semicolonIdx = part.indexOf(";");
+		let type;
+		let q = 1;
+		if (semicolonIdx === -1) type = part;
+		else {
+			type = part.slice(0, semicolonIdx).trim();
+			const paramStr = part.slice(semicolonIdx + 1);
+			const qIdx = paramStr.indexOf("q=");
+			if (qIdx !== -1) {
+				const qStart = qIdx + 2;
+				let qEnd = qStart;
+				while (qEnd < paramStr.length && paramStr.charCodeAt(qEnd) !== 59 && paramStr.charCodeAt(qEnd) !== 32) qEnd++;
+				q = +paramStr.slice(qStart, qEnd) || 0;
+			}
+		}
+		if (type === "text/markdown" || type === "text/plain") {
+			if (q > bestMdQ || q === bestMdQ && (bestMdPos === -1 || i < bestMdPos)) {
+				bestMdQ = q;
+				bestMdPos = i;
+			}
+		} else if (type === "text/html") {
+			htmlQ = q;
+			htmlPos = i;
+		}
+	}
+	if (bestMdPos === -1) return false;
+	if (htmlPos === -1) return true;
+	if (bestMdQ > htmlQ) return true;
+	if (bestMdQ === htmlQ && bestMdPos < htmlPos) return true;
+	return false;
+}
+//#endregion
+export { parseAcceptHeader, shouldServeMarkdown };

package/dist/plugins.d.mts

@@ -1,8 +1,7 @@
-import { d as Plugin, x as extractionPlugin } from "./_chunks/types.mjs";
+import { b as extractionPlugin, u as Plugin } from "./_chunks/types.mjs";
 import { t as createPlugin } from "./_chunks/plugin.mjs";
 
 //#region src/plugins/filter.d.ts
-
 /**
  * Plugin that filters nodes based on CSS selectors.
  * Allows including or excluding nodes based on selectors.
@@ -16,11 +15,8 @@ import { t as createPlugin } from "./_chunks/plugin.mjs";
  * withQuerySelectorPlugin({ exclude: ['nav', '#sidebar', '.footer'] })
  */
 declare function filterPlugin(options?: {
-  /** CSS selectors (or Tag Ids) for elements to include (all others will be excluded) */
-  include?: (string | number)[];
-  /** CSS selectors (or Tag Ids) for elements to exclude */
-  exclude?: (string | number)[];
-  /** Whether to also process the children of matching elements */
+  /** CSS selectors (or Tag Ids) for elements to include (all others will be excluded) */include?: (string | number)[]; /** CSS selectors (or Tag Ids) for elements to exclude */
+  exclude?: (string | number)[]; /** Whether to also process the children of matching elements */
   processChildren?: boolean;
   keepAbsolute?: boolean;
 }): Plugin;
@@ -72,17 +68,10 @@ declare function frontmatterPlugin(options?: FrontmatterPluginOptions): Plugin;
  */
 declare function isolateMainPlugin(): Plugin;
 //#endregion
-//#region src/plugins/readability.d.ts
-/**
- * Creates a plugin that implements readability.js style heuristics for content quality assessment
- * Controls content inclusion/exclusion using buffer regions
- */
-declare function readabilityPlugin(): Plugin;
-//#endregion
 //#region src/plugins/tailwind.d.ts
 /**
  * Creates a plugin that adds Tailwind class processing
  */
 declare function tailwindPlugin(): Plugin;
 //#endregion
-export { createPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, readabilityPlugin, tailwindPlugin };
+export { createPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };

package/dist/plugins.mjs

@@ -1,5 +1,5 @@
+import "./_chunks/const.mjs";
 import { t as createPlugin } from "./_chunks/plugin.mjs";
 import { t as extractionPlugin } from "./_chunks/extraction.mjs";
-import { a as filterPlugin, i as frontmatterPlugin, n as readabilityPlugin, r as isolateMainPlugin, t as tailwindPlugin } from "./_chunks/plugins.mjs";
-
-export { createPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, readabilityPlugin, tailwindPlugin };
+import { i as filterPlugin, n as isolateMainPlugin, r as frontmatterPlugin, t as tailwindPlugin } from "./_chunks/plugins.mjs";
+export { createPlugin, extractionPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin };