mdream 0.15.2 → 0.16.0

package/dist/_chunks/llms-txt-Czb_M48B.mjs

@@ -1,440 +0,0 @@
-import { t as htmlToMarkdown } from "./src-BJpipdul.mjs";
-import { t as extractionPlugin } from "./extraction-BA9MDtq3.mjs";
-import { mkdir, open, readFile } from "node:fs/promises";
-import { basename, dirname, join, relative, sep } from "pathe";
-import { glob } from "tinyglobby";
-
-//#region src/llms-txt.ts
-/**
-* Extract metadata from HTML content using mdream's extraction plugin
-*/
-function extractMetadata(html, url) {
-	let title = "";
-	let description = "";
-	let keywords = "";
-	let author = "";
-	htmlToMarkdown(html, {
-		plugins: [extractionPlugin({
-			"title": (element) => {
-				if (!title && element.textContent) title = element.textContent.trim();
-			},
-			"meta[name=\"description\"]": (element) => {
-				if (!description && element.attributes?.content) description = element.attributes.content.trim();
-			},
-			"meta[property=\"og:description\"]": (element) => {
-				if (!description && element.attributes?.content) description = element.attributes.content.trim();
-			},
-			"meta[name=\"keywords\"]": (element) => {
-				if (!keywords && element.attributes?.content) keywords = element.attributes.content.trim();
-			},
-			"meta[name=\"author\"]": (element) => {
-				if (!author && element.attributes?.content) author = element.attributes.content.trim();
-			},
-			"meta[property=\"og:title\"]": (element) => {
-				if (!title && element.attributes?.content) title = element.attributes.content.trim();
-			}
-		})],
-		origin: url
-	});
-	return {
-		title: title || void 0,
-		description: description || void 0,
-		keywords: keywords || void 0,
-		author: author || void 0
-	};
-}
-/**
-* Convert file path to URL path
-*/
-function pathToUrl(filePath, baseDir) {
-	let url = relative(baseDir, filePath);
-	url = url.split(sep).join("/");
-	if (url.endsWith(".html")) url = url.slice(0, -5);
-	if (url.endsWith("/index")) url = url.slice(0, -6);
-	if (url === "index") return "/";
-	if (!url.startsWith("/")) url = `/${url}`;
-	return url;
-}
-/**
-* Process HTML files from glob patterns
-*/
-async function processHtmlFiles(patterns, origin) {
-	const allPatterns = Array.isArray(patterns) ? patterns : [patterns];
-	const allFiles = [];
-	for (const pattern of allPatterns) {
-		const files = await glob(pattern);
-		allFiles.push(...files);
-	}
-	const uniqueFiles = [...new Set(allFiles)];
-	const results = [];
-	const baseDir = uniqueFiles.length > 0 ? dirname(uniqueFiles[0]) : ".";
-	for (const filePath of uniqueFiles) try {
-		const html = await readFile(filePath, "utf-8");
-		const metadata = extractMetadata(html, origin || filePath);
-		const content = htmlToMarkdown(html, { origin });
-		const url = pathToUrl(filePath, baseDir);
-		results.push({
-			filePath,
-			title: metadata?.title || basename(filePath, ".html"),
-			content,
-			url,
-			metadata
-		});
-	} catch (error) {
-		console.error(`Error processing ${filePath}:`, error);
-	}
-	return results;
-}
-/**
-* Generate llms.txt content
-*/
-function generateLlmsTxtContent(files, options) {
-	const { siteName = "Site", description, origin = "", sections, notes } = options;
-	let content = `# ${siteName}\n\n`;
-	if (description) content += `> ${description}\n\n`;
-	if (origin) content += `Canonical Origin: ${origin}\n\n`;
-	if (sections) for (const section of sections) content += formatSection(section);
-	if (files.length > 0) {
-		content += `## Pages\n\n`;
-		for (const file of files) {
-			const desc = file.metadata?.description;
-			const descText = desc ? `: ${desc.substring(0, 100)}${desc.length > 100 ? "..." : ""}` : "";
-			if (file.filePath && options.outputDir && file.filePath.endsWith(".md")) {
-				const relativePath = relative(options.outputDir, file.filePath);
-				content += `- [${file.title}](${relativePath})${descText}\n`;
-			} else {
-				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
-				content += `- [${file.title}](${url})${descText}\n`;
-			}
-		}
-	}
-	if (notes) content += `\n${formatNotes(notes)}`;
-	return content;
-}
-/**
-* Parse frontmatter from markdown content
-*/
-function parseFrontmatter(content) {
-	const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
-	if (!match) return {
-		frontmatter: null,
-		body: content
-	};
-	const frontmatterContent = match[1];
-	const body = match[2];
-	const frontmatter = {};
-	const lines = frontmatterContent.split("\n");
-	for (const line of lines) {
-		const colonIndex = line.indexOf(":");
-		if (colonIndex > 0) {
-			const key = line.substring(0, colonIndex).trim();
-			frontmatter[key] = line.substring(colonIndex + 1).trim();
-		}
-	}
-	return {
-		frontmatter,
-		body
-	};
-}
-/**
-* Serialize frontmatter object to YAML-like format
-*/
-function serializeFrontmatter(data) {
-	const lines = [];
-	for (const [key, value] of Object.entries(data)) if (value !== void 0 && value !== null) lines.push(`${key}: ${String(value)}`);
-	return lines.join("\n");
-}
-/**
-* Generate llms-full.txt content with complete page content
-*/
-function generateLlmsFullTxtContent(files, options) {
-	const { siteName = "Site", description, origin = "", sections, notes } = options;
-	let content = `# ${siteName}\n\n`;
-	if (description) content += `> ${description}\n\n`;
-	if (origin) content += `Canonical Origin: ${origin}\n\n`;
-	if (sections) for (const section of sections) content += formatSection(section);
-	if (files.length > 0) {
-		content += `## Table of Contents\n\n`;
-		for (const file of files) {
-			const anchor = file.title.toLowerCase().replace(/[^a-z0-9]/g, "-");
-			content += `- [${file.title}](#${anchor})\n`;
-		}
-		content += `\n---\n\n`;
-		for (const file of files) {
-			const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
-			const { frontmatter, body } = parseFrontmatter(file.content);
-			const metadata = {
-				title: file.title,
-				url
-			};
-			if (file.filePath && options.outputDir) metadata.file = relative(options.outputDir, file.filePath);
-			else if (file.filePath) metadata.file = file.filePath;
-			if (file.metadata) {
-				if (file.metadata.description) metadata.description = file.metadata.description;
-				if (file.metadata.keywords) metadata.keywords = file.metadata.keywords;
-				if (file.metadata.author) metadata.author = file.metadata.author;
-			}
-			const frontmatterString = serializeFrontmatter(frontmatter ? {
-				...frontmatter,
-				...metadata
-			} : metadata);
-			let contentBody = frontmatter ? body : file.content;
-			const titleLine = contentBody.trim().split("\n")[0];
-			if (titleLine === file.title || titleLine === `# ${file.title}`) contentBody = contentBody.trim().split("\n").slice(1).join("\n").trimStart();
-			content += `---\n${frontmatterString}\n---\n\n${contentBody}\n\n---\n\n`;
-		}
-	}
-	if (notes) content += `\n${formatNotes(notes)}`;
-	return content;
-}
-/**
-* Generate individual markdown files structure
-*/
-function generateMarkdownFilesContent(files) {
-	const markdownFiles = [];
-	for (const file of files) {
-		const mdPath = `md/${file.url === "/" ? "index" : file.url.replace(/^\//, "").replace(/\/$/, "")}.md`;
-		markdownFiles.push({
-			path: mdPath,
-			content: file.content
-		});
-	}
-	return markdownFiles;
-}
-/**
-* Main function to process files and generate llms.txt artifacts
-*/
-async function generateLlmsTxtArtifacts(options) {
-	let files;
-	if (options.files) files = options.files;
-	else if (options.patterns) files = await processHtmlFiles(options.patterns, options.origin);
-	else throw new Error("Either patterns or files must be provided");
-	const llmsTxt = generateLlmsTxtContent(files, options);
-	let llmsFullTxt;
-	if (options.generateFull) llmsFullTxt = generateLlmsFullTxtContent(files, options);
-	let markdownFiles;
-	if (options.generateMarkdown) markdownFiles = generateMarkdownFilesContent(files);
-	return {
-		llmsTxt,
-		llmsFullTxt,
-		markdownFiles,
-		processedFiles: files
-	};
-}
-/**
-* Format a section with title, description, and links
-*/
-function formatSection(section) {
-	let content = `## ${section.title}\n\n`;
-	if (section.description) {
-		const descriptions = Array.isArray(section.description) ? section.description : [section.description];
-		for (const desc of descriptions) content += `${desc}\n\n`;
-	}
-	if (section.links?.length) {
-		for (const link of section.links) {
-			const desc = link.description ? `: ${link.description}` : "";
-			content += `- [${link.title}](${link.href})${desc}\n`;
-		}
-		content += "\n";
-	}
-	return content;
-}
-/**
-* Format notes section
-*/
-function formatNotes(notes) {
-	const noteLines = Array.isArray(notes) ? notes : [notes];
-	let content = "";
-	for (const note of noteLines) content += `${note}\n\n`;
-	return content;
-}
-/**
-* Create a WritableStream that generates llms.txt artifacts by streaming pages to disk
-*
-* Writes llms.txt (and optionally llms-full.txt) incrementally as pages are written,
-* never keeping full content in memory. Creates outputDir recursively if needed.
-*
-* @example
-* ```typescript
-* const stream = createLlmsTxtStream({
-*   siteName: 'My Docs',
-*   description: 'Documentation site',
-*   origin: 'https://example.com',
-*   generateFull: true,
-*   outputDir: './dist',
-*   sections: [
-*     {
-*       title: 'Getting Started',
-*       description: 'Quick start guide',
-*       links: [
-*         { title: 'Installation', href: '/install', description: 'How to install' },
-*         { title: 'Quick Start', href: '/quickstart' },
-*       ],
-*     },
-*   ],
-*   notes: ['Generated by mdream', 'Last updated: 2024'],
-* })
-*
-* const writer = stream.getWriter()
-* await writer.write({
-*   title: 'Home',
-*   content: '# Welcome\n\nHome page content.',
-*   url: '/',
-* })
-* await writer.close()
-* ```
-*
-* @param options - Configuration options
-* @returns WritableStream that accepts ProcessedFile objects
-*/
-/**
-* Get the group key for a URL (up to 2 segments deep)
-*/
-/**
-* Sort pages by URL path in hierarchical order (directory tree structure)
-* Groups by first segment, with root-level pages without nesting grouped together
-*/
-function sortPagesByPath(pages) {
-	const segmentHasNested = /* @__PURE__ */ new Map();
-	for (const page of pages) {
-		const segments = page.url.split("/").filter(Boolean);
-		const firstSegment = segments.length > 0 ? segments[0] : "";
-		if (!segmentHasNested.has(firstSegment)) segmentHasNested.set(firstSegment, false);
-		if (segments.length > 1) segmentHasNested.set(firstSegment, true);
-	}
-	return pages.sort((a, b) => {
-		const segmentsA = a.url.split("/").filter(Boolean);
-		const segmentsB = b.url.split("/").filter(Boolean);
-		const firstSegmentA = segmentsA.length > 0 ? segmentsA[0] : "";
-		const firstSegmentB = segmentsB.length > 0 ? segmentsB[0] : "";
-		const isRootLevelA = segmentsA.length <= 1;
-		const isRootLevelB = segmentsB.length <= 1;
-		const hasNestedA = segmentHasNested.get(firstSegmentA);
-		const hasNestedB = segmentHasNested.get(firstSegmentB);
-		const groupKeyA = isRootLevelA && !hasNestedA ? "" : firstSegmentA;
-		const groupKeyB = isRootLevelB && !hasNestedB ? "" : firstSegmentB;
-		if (groupKeyA === "" && groupKeyB !== "") return -1;
-		if (groupKeyA !== "" && groupKeyB === "") return 1;
-		if (groupKeyA !== groupKeyB) return groupKeyA.localeCompare(groupKeyB);
-		if (segmentsA.length === 0) return -1;
-		if (segmentsB.length === 0) return 1;
-		const minLen = Math.min(segmentsA.length, segmentsB.length);
-		for (let i = 0; i < minLen; i++) {
-			const cmp = segmentsA[i].localeCompare(segmentsB[i]);
-			if (cmp !== 0) return cmp;
-		}
-		return segmentsA.length - segmentsB.length;
-	});
-}
-function createLlmsTxtStream(options = {}) {
-	const { siteName = "Site", description, origin = "", generateFull, outputDir = process.cwd(), sections, notes } = options;
-	let llmsTxtHandle;
-	let llmsFullTxtHandle;
-	const bufferedPages = [];
-	return new WritableStream({
-		async start() {
-			await mkdir(outputDir, { recursive: true });
-			llmsTxtHandle = await open(join(outputDir, "llms.txt"), "w");
-			let header = `# ${siteName}\n\n`;
-			if (description) header += `> ${description}\n\n`;
-			if (origin) header += `Canonical Origin: ${origin}\n\n`;
-			if (sections) for (const section of sections) header += formatSection(section);
-			await llmsTxtHandle.write(header);
-			if (generateFull) {
-				llmsFullTxtHandle = await open(join(outputDir, "llms-full.txt"), "w");
-				let fullHeader = `# ${siteName}\n\n`;
-				if (description) fullHeader += `> ${description}\n\n`;
-				if (origin) fullHeader += `Canonical Origin: ${origin}\n\n`;
-				if (sections) for (const section of sections) fullHeader += formatSection(section);
-				await llmsFullTxtHandle.write(fullHeader);
-			}
-		},
-		async write(file) {
-			const desc = file.metadata?.description;
-			bufferedPages.push({
-				url: file.url,
-				title: file.title,
-				description: desc,
-				filePath: file.filePath
-			});
-			if (generateFull && llmsFullTxtHandle) {
-				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
-				const { frontmatter, body } = parseFrontmatter(file.content);
-				const metadata = {
-					title: file.title,
-					url
-				};
-				if (file.filePath) metadata.file = relative(outputDir, file.filePath);
-				if (file.metadata) {
-					if (file.metadata.description) metadata.description = file.metadata.description;
-					if (file.metadata.keywords) metadata.keywords = file.metadata.keywords;
-					if (file.metadata.author) metadata.author = file.metadata.author;
-				}
-				const frontmatterString = serializeFrontmatter(frontmatter ? {
-					...frontmatter,
-					...metadata
-				} : metadata);
-				let contentBody = frontmatter ? body : file.content;
-				const titleLine = contentBody.trim().split("\n")[0];
-				if (titleLine === file.title || titleLine === `# ${file.title}`) contentBody = contentBody.trim().split("\n").slice(1).join("\n").trimStart();
-				const fullChunk = `---\n${frontmatterString}\n---\n\n${contentBody}\n\n---\n\n`;
-				await llmsFullTxtHandle.write(fullChunk);
-			}
-		},
-		async close() {
-			const sortedPages = sortPagesByPath(bufferedPages);
-			const segmentHasNested = /* @__PURE__ */ new Map();
-			for (const page of sortedPages) {
-				const segments = page.url.split("/").filter(Boolean);
-				const firstSegment = segments.length > 0 ? segments[0] : "";
-				if (!segmentHasNested.has(firstSegment)) segmentHasNested.set(firstSegment, false);
-				if (segments.length > 1) segmentHasNested.set(firstSegment, true);
-			}
-			await llmsTxtHandle?.write(`## Pages\n\n`);
-			let currentGroup = "";
-			let segmentGroupIndex = 0;
-			let urlsInCurrentGroup = 0;
-			for (let i = 0; i < sortedPages.length; i++) {
-				const page = sortedPages[i];
-				const segments = page.url.split("/").filter(Boolean);
-				const firstSegment = segments.length > 0 ? segments[0] : "";
-				const isRootLevel = segments.length <= 1;
-				const hasNested = segmentHasNested.get(firstSegment);
-				const groupKey = isRootLevel && !hasNested ? "" : firstSegment;
-				if (groupKey !== currentGroup) {
-					if (urlsInCurrentGroup > 0) {
-						if (segmentGroupIndex === 0 || segmentGroupIndex >= 1 && segmentGroupIndex <= 2 && urlsInCurrentGroup > 1) await llmsTxtHandle?.write("\n");
-					}
-					currentGroup = groupKey;
-					segmentGroupIndex++;
-					urlsInCurrentGroup = 0;
-				}
-				urlsInCurrentGroup++;
-				const descText = page.description ? `: ${page.description.substring(0, 160)}${page.description.length > 160 ? "..." : ""}` : "";
-				let chunk = "";
-				if (page.filePath && page.filePath.endsWith(".md")) {
-					const relativePath = relative(outputDir, page.filePath);
-					chunk = `- [${page.title}](${relativePath})${descText}\n`;
-				} else {
-					const url = page.url.startsWith("http://") || page.url.startsWith("https://") ? page.url : origin ? origin + page.url : page.url;
-					chunk = `- [${page.title}](${url})${descText}\n`;
-				}
-				await llmsTxtHandle?.write(chunk);
-			}
-			if (notes) {
-				const notesContent = formatNotes(notes);
-				await llmsTxtHandle?.write(`\n${notesContent}`);
-				if (generateFull && llmsFullTxtHandle) await llmsFullTxtHandle.write(`\n${notesContent}`);
-			}
-			await llmsTxtHandle?.close();
-			await llmsFullTxtHandle?.close();
-		},
-		async abort(reason) {
-			await llmsTxtHandle?.close();
-			await llmsFullTxtHandle?.close();
-		}
-	});
-}
-
-//#endregion
-export { generateLlmsTxtArtifacts as n, createLlmsTxtStream as t };

package/dist/_chunks/minimal-BiDhcwif.mjs

@@ -1,40 +0,0 @@
-import { A as TAG_BUTTON, Dt as TAG_OBJECT, Ht as TAG_SELECT, K as TAG_EMBED, S as TAG_ASIDE, X as TAG_FOOTER, Y as TAG_FIGURE, Z as TAG_FORM, dt as TAG_INPUT, lt as TAG_IFRAME, nn as TAG_TEXTAREA, q as TAG_FIELDSET, wt as TAG_NAV } from "./const-Bf_XN9U9.mjs";
-import { a as filterPlugin, i as frontmatterPlugin, r as isolateMainPlugin, t as tailwindPlugin } from "./plugins-DJnqR2fA.mjs";
-
-//#region src/preset/minimal.ts
-/**
-* Creates a configurable minimal preset with advanced options
-*
-* @param options HTML to Markdown options
-* @returns HTML to Markdown options with configured plugins
-*/
-function withMinimalPreset(options = {}) {
-	const plugins = [
-		frontmatterPlugin(),
-		isolateMainPlugin(),
-		tailwindPlugin(),
-		filterPlugin({ exclude: [
-			TAG_FORM,
-			TAG_FIELDSET,
-			TAG_OBJECT,
-			TAG_EMBED,
-			TAG_FIGURE,
-			TAG_FOOTER,
-			TAG_ASIDE,
-			TAG_IFRAME,
-			TAG_INPUT,
-			TAG_TEXTAREA,
-			TAG_SELECT,
-			TAG_BUTTON,
-			TAG_NAV
-		] })
-	];
-	if (options.plugins) plugins.push(...options.plugins);
-	return {
-		...options,
-		plugins
-	};
-}
-
-//#endregion
-export { withMinimalPreset as t };

package/dist/_chunks/plugin-CjWWQTuL.mjs

@@ -1,12 +0,0 @@
-//#region src/pluggable/plugin.ts
-/**
-* Create a plugin that implements the Plugin interface with improved type inference
-*
-* @returns A complete plugin implementation
-*/
-function createPlugin(plugin) {
-	return plugin;
-}
-
-//#endregion
-export { createPlugin as t };