mdream 0.15.1 → 0.15.2

package/dist/_chunks/{llms-txt-BXtLmgK6.mjs → llms-txt-Czb_M48B.mjs}

@@ -92,6 +92,7 @@ function generateLlmsTxtContent(files, options) {
 	const { siteName = "Site", description, origin = "", sections, notes } = options;
 	let content = `# ${siteName}\n\n`;
 	if (description) content += `> ${description}\n\n`;
+	if (origin) content += `Canonical Origin: ${origin}\n\n`;
 	if (sections) for (const section of sections) content += formatSection(section);
 	if (files.length > 0) {
 		content += `## Pages\n\n`;
@@ -102,7 +103,7 @@ function generateLlmsTxtContent(files, options) {
 				const relativePath = relative(options.outputDir, file.filePath);
 				content += `- [${file.title}](${relativePath})${descText}\n`;
 			} else {
-				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin + file.url;
+				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
 				content += `- [${file.title}](${url})${descText}\n`;
 			}
 		}
@@ -150,6 +151,7 @@ function generateLlmsFullTxtContent(files, options) {
 	const { siteName = "Site", description, origin = "", sections, notes } = options;
 	let content = `# ${siteName}\n\n`;
 	if (description) content += `> ${description}\n\n`;
+	if (origin) content += `Canonical Origin: ${origin}\n\n`;
 	if (sections) for (const section of sections) content += formatSection(section);
 	if (files.length > 0) {
 		content += `## Table of Contents\n\n`;
@@ -285,39 +287,76 @@ function formatNotes(notes) {
 * @param options - Configuration options
 * @returns WritableStream that accepts ProcessedFile objects
 */
+/**
+* Get the group key for a URL (up to 2 segments deep)
+*/
+/**
+* Sort pages by URL path in hierarchical order (directory tree structure)
+* Groups by first segment, with root-level pages without nesting grouped together
+*/
+function sortPagesByPath(pages) {
+	const segmentHasNested = /* @__PURE__ */ new Map();
+	for (const page of pages) {
+		const segments = page.url.split("/").filter(Boolean);
+		const firstSegment = segments.length > 0 ? segments[0] : "";
+		if (!segmentHasNested.has(firstSegment)) segmentHasNested.set(firstSegment, false);
+		if (segments.length > 1) segmentHasNested.set(firstSegment, true);
+	}
+	return pages.sort((a, b) => {
+		const segmentsA = a.url.split("/").filter(Boolean);
+		const segmentsB = b.url.split("/").filter(Boolean);
+		const firstSegmentA = segmentsA.length > 0 ? segmentsA[0] : "";
+		const firstSegmentB = segmentsB.length > 0 ? segmentsB[0] : "";
+		const isRootLevelA = segmentsA.length <= 1;
+		const isRootLevelB = segmentsB.length <= 1;
+		const hasNestedA = segmentHasNested.get(firstSegmentA);
+		const hasNestedB = segmentHasNested.get(firstSegmentB);
+		const groupKeyA = isRootLevelA && !hasNestedA ? "" : firstSegmentA;
+		const groupKeyB = isRootLevelB && !hasNestedB ? "" : firstSegmentB;
+		if (groupKeyA === "" && groupKeyB !== "") return -1;
+		if (groupKeyA !== "" && groupKeyB === "") return 1;
+		if (groupKeyA !== groupKeyB) return groupKeyA.localeCompare(groupKeyB);
+		if (segmentsA.length === 0) return -1;
+		if (segmentsB.length === 0) return 1;
+		const minLen = Math.min(segmentsA.length, segmentsB.length);
+		for (let i = 0; i < minLen; i++) {
+			const cmp = segmentsA[i].localeCompare(segmentsB[i]);
+			if (cmp !== 0) return cmp;
+		}
+		return segmentsA.length - segmentsB.length;
+	});
+}
 function createLlmsTxtStream(options = {}) {
 	const { siteName = "Site", description, origin = "", generateFull, outputDir = process.cwd(), sections, notes } = options;
 	let llmsTxtHandle;
 	let llmsFullTxtHandle;
+	const bufferedPages = [];
 	return new WritableStream({
 		async start() {
 			await mkdir(outputDir, { recursive: true });
 			llmsTxtHandle = await open(join(outputDir, "llms.txt"), "w");
 			let header = `# ${siteName}\n\n`;
 			if (description) header += `> ${description}\n\n`;
+			if (origin) header += `Canonical Origin: ${origin}\n\n`;
 			if (sections) for (const section of sections) header += formatSection(section);
-			header += `## Pages\n\n`;
 			await llmsTxtHandle.write(header);
 			if (generateFull) {
 				llmsFullTxtHandle = await open(join(outputDir, "llms-full.txt"), "w");
 				let fullHeader = `# ${siteName}\n\n`;
 				if (description) fullHeader += `> ${description}\n\n`;
+				if (origin) fullHeader += `Canonical Origin: ${origin}\n\n`;
 				if (sections) for (const section of sections) fullHeader += formatSection(section);
 				await llmsFullTxtHandle.write(fullHeader);
 			}
 		},
 		async write(file) {
 			const desc = file.metadata?.description;
-			const descText = desc ? `: ${desc.substring(0, 100)}${desc.length > 100 ? "..." : ""}` : "";
-			let chunk = "";
-			if (file.filePath && file.filePath.endsWith(".md")) {
-				const relativePath = relative(outputDir, file.filePath);
-				chunk = `- [${file.title}](${relativePath})${descText}\n`;
-			} else {
-				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin + file.url;
-				chunk = `- [${file.title}](${url})${descText}\n`;
-			}
-			await llmsTxtHandle?.write(chunk);
+			bufferedPages.push({
+				url: file.url,
+				title: file.title,
+				description: desc,
+				filePath: file.filePath
+			});
 			if (generateFull && llmsFullTxtHandle) {
 				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
 				const { frontmatter, body } = parseFrontmatter(file.content);
@@ -343,6 +382,45 @@ function createLlmsTxtStream(options = {}) {
 			}
 		},
 		async close() {
+			const sortedPages = sortPagesByPath(bufferedPages);
+			const segmentHasNested = /* @__PURE__ */ new Map();
+			for (const page of sortedPages) {
+				const segments = page.url.split("/").filter(Boolean);
+				const firstSegment = segments.length > 0 ? segments[0] : "";
+				if (!segmentHasNested.has(firstSegment)) segmentHasNested.set(firstSegment, false);
+				if (segments.length > 1) segmentHasNested.set(firstSegment, true);
+			}
+			await llmsTxtHandle?.write(`## Pages\n\n`);
+			let currentGroup = "";
+			let segmentGroupIndex = 0;
+			let urlsInCurrentGroup = 0;
+			for (let i = 0; i < sortedPages.length; i++) {
+				const page = sortedPages[i];
+				const segments = page.url.split("/").filter(Boolean);
+				const firstSegment = segments.length > 0 ? segments[0] : "";
+				const isRootLevel = segments.length <= 1;
+				const hasNested = segmentHasNested.get(firstSegment);
+				const groupKey = isRootLevel && !hasNested ? "" : firstSegment;
+				if (groupKey !== currentGroup) {
+					if (urlsInCurrentGroup > 0) {
+						if (segmentGroupIndex === 0 || segmentGroupIndex >= 1 && segmentGroupIndex <= 2 && urlsInCurrentGroup > 1) await llmsTxtHandle?.write("\n");
+					}
+					currentGroup = groupKey;
+					segmentGroupIndex++;
+					urlsInCurrentGroup = 0;
+				}
+				urlsInCurrentGroup++;
+				const descText = page.description ? `: ${page.description.substring(0, 160)}${page.description.length > 160 ? "..." : ""}` : "";
+				let chunk = "";
+				if (page.filePath && page.filePath.endsWith(".md")) {
+					const relativePath = relative(outputDir, page.filePath);
+					chunk = `- [${page.title}](${relativePath})${descText}\n`;
+				} else {
+					const url = page.url.startsWith("http://") || page.url.startsWith("https://") ? page.url : origin ? origin + page.url : page.url;
+					chunk = `- [${page.title}](${url})${descText}\n`;
+				}
+				await llmsTxtHandle?.write(chunk);
+			}
 			if (notes) {
 				const notesContent = formatNotes(notes);
 				await llmsTxtHandle?.write(`\n${notesContent}`);

package/dist/cli.mjs

@@ -1,6 +1,6 @@
 import "./_chunks/markdown-processor-D26Uo5td.mjs";
 import { n as streamHtmlToMarkdown } from "./_chunks/src-BJpipdul.mjs";
-import { n as generateLlmsTxtArtifacts } from "./_chunks/llms-txt-BXtLmgK6.mjs";
+import { n as generateLlmsTxtArtifacts } from "./_chunks/llms-txt-Czb_M48B.mjs";
 import "./_chunks/plugins-DJnqR2fA.mjs";
 import { t as withMinimalPreset } from "./_chunks/minimal-BiDhcwif.mjs";
 import { readFileSync } from "node:fs";

package/dist/llms-txt.d.mts

@@ -79,45 +79,6 @@ interface CreateLlmsTxtStreamOptions extends Omit<LlmsTxtArtifactsOptions, 'patt
   /** Notes to write at the end */
   notes?: string | string[];
 }
-/**
- * Create a WritableStream that generates llms.txt artifacts by streaming pages to disk
- *
- * Writes llms.txt (and optionally llms-full.txt) incrementally as pages are written,
- * never keeping full content in memory. Creates outputDir recursively if needed.
- *
- * @example
- * ```typescript
- * const stream = createLlmsTxtStream({
- *   siteName: 'My Docs',
- *   description: 'Documentation site',
- *   origin: 'https://example.com',
- *   generateFull: true,
- *   outputDir: './dist',
- *   sections: [
- *     {
- *       title: 'Getting Started',
- *       description: 'Quick start guide',
- *       links: [
- *         { title: 'Installation', href: '/install', description: 'How to install' },
- *         { title: 'Quick Start', href: '/quickstart' },
- *       ],
- *     },
- *   ],
- *   notes: ['Generated by mdream', 'Last updated: 2024'],
- * })
- *
- * const writer = stream.getWriter()
- * await writer.write({
- *   title: 'Home',
- *   content: '# Welcome\n\nHome page content.',
- *   url: '/',
- * })
- * await writer.close()
- * ```
- *
- * @param options - Configuration options
- * @returns WritableStream that accepts ProcessedFile objects
- */
 declare function createLlmsTxtStream(options?: CreateLlmsTxtStreamOptions): WritableStream<ProcessedFile>;
 //#endregion
 export { CreateLlmsTxtStreamOptions, LlmsTxtArtifactsOptions, LlmsTxtArtifactsResult, LlmsTxtLink, LlmsTxtSection, ProcessedFile, createLlmsTxtStream, generateLlmsTxtArtifacts };

package/dist/llms-txt.mjs

@@ -1,5 +1,5 @@
 import "./_chunks/markdown-processor-D26Uo5td.mjs";
 import "./_chunks/src-BJpipdul.mjs";
-import { n as generateLlmsTxtArtifacts, t as createLlmsTxtStream } from "./_chunks/llms-txt-BXtLmgK6.mjs";
+import { n as generateLlmsTxtArtifacts, t as createLlmsTxtStream } from "./_chunks/llms-txt-Czb_M48B.mjs";
 
 export { createLlmsTxtStream, generateLlmsTxtArtifacts };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mdream",
   "type": "module",
-  "version": "0.15.1",
+  "version": "0.15.2",
   "description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
   "author": {
     "name": "Harlan Wilton",