mdream 0.14.0 → 0.15.1

package/README.md

@@ -422,6 +422,102 @@ const chunks = htmlToMarkdownSplitChunks(html, withMinimalPreset({
 }))
 ```
 
+## llms.txt Generation
+
+Generate [llms.txt](https://llmstxt.org) files from HTML content for improved LLM discoverability. Mdream provides both streaming and batch APIs for creating llms.txt artifacts.
+
+### createLlmsTxtStream
+
+Stream llms.txt generation without keeping full content in memory:
+
+```ts
+import { createLlmsTxtStream } from 'mdream'
+
+const stream = createLlmsTxtStream({
+  siteName: 'My Docs',
+  description: 'Documentation site',
+  origin: 'https://example.com',
+  outputDir: './dist',
+  generateFull: true, // Also generate llms-full.txt
+  sections: [
+    {
+      title: 'Getting Started',
+      description: 'Quick start guide',
+      links: [
+        { title: 'Installation', href: '/install', description: 'How to install' },
+        { title: 'Quick Start', href: '/quickstart' },
+      ],
+    },
+  ],
+  notes: ['Generated by mdream', 'Last updated: 2024'],
+})
+
+const writer = stream.getWriter()
+await writer.write({
+  title: 'Home',
+  content: '# Welcome\n\nHome page content.',
+  url: '/',
+  metadata: {
+    description: 'Welcome page',
+  },
+})
+await writer.close()
+```
+
+This creates:
+- `llms.txt` - Links to all pages with metadata
+- `llms-full.txt` - Complete content with frontmatter (if `generateFull: true`)
+
+### generateLlmsTxtArtifacts
+
+Process HTML files or ProcessedFile objects:
+
+```ts
+import { generateLlmsTxtArtifacts } from 'mdream'
+
+const result = await generateLlmsTxtArtifacts({
+  patterns: '**/*.html', // Glob pattern for HTML files
+  siteName: 'My Site',
+  origin: 'https://example.com',
+  generateFull: true,
+  sections: [
+    {
+      title: 'Resources',
+      links: [
+        { title: 'Docs', href: '/docs' },
+      ],
+    },
+  ],
+  notes: 'Footer notes',
+})
+
+console.log(result.llmsTxt) // llms.txt content
+console.log(result.llmsFullTxt) // llms-full.txt content
+console.log(result.processedFiles) // Array of processed files
+```
+
+### Structure
+
+llms.txt follows this structure:
+
+```markdown
+# Site Name
+
+> Site description
+
+## Custom Section
+
+Section description
+
+- [Link Title](url): Optional description
+
+## Pages
+
+- [Page Title](url): Page description
+
+Custom notes
+```
+
 ## Credits
 
 - [ultrahtml](https://github.com/natemoo-re/ultrahtml): HTML parsing inspiration

package/dist/_chunks/{llms-txt-D7Hduhij.mjs → llms-txt-BXtLmgK6.mjs}

@@ -1,7 +1,7 @@
 import { t as htmlToMarkdown } from "./src-BJpipdul.mjs";
 import { t as extractionPlugin } from "./extraction-BA9MDtq3.mjs";
-import { readFile } from "node:fs/promises";
-import { basename, dirname, relative, sep } from "pathe";
+import { mkdir, open, readFile } from "node:fs/promises";
+import { basename, dirname, join, relative, sep } from "pathe";
 import { glob } from "tinyglobby";
 
 //#region src/llms-txt.ts
@@ -89,9 +89,10 @@ async function processHtmlFiles(patterns, origin) {
 * Generate llms.txt content
 */
 function generateLlmsTxtContent(files, options) {
-	const { siteName = "Site", description, origin = "" } = options;
+	const { siteName = "Site", description, origin = "", sections, notes } = options;
 	let content = `# ${siteName}\n\n`;
 	if (description) content += `> ${description}\n\n`;
+	if (sections) for (const section of sections) content += formatSection(section);
 	if (files.length > 0) {
 		content += `## Pages\n\n`;
 		for (const file of files) {
@@ -106,6 +107,7 @@ function generateLlmsTxtContent(files, options) {
 			}
 		}
 	}
+	if (notes) content += `\n${formatNotes(notes)}`;
 	return content;
 }
 /**
@@ -145,9 +147,10 @@ function serializeFrontmatter(data) {
 * Generate llms-full.txt content with complete page content
 */
 function generateLlmsFullTxtContent(files, options) {
-	const { siteName = "Site", description, origin = "" } = options;
+	const { siteName = "Site", description, origin = "", sections, notes } = options;
 	let content = `# ${siteName}\n\n`;
 	if (description) content += `> ${description}\n\n`;
+	if (sections) for (const section of sections) content += formatSection(section);
 	if (files.length > 0) {
 		content += `## Table of Contents\n\n`;
 		for (const file of files) {
@@ -179,6 +182,7 @@ function generateLlmsFullTxtContent(files, options) {
 			content += `---\n${frontmatterString}\n---\n\n${contentBody}\n\n---\n\n`;
 		}
 	}
+	if (notes) content += `\n${formatNotes(notes)}`;
 	return content;
 }
 /**
@@ -215,6 +219,144 @@ async function generateLlmsTxtArtifacts(options) {
 		processedFiles: files
 	};
 }
+/**
+* Format a section with title, description, and links
+*/
+function formatSection(section) {
+	let content = `## ${section.title}\n\n`;
+	if (section.description) {
+		const descriptions = Array.isArray(section.description) ? section.description : [section.description];
+		for (const desc of descriptions) content += `${desc}\n\n`;
+	}
+	if (section.links?.length) {
+		for (const link of section.links) {
+			const desc = link.description ? `: ${link.description}` : "";
+			content += `- [${link.title}](${link.href})${desc}\n`;
+		}
+		content += "\n";
+	}
+	return content;
+}
+/**
+* Format notes section
+*/
+function formatNotes(notes) {
+	const noteLines = Array.isArray(notes) ? notes : [notes];
+	let content = "";
+	for (const note of noteLines) content += `${note}\n\n`;
+	return content;
+}
+/**
+* Create a WritableStream that generates llms.txt artifacts by streaming pages to disk
+*
+* Writes llms.txt (and optionally llms-full.txt) incrementally as pages are written,
+* never keeping full content in memory. Creates outputDir recursively if needed.
+*
+* @example
+* ```typescript
+* const stream = createLlmsTxtStream({
+*   siteName: 'My Docs',
+*   description: 'Documentation site',
+*   origin: 'https://example.com',
+*   generateFull: true,
+*   outputDir: './dist',
+*   sections: [
+*     {
+*       title: 'Getting Started',
+*       description: 'Quick start guide',
+*       links: [
+*         { title: 'Installation', href: '/install', description: 'How to install' },
+*         { title: 'Quick Start', href: '/quickstart' },
+*       ],
+*     },
+*   ],
+*   notes: ['Generated by mdream', 'Last updated: 2024'],
+* })
+*
+* const writer = stream.getWriter()
+* await writer.write({
+*   title: 'Home',
+*   content: '# Welcome\n\nHome page content.',
+*   url: '/',
+* })
+* await writer.close()
+* ```
+*
+* @param options - Configuration options
+* @returns WritableStream that accepts ProcessedFile objects
+*/
+function createLlmsTxtStream(options = {}) {
+	const { siteName = "Site", description, origin = "", generateFull, outputDir = process.cwd(), sections, notes } = options;
+	let llmsTxtHandle;
+	let llmsFullTxtHandle;
+	return new WritableStream({
+		async start() {
+			await mkdir(outputDir, { recursive: true });
+			llmsTxtHandle = await open(join(outputDir, "llms.txt"), "w");
+			let header = `# ${siteName}\n\n`;
+			if (description) header += `> ${description}\n\n`;
+			if (sections) for (const section of sections) header += formatSection(section);
+			header += `## Pages\n\n`;
+			await llmsTxtHandle.write(header);
+			if (generateFull) {
+				llmsFullTxtHandle = await open(join(outputDir, "llms-full.txt"), "w");
+				let fullHeader = `# ${siteName}\n\n`;
+				if (description) fullHeader += `> ${description}\n\n`;
+				if (sections) for (const section of sections) fullHeader += formatSection(section);
+				await llmsFullTxtHandle.write(fullHeader);
+			}
+		},
+		async write(file) {
+			const desc = file.metadata?.description;
+			const descText = desc ? `: ${desc.substring(0, 100)}${desc.length > 100 ? "..." : ""}` : "";
+			let chunk = "";
+			if (file.filePath && file.filePath.endsWith(".md")) {
+				const relativePath = relative(outputDir, file.filePath);
+				chunk = `- [${file.title}](${relativePath})${descText}\n`;
+			} else {
+				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin + file.url;
+				chunk = `- [${file.title}](${url})${descText}\n`;
+			}
+			await llmsTxtHandle?.write(chunk);
+			if (generateFull && llmsFullTxtHandle) {
+				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
+				const { frontmatter, body } = parseFrontmatter(file.content);
+				const metadata = {
+					title: file.title,
+					url
+				};
+				if (file.filePath) metadata.file = relative(outputDir, file.filePath);
+				if (file.metadata) {
+					if (file.metadata.description) metadata.description = file.metadata.description;
+					if (file.metadata.keywords) metadata.keywords = file.metadata.keywords;
+					if (file.metadata.author) metadata.author = file.metadata.author;
+				}
+				const frontmatterString = serializeFrontmatter(frontmatter ? {
+					...frontmatter,
+					...metadata
+				} : metadata);
+				let contentBody = frontmatter ? body : file.content;
+				const titleLine = contentBody.trim().split("\n")[0];
+				if (titleLine === file.title || titleLine === `# ${file.title}`) contentBody = contentBody.trim().split("\n").slice(1).join("\n").trimStart();
+				const fullChunk = `---\n${frontmatterString}\n---\n\n${contentBody}\n\n---\n\n`;
+				await llmsFullTxtHandle.write(fullChunk);
+			}
+		},
+		async close() {
+			if (notes) {
+				const notesContent = formatNotes(notes);
+				await llmsTxtHandle?.write(`\n${notesContent}`);
+				if (generateFull && llmsFullTxtHandle) await llmsFullTxtHandle.write(`\n${notesContent}`);
+			}
+			await llmsTxtHandle?.close();
+			await llmsFullTxtHandle?.close();
+		},
+		async abort(reason) {
+			await llmsTxtHandle?.close();
+			await llmsFullTxtHandle?.close();
+		}
+	});
+}
 
 //#endregion
-export { generateLlmsTxtArtifacts as t };
+export { generateLlmsTxtArtifacts as n, createLlmsTxtStream as t };

package/dist/cli.mjs

@@ -1,9 +1,6 @@
-import "./_chunks/const-Bf_XN9U9.mjs";
 import "./_chunks/markdown-processor-D26Uo5td.mjs";
-import "./_chunks/plugin-CjWWQTuL.mjs";
 import { n as streamHtmlToMarkdown } from "./_chunks/src-BJpipdul.mjs";
-import "./_chunks/extraction-BA9MDtq3.mjs";
-import { t as generateLlmsTxtArtifacts } from "./_chunks/llms-txt-D7Hduhij.mjs";
+import { n as generateLlmsTxtArtifacts } from "./_chunks/llms-txt-BXtLmgK6.mjs";
 import "./_chunks/plugins-DJnqR2fA.mjs";
 import { t as withMinimalPreset } from "./_chunks/minimal-BiDhcwif.mjs";
 import { readFileSync } from "node:fs";

package/dist/llms-txt.d.mts

@@ -1,4 +1,26 @@
 //#region src/llms-txt.d.ts
+/**
+ * Link in llms.txt section
+ */
+interface LlmsTxtLink {
+  /** The title of the link */
+  title: string;
+  /** The description of the link */
+  description?: string;
+  /** The href of the link */
+  href: string;
+}
+/**
+ * Section in llms.txt
+ */
+interface LlmsTxtSection {
+  /** The title of the section */
+  title: string;
+  /** The description of the section (can be array for multiple paragraphs) */
+  description?: string | string[];
+  /** The links of the section */
+  links?: LlmsTxtLink[];
+}
 interface LlmsTxtArtifactsOptions {
   patterns?: string | string[];
   files?: ProcessedFile[];
@@ -8,6 +30,10 @@ interface LlmsTxtArtifactsOptions {
   generateFull?: boolean;
   generateMarkdown?: boolean;
   outputDir?: string;
+  /** The sections to write before pages */
+  sections?: LlmsTxtSection[];
+  /** Notes to write at the end */
+  notes?: string | string[];
 }
 interface ProcessedFile {
   filePath?: string;
@@ -34,5 +60,64 @@ interface LlmsTxtArtifactsResult {
  * Main function to process files and generate llms.txt artifacts
  */
 declare function generateLlmsTxtArtifacts(options: LlmsTxtArtifactsOptions): Promise<LlmsTxtArtifactsResult>;
+/**
+ * Options for creating an llms.txt stream
+ */
+interface CreateLlmsTxtStreamOptions extends Omit<LlmsTxtArtifactsOptions, 'patterns' | 'files' | 'outputDir' | 'generateMarkdown'> {
+  /** Directory to write files to (defaults to process.cwd()) */
+  outputDir?: string;
+  /** Site name for the header (defaults to 'Site') */
+  siteName?: string;
+  /** Site description for the header */
+  description?: string;
+  /** Origin URL to prepend to relative URLs */
+  origin?: string;
+  /** Generate llms-full.txt with complete page content (defaults to false) */
+  generateFull?: boolean;
+  /** The sections to write before pages */
+  sections?: LlmsTxtSection[];
+  /** Notes to write at the end */
+  notes?: string | string[];
+}
+/**
+ * Create a WritableStream that generates llms.txt artifacts by streaming pages to disk
+ *
+ * Writes llms.txt (and optionally llms-full.txt) incrementally as pages are written,
+ * never keeping full content in memory. Creates outputDir recursively if needed.
+ *
+ * @example
+ * ```typescript
+ * const stream = createLlmsTxtStream({
+ *   siteName: 'My Docs',
+ *   description: 'Documentation site',
+ *   origin: 'https://example.com',
+ *   generateFull: true,
+ *   outputDir: './dist',
+ *   sections: [
+ *     {
+ *       title: 'Getting Started',
+ *       description: 'Quick start guide',
+ *       links: [
+ *         { title: 'Installation', href: '/install', description: 'How to install' },
+ *         { title: 'Quick Start', href: '/quickstart' },
+ *       ],
+ *     },
+ *   ],
+ *   notes: ['Generated by mdream', 'Last updated: 2024'],
+ * })
+ *
+ * const writer = stream.getWriter()
+ * await writer.write({
+ *   title: 'Home',
+ *   content: '# Welcome\n\nHome page content.',
+ *   url: '/',
+ * })
+ * await writer.close()
+ * ```
+ *
+ * @param options - Configuration options
+ * @returns WritableStream that accepts ProcessedFile objects
+ */
+declare function createLlmsTxtStream(options?: CreateLlmsTxtStreamOptions): WritableStream<ProcessedFile>;
 //#endregion
-export { LlmsTxtArtifactsOptions, LlmsTxtArtifactsResult, ProcessedFile, generateLlmsTxtArtifacts };
+export { CreateLlmsTxtStreamOptions, LlmsTxtArtifactsOptions, LlmsTxtArtifactsResult, LlmsTxtLink, LlmsTxtSection, ProcessedFile, createLlmsTxtStream, generateLlmsTxtArtifacts };

package/dist/llms-txt.mjs

@@ -1,8 +1,5 @@
-import "./_chunks/const-Bf_XN9U9.mjs";
 import "./_chunks/markdown-processor-D26Uo5td.mjs";
-import "./_chunks/plugin-CjWWQTuL.mjs";
 import "./_chunks/src-BJpipdul.mjs";
-import "./_chunks/extraction-BA9MDtq3.mjs";
-import { t as generateLlmsTxtArtifacts } from "./_chunks/llms-txt-D7Hduhij.mjs";
+import { n as generateLlmsTxtArtifacts, t as createLlmsTxtStream } from "./_chunks/llms-txt-BXtLmgK6.mjs";
 
-export { generateLlmsTxtArtifacts };
+export { createLlmsTxtStream, generateLlmsTxtArtifacts };

package/dist/plugins.mjs

@@ -1,4 +1,3 @@
-import "./_chunks/const-Bf_XN9U9.mjs";
 import { t as createPlugin } from "./_chunks/plugin-CjWWQTuL.mjs";
 import { t as extractionPlugin } from "./_chunks/extraction-BA9MDtq3.mjs";
 import { a as filterPlugin, i as frontmatterPlugin, n as readabilityPlugin, r as isolateMainPlugin, t as tailwindPlugin } from "./_chunks/plugins-DJnqR2fA.mjs";

package/dist/preset/minimal.mjs

@@ -1,6 +1,3 @@
-import "../_chunks/const-Bf_XN9U9.mjs";
-import "../_chunks/plugin-CjWWQTuL.mjs";
-import "../_chunks/extraction-BA9MDtq3.mjs";
 import "../_chunks/plugins-DJnqR2fA.mjs";
 import { t as withMinimalPreset } from "../_chunks/minimal-BiDhcwif.mjs";
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mdream",
   "type": "module",
-  "version": "0.14.0",
+  "version": "0.15.1",
   "description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
   "author": {
     "name": "Harlan Wilton",