mdream 0.14.0 → 0.15.0

package/dist/_chunks/{llms-txt-D7Hduhij.mjs → llms-txt-T79S7X24.mjs}

@@ -1,7 +1,7 @@
 import { t as htmlToMarkdown } from "./src-BJpipdul.mjs";
 import { t as extractionPlugin } from "./extraction-BA9MDtq3.mjs";
-import { readFile } from "node:fs/promises";
-import { basename, dirname, relative, sep } from "pathe";
+import { mkdir, open, readFile } from "node:fs/promises";
+import { basename, dirname, join, relative, sep } from "pathe";
 import { glob } from "tinyglobby";
 
 //#region src/llms-txt.ts
@@ -215,6 +215,99 @@ async function generateLlmsTxtArtifacts(options) {
 		processedFiles: files
 	};
 }
+/**
+* Create a WritableStream that generates llms.txt artifacts by streaming pages to disk
+*
+* Writes llms.txt (and optionally llms-full.txt) incrementally as pages are written,
+* never keeping full content in memory. Creates outputDir recursively if needed.
+*
+* @example
+* ```typescript
+* const stream = createLlmsTxtStream({
+*   siteName: 'My Docs',
+*   description: 'Documentation site',
+*   origin: 'https://example.com',
+*   generateFull: true,
+*   outputDir: './dist',
+* })
+*
+* const writer = stream.getWriter()
+* await writer.write({
+*   title: 'Home',
+*   content: '# Welcome\n\nHome page content.',
+*   url: '/',
+* })
+* await writer.close()
+* ```
+*
+* @param options - Configuration options
+* @returns WritableStream that accepts ProcessedFile objects
+*/
+function createLlmsTxtStream(options = {}) {
+	const { siteName = "Site", description, origin = "", generateFull, outputDir = process.cwd() } = options;
+	let llmsTxtHandle;
+	let llmsFullTxtHandle;
+	return new WritableStream({
+		async start() {
+			await mkdir(outputDir, { recursive: true });
+			llmsTxtHandle = await open(join(outputDir, "llms.txt"), "w");
+			let header = `# ${siteName}\n\n`;
+			if (description) header += `> ${description}\n\n`;
+			header += `## Pages\n\n`;
+			await llmsTxtHandle.write(header);
+			if (generateFull) {
+				llmsFullTxtHandle = await open(join(outputDir, "llms-full.txt"), "w");
+				let fullHeader = `# ${siteName}\n\n`;
+				if (description) fullHeader += `> ${description}\n\n`;
+				await llmsFullTxtHandle.write(fullHeader);
+			}
+		},
+		async write(file) {
+			const desc = file.metadata?.description;
+			const descText = desc ? `: ${desc.substring(0, 100)}${desc.length > 100 ? "..." : ""}` : "";
+			let chunk = "";
+			if (file.filePath && file.filePath.endsWith(".md")) {
+				const relativePath = relative(outputDir, file.filePath);
+				chunk = `- [${file.title}](${relativePath})${descText}\n`;
+			} else {
+				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin + file.url;
+				chunk = `- [${file.title}](${url})${descText}\n`;
+			}
+			await llmsTxtHandle?.write(chunk);
+			if (generateFull && llmsFullTxtHandle) {
+				const url = file.url.startsWith("http://") || file.url.startsWith("https://") ? file.url : origin ? origin + file.url : file.url;
+				const { frontmatter, body } = parseFrontmatter(file.content);
+				const metadata = {
+					title: file.title,
+					url
+				};
+				if (file.filePath) metadata.file = relative(outputDir, file.filePath);
+				if (file.metadata) {
+					if (file.metadata.description) metadata.description = file.metadata.description;
+					if (file.metadata.keywords) metadata.keywords = file.metadata.keywords;
+					if (file.metadata.author) metadata.author = file.metadata.author;
+				}
+				const frontmatterString = serializeFrontmatter(frontmatter ? {
+					...frontmatter,
+					...metadata
+				} : metadata);
+				let contentBody = frontmatter ? body : file.content;
+				const titleLine = contentBody.trim().split("\n")[0];
+				if (titleLine === file.title || titleLine === `# ${file.title}`) contentBody = contentBody.trim().split("\n").slice(1).join("\n").trimStart();
+				const fullChunk = `---\n${frontmatterString}\n---\n\n${contentBody}\n\n---\n\n`;
+				await llmsFullTxtHandle.write(fullChunk);
+			}
+		},
+		async close() {
+			await llmsTxtHandle?.close();
+			await llmsFullTxtHandle?.close();
+		},
+		async abort(reason) {
+			await llmsTxtHandle?.close();
+			await llmsFullTxtHandle?.close();
+		}
+	});
+}
 
 //#endregion
-export { generateLlmsTxtArtifacts as t };
+export { generateLlmsTxtArtifacts as n, createLlmsTxtStream as t };

package/dist/cli.mjs

@@ -3,7 +3,7 @@ import "./_chunks/markdown-processor-D26Uo5td.mjs";
 import "./_chunks/plugin-CjWWQTuL.mjs";
 import { n as streamHtmlToMarkdown } from "./_chunks/src-BJpipdul.mjs";
 import "./_chunks/extraction-BA9MDtq3.mjs";
-import { t as generateLlmsTxtArtifacts } from "./_chunks/llms-txt-D7Hduhij.mjs";
+import { n as generateLlmsTxtArtifacts } from "./_chunks/llms-txt-T79S7X24.mjs";
 import "./_chunks/plugins-DJnqR2fA.mjs";
 import { t as withMinimalPreset } from "./_chunks/minimal-BiDhcwif.mjs";
 import { readFileSync } from "node:fs";

package/dist/llms-txt.d.mts

@@ -34,5 +34,49 @@ interface LlmsTxtArtifactsResult {
  * Main function to process files and generate llms.txt artifacts
  */
 declare function generateLlmsTxtArtifacts(options: LlmsTxtArtifactsOptions): Promise<LlmsTxtArtifactsResult>;
+/**
+ * Options for creating an llms.txt stream
+ */
+interface CreateLlmsTxtStreamOptions extends Omit<LlmsTxtArtifactsOptions, 'patterns' | 'files' | 'outputDir' | 'generateMarkdown'> {
+  /** Directory to write files to (defaults to process.cwd()) */
+  outputDir?: string;
+  /** Site name for the header (defaults to 'Site') */
+  siteName?: string;
+  /** Site description for the header */
+  description?: string;
+  /** Origin URL to prepend to relative URLs */
+  origin?: string;
+  /** Generate llms-full.txt with complete page content (defaults to false) */
+  generateFull?: boolean;
+}
+/**
+ * Create a WritableStream that generates llms.txt artifacts by streaming pages to disk
+ *
+ * Writes llms.txt (and optionally llms-full.txt) incrementally as pages are written,
+ * never keeping full content in memory. Creates outputDir recursively if needed.
+ *
+ * @example
+ * ```typescript
+ * const stream = createLlmsTxtStream({
+ *   siteName: 'My Docs',
+ *   description: 'Documentation site',
+ *   origin: 'https://example.com',
+ *   generateFull: true,
+ *   outputDir: './dist',
+ * })
+ *
+ * const writer = stream.getWriter()
+ * await writer.write({
+ *   title: 'Home',
+ *   content: '# Welcome\n\nHome page content.',
+ *   url: '/',
+ * })
+ * await writer.close()
+ * ```
+ *
+ * @param options - Configuration options
+ * @returns WritableStream that accepts ProcessedFile objects
+ */
+declare function createLlmsTxtStream(options?: CreateLlmsTxtStreamOptions): WritableStream<ProcessedFile>;
 //#endregion
-export { LlmsTxtArtifactsOptions, LlmsTxtArtifactsResult, ProcessedFile, generateLlmsTxtArtifacts };
+export { CreateLlmsTxtStreamOptions, LlmsTxtArtifactsOptions, LlmsTxtArtifactsResult, ProcessedFile, createLlmsTxtStream, generateLlmsTxtArtifacts };

package/dist/llms-txt.mjs

@@ -3,6 +3,6 @@ import "./_chunks/markdown-processor-D26Uo5td.mjs";
 import "./_chunks/plugin-CjWWQTuL.mjs";
 import "./_chunks/src-BJpipdul.mjs";
 import "./_chunks/extraction-BA9MDtq3.mjs";
-import { t as generateLlmsTxtArtifacts } from "./_chunks/llms-txt-D7Hduhij.mjs";
+import { n as generateLlmsTxtArtifacts, t as createLlmsTxtStream } from "./_chunks/llms-txt-T79S7X24.mjs";
 
-export { generateLlmsTxtArtifacts };
+export { createLlmsTxtStream, generateLlmsTxtArtifacts };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mdream",
   "type": "module",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
   "author": {
     "name": "Harlan Wilton",