nuxt-ai-ready 0.1.2 → 0.1.3

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -1,31 +1,37 @@
-import { createHash } from 'node:crypto';
-import { mkdirSync, createWriteStream } from 'node:fs';
 import { useLogger, useNuxt, defineNuxtModule, createResolver, addTypeTemplate, hasNuxtModule, addServerHandler, addPlugin } from '@nuxt/kit';
 import defu from 'defu';
-import { TagIdMap } from 'mdream';
-import { extractionPlugin } from 'mdream/plugins';
-import { htmlToMarkdownSplitChunksStream } from 'mdream/splitter';
 import { useSiteConfig, installNuxtSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
-import { isPathFile } from 'nuxt-site-config/urls';
-import { relative, resolve, dirname } from 'pathe';
+import { relative as relative$1 } from 'pathe';
 import { readPackageJSON } from 'pkg-types';
-import { estimateTokenCount } from 'tokenx';
+import { createHash } from 'node:crypto';
+import { mkdirSync, createWriteStream } from 'node:fs';
 import { stat } from 'node:fs/promises';
-import { join } from 'node:path';
+import { join, dirname, relative } from 'node:path';
 import { createLlmsTxtStream } from 'mdream/llms-txt';
 
 const logger = useLogger("nuxt-ai-ready");
 
+function generateVectorId(route, chunkIdx) {
+  const hash = createHash("sha256").update(route).digest("hex").substring(0, 48);
+  return `${hash}-${chunkIdx}`;
+}
 function setupPrerenderHandler() {
   const nuxt = useNuxt();
   nuxt.hooks.hook("nitro:init", async (nitro) => {
     let writer = null;
+    let bulkStream = null;
+    let bulkStreamEntries = 0;
     let pageCount = 0;
     const startTime = Date.now();
+    const bulkPath = join(nitro.options.output.publicDir, "content.jsonl");
     nitro.hooks.hook("prerender:generate", async (route) => {
       if (!route.fileName?.endsWith(".md")) {
         return;
       }
+      let pageRoute = route.route.replace(/\.md$/, "");
+      if (pageRoute === "/index") {
+        pageRoute = "/";
+      }
       if (!writer) {
         const siteConfig = useSiteConfig();
         const stream = createLlmsTxtStream({
@@ -37,10 +43,16 @@ function setupPrerenderHandler() {
         });
         writer = stream.getWriter();
       }
-      const { markdown, title, description } = JSON.parse(route.contents || "{}");
+      if (!bulkStream) {
+        mkdirSync(dirname(bulkPath), { recursive: true });
+        bulkStream = createWriteStream(bulkPath);
+        logger.info(`Bulk JSONL stream created at ${relative(nuxt.options.rootDir, bulkPath)}`);
+      }
+      const { chunks, title, description, headings } = JSON.parse(route.contents || "{}");
+      const markdown = chunks.map((c) => c.content).join("\n\n");
       await writer.write({
         filePath: route.fileName,
-        url: route.route,
+        url: pageRoute,
         title,
         content: markdown,
         metadata: {
@@ -49,6 +61,33 @@ function setupPrerenderHandler() {
         }
       });
       pageCount++;
+      logger.debug(`Processing ${chunks.length} chunks for route: ${pageRoute}`);
+      for (let idx = 0; idx < chunks.length; idx++) {
+        const chunk = chunks[idx];
+        if (!chunk)
+          continue;
+        const bulkChunk = {
+          id: generateVectorId(pageRoute, idx),
+          route: pageRoute,
+          chunkIndex: idx,
+          content: chunk.content,
+          headers: chunk.metadata?.headers,
+          loc: chunk.metadata?.loc,
+          title,
+          description
+        };
+        await nuxt.hooks.callHook("ai-ready:chunk", {
+          chunk: bulkChunk,
+          route: pageRoute,
+          title,
+          description,
+          headings
+        });
+        bulkStream.write(`${JSON.stringify(bulkChunk)}
+`);
+        bulkStreamEntries++;
+      }
+      logger.debug(`Completed ${chunks.length} chunks for ${pageRoute}`);
       route.contents = markdown;
     });
     nitro.hooks.hook("prerender:done", async () => {
@@ -56,13 +95,13 @@ function setupPrerenderHandler() {
         return;
       }
       await writer.close();
+      if (bulkStream) {
+        bulkStream.end();
+        logger.info(`Bulk JSONL stream closed with ${bulkStreamEntries} chunks`);
+      }
       const llmsTxtPath = join(nitro.options.output.publicDir, "llms.txt");
       const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
-      const [llmsStats, llmsFullStats] = await Promise.all([
-        stat(llmsTxtPath),
-        stat(llmsFullTxtPath)
-      ]);
-      nitro._prerenderedRoutes.push(
+      const files = [
         {
           route: "/llms.txt",
           fileName: llmsTxtPath,
@@ -73,19 +112,29 @@ function setupPrerenderHandler() {
           fileName: llmsFullTxtPath,
           generateTimeMS: 0
         }
-      );
+      ];
+      if (bulkStream) {
+        files.push({
+          route: "/content.jsonl",
+          fileName: bulkPath,
+          generateTimeMS: 0
+        });
+      }
+      const [llmsStats, llmsFullStats, bulkStats] = await Promise.all([
+        stat(llmsTxtPath),
+        stat(llmsFullTxtPath),
+        bulkStream ? stat(bulkPath) : Promise.resolve(null)
+      ]);
+      nitro._prerenderedRoutes.push(...files);
       const elapsed = Date.now() - startTime;
       const llmsKb = (llmsStats.size / 1024).toFixed(2);
       const llmsFullKb = (llmsFullStats.size / 1024).toFixed(2);
-      logger.info(`Generated llms.txt (${llmsKb}kb) and llms-full.txt (${llmsFullKb}kb) from ${pageCount} pages in ${elapsed}ms`);
+      const bulkKb = bulkStats ? (bulkStats.size / 1024).toFixed(2) : "0";
+      logger.info(`Generated llms.txt (${llmsKb}kb), llms-full.txt (${llmsFullKb}kb), and content.jsonl (${bulkKb}kb) from ${pageCount} pages (${bulkStreamEntries} chunks) in ${elapsed}ms`);
     });
   });
 }
 
-function generateVectorId(route, chunkIdx) {
-  const hash = createHash("sha256").update(route).digest("hex").substring(0, 48);
-  return `${hash}-${chunkIdx}`;
-}
 const module = defineNuxtModule({
   meta: {
     name: "nuxt-ai-ready",
@@ -122,8 +171,8 @@ const module = defineNuxtModule({
     };
   },
   async setup(config, nuxt) {
-    const { resolve: resolve$1 } = createResolver(import.meta.url);
-    const { version } = await readPackageJSON(resolve$1("../package.json"));
+    const { resolve } = createResolver(import.meta.url);
+    const { version } = await readPackageJSON(resolve("../package.json"));
     logger.level = config.debug || nuxt.options.debug ? 4 : 3;
     if (config.enabled === false) {
       logger.debug("Module is disabled, skipping setup.");
@@ -131,15 +180,15 @@ const module = defineNuxtModule({
     }
     await installNuxtSiteConfig();
     nuxt.options.nitro.alias = nuxt.options.nitro.alias || {};
-    nuxt.options.alias["#ai-ready"] = resolve$1("./runtime");
+    nuxt.options.alias["#ai-ready"] = resolve("./runtime");
     if (!nuxt.options.mcp?.name) {
       nuxt.options.mcp = nuxt.options.mcp || {};
       nuxt.options.mcp.name = useSiteConfig().name;
     }
     nuxt.options.nitro.scanDirs = nuxt.options.nitro.scanDirs || [];
     nuxt.options.nitro.scanDirs.push(
-      resolve$1("./runtime/server/utils"),
-      resolve$1("./runtime/server/mcp")
+      resolve("./runtime/server/utils"),
+      resolve("./runtime/server/mcp")
     );
     if (typeof config.contentSignal === "object") {
       nuxt.options.robots.groups.push({
@@ -151,7 +200,7 @@ const module = defineNuxtModule({
     addTypeTemplate({
       filename: "module/nuxt-ai-ready.d.ts",
       getContents: (data) => {
-        const typesPath = relative(resolve$1(data.nuxt.options.rootDir, data.nuxt.options.buildDir, "module"), resolve$1("runtime/types"));
+        const typesPath = relative$1(resolve(data.nuxt.options.rootDir, data.nuxt.options.buildDir, "module"), resolve("runtime/types"));
         const nitroTypes = `  interface NitroRuntimeHooks {
     'ai-ready:markdown': (context: import('${typesPath}').MarkdownContext) => void | Promise<void>
     'ai-ready:mdreamConfig': (config: import('mdream').HTMLToMarkdownOptions) => void | Promise<void>
@@ -192,7 +241,7 @@ Returns JSONL (newline-delimited JSON) with all indexed content.`
     const hasMCP = hasNuxtModule("@nuxtjs/mcp-toolkit");
     if (hasMCP) {
       nuxt.hook("mcp:definitions:paths", (paths) => {
-        const mcpRuntimeDir = resolve$1("./runtime/server/mcp");
+        const mcpRuntimeDir = resolve("./runtime/server/mcp");
         paths.tools = paths.tools || [];
         paths.resources = paths.resources || [];
         paths.prompts = paths.prompts || [];
@@ -233,13 +282,13 @@ Returns JSONL (newline-delimited JSON) with all indexed content.`
     };
     addServerHandler({
       middleware: true,
-      handler: resolve$1("./runtime/server/middleware/mdream")
+      handler: resolve("./runtime/server/middleware/mdream")
     });
     if (nuxt.options.build) {
-      addPlugin({ mode: "server", src: resolve$1("./runtime/nuxt/plugins/prerender") });
+      addPlugin({ mode: "server", src: resolve("./runtime/nuxt/plugins/prerender") });
     }
     if (nuxt.options.dev) {
-      addServerHandler({ route: "/llms.txt", handler: resolve$1("./runtime/server/routes/llms.txt.get") });
+      addServerHandler({ route: "/llms.txt", handler: resolve("./runtime/server/routes/llms.txt.get") });
     }
     const isStatic = nuxt.options.nitro.static || nuxt.options._generate || false;
     if (isStatic || nuxt.options.nitro.prerender?.routes?.length) {
@@ -253,104 +302,6 @@ Returns JSONL (newline-delimited JSON) with all indexed content.`
         }
       };
     }
-    const isBuildMode = !nuxt.options._prepare && !nuxt.options.dev;
-    nuxt.hooks.hook("modules:done", () => {
-      nuxt.hook("nitro:init", async (nitro) => {
-        if (!isBuildMode) {
-          logger.debug("Dev mode: skipping llms.txt generation");
-          return;
-        }
-        if (config.bulkRoute === false) {
-          logger.debug("Bulk route disabled, skipping bulk generation");
-          return;
-        }
-        const bulkPath = resolve(nitro.options.output.dir, `public${config.bulkRoute}`);
-        let bulkStream = null;
-        let bulkStreamEntries = 0;
-        nitro.hooks.hook("prerender:route", async (route) => {
-          const isHtml = route.fileName?.endsWith(".html") && route.contents.startsWith("<!DOCTYPE html");
-          if (!isHtml || !route.contents) {
-            return;
-          }
-          if (typeof route._sitemap !== "undefined" && !route._sitemap) {
-            return;
-          }
-          if (isPathFile(route.route)) {
-            return;
-          }
-          let title = "";
-          let description = "";
-          const headings = [];
-          const extractPlugin = extractionPlugin({
-            title(el) {
-              title = el.textContent;
-            },
-            'meta[name="description"]': (el) => {
-              description = el.attributes.content || "";
-            },
-            "h1, h2, h3, h4, h5, h6": (el) => {
-              const text = el.textContent?.trim();
-              const level = el.name.toLowerCase();
-              if (text)
-                headings.push({ [level]: text });
-            }
-          });
-          const options = {
-            origin: useSiteConfig().url,
-            ...config.mdreamOptions || {}
-          };
-          options.plugins = [extractPlugin, ...options.plugins || []];
-          const chunksStream = htmlToMarkdownSplitChunksStream(route.contents, {
-            ...options,
-            headersToSplitOn: [TagIdMap.h1, TagIdMap.h2, TagIdMap.h3],
-            origin: useSiteConfig().url,
-            chunkSize: 256,
-            stripHeaders: false,
-            lengthFunction(text) {
-              return estimateTokenCount(text);
-            }
-          });
-          if (!bulkStream) {
-            mkdirSync(dirname(bulkPath), { recursive: true });
-            bulkStream = createWriteStream(bulkPath);
-            logger.info(`Bulk JSONL stream created at ${relative(nuxt.options.rootDir, bulkPath)}`);
-          }
-          logger.debug(`Processing chunks for route: ${route.route}`);
-          let idx = 0;
-          for await (const chunk of chunksStream) {
-            logger.debug(`  Chunk ${idx}: ${chunk.content.length} chars, headers: ${JSON.stringify(chunk.metadata?.headers)}`);
-            const bulkChunk = {
-              id: generateVectorId(route.route, idx),
-              route: route.route,
-              chunkIndex: idx,
-              content: chunk.content,
-              headers: chunk.metadata?.headers,
-              loc: chunk.metadata?.loc,
-              title,
-              description
-            };
-            await nuxt.hooks.callHook("ai-ready:chunk", {
-              chunk: bulkChunk,
-              route: route.route,
-              title,
-              description,
-              headings
-            });
-            bulkStream.write(`${JSON.stringify(bulkChunk)}
-`);
-            bulkStreamEntries++;
-            idx++;
-          }
-          logger.debug(`Completed ${idx} chunks for ${route.route}`);
-        });
-        nitro.hooks.hook("prerender:done", () => {
-          if (bulkStream) {
-            bulkStream.end();
-            logger.success(`Bulk JSONL exported ${bulkStreamEntries} entries.`);
-          }
-        });
-      });
-    });
   }
 });
 

package/dist/runtime/server/middleware/mdream.js

@@ -1,9 +1,11 @@
 import { withSiteUrl } from "#site-config/server/composables/utils";
 import { createError, defineEventHandler, getHeader, setHeader } from "h3";
-import { htmlToMarkdown } from "mdream";
+import { htmlToMarkdown, TagIdMap } from "mdream";
 import { extractionPlugin } from "mdream/plugins";
 import { withMinimalPreset } from "mdream/preset/minimal";
+import { htmlToMarkdownSplitChunksStream } from "mdream/splitter";
 import { useNitroApp, useRuntimeConfig } from "nitropack/runtime";
+import { estimateTokenCount } from "tokenx";
 import { logger } from "../logger.js";
 function shouldServeMarkdown(event) {
   const accept = getHeader(event, "accept") || "";
@@ -20,12 +22,19 @@ async function convertHtmlToMarkdown(html, url, config, route, event) {
   const nitroApp = useNitroApp();
   let title = "";
   let description = "";
+  const headings = [];
   const extractPlugin = extractionPlugin({
     title(el) {
       title = el.textContent;
     },
     'meta[name="description"]': (el) => {
       description = el.attributes.content || "";
+    },
+    "h1, h2, h3, h4, h5, h6": (el) => {
+      const text = el.textContent?.trim();
+      const level = el.name.toLowerCase();
+      if (text)
+        headings.push({ [level]: text });
     }
   });
   let options = {
@@ -51,7 +60,51 @@ async function convertHtmlToMarkdown(html, url, config, route, event) {
   };
   await nitroApp.hooks.callHook("ai-ready:markdown", context);
   markdown = context.markdown;
-  return { markdown, title, description };
+  return { markdown, title, description, headings };
+}
+async function convertHtmlToMarkdownChunks(html, url, config) {
+  let title = "";
+  let description = "";
+  const headings = [];
+  const extractPlugin = extractionPlugin({
+    title(el) {
+      title = el.textContent;
+    },
+    'meta[name="description"]': (el) => {
+      description = el.attributes.content || "";
+    },
+    "h1, h2, h3, h4, h5, h6": (el) => {
+      const text = el.textContent?.trim();
+      const level = el.name.toLowerCase();
+      if (text)
+        headings.push({ [level]: text });
+    }
+  });
+  let options = {
+    origin: url,
+    ...config.mdreamOptions
+  };
+  if (config.mdreamOptions?.preset === "minimal") {
+    options = withMinimalPreset(options);
+    options.plugins = [extractPlugin, ...options.plugins || []];
+  } else {
+    options.plugins = [extractPlugin, ...options.plugins || []];
+  }
+  const chunksStream = htmlToMarkdownSplitChunksStream(html, {
+    ...options,
+    headersToSplitOn: [TagIdMap.h1, TagIdMap.h2, TagIdMap.h3],
+    origin: url,
+    chunkSize: 256,
+    stripHeaders: false,
+    lengthFunction(text) {
+      return estimateTokenCount(text);
+    }
+  });
+  const chunks = [];
+  for await (const chunk of chunksStream) {
+    chunks.push(chunk);
+  }
+  return { chunks, title, description, headings };
 }
 export default defineEventHandler(async (event) => {
   let path = event.path;
@@ -112,6 +165,14 @@ export default defineEventHandler(async (event) => {
     }
     return;
   }
+  if (import.meta.prerender) {
+    const result2 = await convertHtmlToMarkdownChunks(
+      html,
+      withSiteUrl(event, path),
+      config
+    );
+    return JSON.stringify(result2);
+  }
   const result = await convertHtmlToMarkdown(
     html,
     withSiteUrl(event, path),
@@ -120,13 +181,10 @@ export default defineEventHandler(async (event) => {
     event
   );
   setHeader(event, "content-type", "text/markdown; charset=utf-8");
-  if (!import.meta.prerender && config.markdownCacheHeaders) {
+  if (config.markdownCacheHeaders) {
     const { maxAge, swr } = config.markdownCacheHeaders;
     const cacheControl = swr ? `public, max-age=${maxAge}, stale-while-revalidate=${maxAge}` : `public, max-age=${maxAge}`;
     setHeader(event, "cache-control", cacheControl);
   }
-  if (import.meta.prerender) {
-    return JSON.stringify(result);
-  }
   return result.markdown;
 });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",