nuxt-ai-ready 0.3.8 → 0.4.1

package/dist/module.mjs

@@ -1,19 +1,15 @@
-import { mkdir, stat, open, access, appendFile } from 'node:fs/promises';
-import { dirname, join } from 'node:path';
-import { useLogger, useNuxt, defineNuxtModule, createResolver, addTypeTemplate, hasNuxtModule, addServerHandler, addPlugin, extendPages } from '@nuxt/kit';
+import { appendFile, mkdir, writeFile, stat, access } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { useLogger, useNuxt, defineNuxtModule, createResolver, addTypeTemplate, hasNuxtModule, addServerHandler, addPlugin } from '@nuxt/kit';
 import defu from 'defu';
 import { useSiteConfig, installNuxtSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
-import { relative } from 'pathe';
 import { readPackageJSON } from 'pkg-types';
 import * as p from '@clack/prompts';
 import { $fetch } from 'ofetch';
 import { isTest, isCI } from 'std-env';
-import { createHash } from 'node:crypto';
-import { mkdirSync, createWriteStream } from 'node:fs';
-import { encodeLines } from '@toon-format/toon';
-import { createLlmsTxtStream } from 'mdream/llms-txt';
-import { createStorage } from 'unstorage';
-import fsDriver from 'unstorage/drivers/fs';
+import { parseSitemapXml } from '@nuxtjs/sitemap/utils';
+import { colorize } from 'consola/utils';
+import { withBase } from 'ufo';
 
 const logger = useLogger("nuxt-ai-ready");
 
@@ -68,350 +64,320 @@ function hookNuxtSeoProLicense() {
   }
 }
 
-function createContentHashManager(options) {
-  const { storagePath, debug = false } = options;
-  let storage;
-  let manifest = {
-    pages: {},
-    version: "1"
-  };
-  async function initStorage() {
-    await mkdir(dirname(storagePath), { recursive: true });
-    storage = createStorage({
-      driver: fsDriver({ base: dirname(storagePath) })
-    });
+function normalizeLink(link) {
+  const parts = [];
+  parts.push(`- [${link.title}](${link.href})`);
+  if (link.description)
+    parts.push(`  ${link.description}`);
+  return parts.join("\n");
+}
+function normalizeSection(section) {
+  const parts = [];
+  parts.push(`## ${section.title}`);
+  parts.push("");
+  if (section.description) {
+    const descriptions = Array.isArray(section.description) ? section.description : [section.description];
+    parts.push(...descriptions);
+    parts.push("");
   }
-  function hashContent(markdown) {
-    return createHash("sha256").update(markdown).digest("hex");
+  if (section.links?.length)
+    parts.push(...section.links.map(normalizeLink));
+  return parts.join("\n");
+}
+function normalizeLlmsTxtConfig(config) {
+  const parts = [];
+  if (config.sections?.length)
+    parts.push(...config.sections.map(normalizeSection));
+  if (config.notes) {
+    parts.push("## Notes");
+    parts.push("");
+    const notes = Array.isArray(config.notes) ? config.notes : [config.notes];
+    parts.push(...notes);
   }
-  async function getManifest() {
-    if (!storage) {
-      await initStorage();
-    }
-    const stored = await storage.getItem("content-hashes.json");
-    if (stored) {
-      manifest = stored;
-      if (debug) {
-        logger.debug(`Loaded manifest with ${Object.keys(manifest.pages).length} pages`);
-      }
-    } else {
-      if (debug) {
-        logger.debug("No existing manifest found, starting fresh");
-      }
+  return parts.join("\n\n");
+}
+function createCrawlerState(pageDataPath, llmsFullTxtPath, siteInfo, llmsTxtConfig) {
+  return {
+    prerenderedRoutes: /* @__PURE__ */ new Set(),
+    totalProcessingTime: 0,
+    initialized: false,
+    jsonlInitialized: false,
+    pageDataPath,
+    llmsFullTxtPath,
+    siteInfo,
+    llmsTxtConfig
+  };
+}
+function buildLlmsFullTxtHeader(siteInfo, llmsTxtConfig) {
+  const parts = [];
+  parts.push(`# ${siteInfo?.name || siteInfo?.url || "Site"}`);
+  if (siteInfo?.description)
+    parts.push(`
+> ${siteInfo.description}`);
+  if (siteInfo?.url)
+    parts.push(`
+Canonical Origin: ${siteInfo.url}`);
+  parts.push("");
+  if (llmsTxtConfig) {
+    const normalizedContent = normalizeLlmsTxtConfig(llmsTxtConfig);
+    if (normalizedContent) {
+      parts.push(normalizedContent);
+      parts.push("");
     }
-    return manifest;
   }
-  async function saveManifest() {
-    if (!storage) {
-      await initStorage();
-    }
-    await storage.setItem("content-hashes.json", manifest);
-    if (debug) {
-      logger.debug(`Saved manifest with ${Object.keys(manifest.pages).length} pages`);
-    }
+  parts.push("## Pages\n\n");
+  return parts.join("\n");
+}
+async function initCrawler(state) {
+  if (state.initialized)
+    return;
+  if (state.pageDataPath) {
+    await mkdir(dirname(state.pageDataPath), { recursive: true });
+    await writeFile(state.pageDataPath, "", "utf-8");
+    state.jsonlInitialized = true;
+    logger.debug(`Crawler initialized with JSONL at ${state.pageDataPath}`);
   }
-  function updatePageHash(route, markdown, previousManifest) {
-    const contentHash = hashContent(markdown);
-    const now = (/* @__PURE__ */ new Date()).toISOString();
-    const existing = previousManifest.pages[route];
-    let result;
-    if (!existing) {
-      result = {
-        contentHash,
-        updatedAt: now,
-        firstSeenAt: now
-      };
-      if (debug) {
-        logger.debug(`New page detected: ${route}`);
-      }
-    } else if (existing.contentHash !== contentHash) {
-      result = {
-        contentHash,
-        updatedAt: now,
-        firstSeenAt: existing.firstSeenAt
-      };
-      if (debug) {
-        logger.debug(`Content changed: ${route}`);
-      }
-    } else {
-      result = {
-        contentHash: existing.contentHash,
-        updatedAt: existing.updatedAt,
-        firstSeenAt: existing.firstSeenAt
-      };
-      if (debug) {
-        logger.debug(`Content unchanged: ${route}`);
-      }
-    }
-    manifest.pages[route] = result;
-    return result;
+  if (state.llmsFullTxtPath) {
+    await mkdir(dirname(state.llmsFullTxtPath), { recursive: true });
+    const header = buildLlmsFullTxtHeader(state.siteInfo, state.llmsTxtConfig);
+    await writeFile(state.llmsFullTxtPath, header, "utf-8");
+    logger.debug(`llms-full.txt initialized at ${state.llmsFullTxtPath}`);
+  }
+  state.initialized = true;
+}
+function flattenHeadings(headings) {
+  return (headings || []).map((h) => Object.entries(h).map(([tag, text]) => `${tag}:${text}`).join("")).join("|");
+}
+function stripFrontmatter(markdown) {
+  return markdown.replace(/^---\n[\s\S]*?\n---\n*/, "");
+}
+function normalizeHeadings(markdown) {
+  return markdown.replace(/^(#{1,6})\s+(.+)$/gm, (_, hashes, text) => {
+    const level = hashes.length;
+    return `h${level}. ${text}`;
+  });
+}
+function formatPageForLlmsFullTxt(route, title, description, markdown, siteUrl) {
+  const canonicalUrl = siteUrl ? `${siteUrl.replace(/\/$/, "")}${route}` : route;
+  const heading = title && title !== route ? `### ${title}` : `### ${route}`;
+  let content = stripFrontmatter(markdown);
+  content = normalizeHeadings(content);
+  const parts = [heading, ""];
+  parts.push(`Source: ${canonicalUrl}`);
+  if (description)
+    parts.push(`Description: ${description}`);
+  parts.push("");
+  if (content.trim()) {
+    parts.push(content.trim());
+    parts.push("");
+  }
+  parts.push("---");
+  parts.push("");
+  return `${parts.join("\n")}
+`;
+}
+async function processMarkdownRoute(state, nuxt, route, parsed, lastmod, options) {
+  const { markdown, title, description, headings, updatedAt: metaUpdatedAt } = parsed;
+  let updatedAt = (lastmod instanceof Date ? lastmod.toISOString() : lastmod) || (/* @__PURE__ */ new Date()).toISOString();
+  if (metaUpdatedAt) {
+    const parsedDate = new Date(metaUpdatedAt);
+    if (!Number.isNaN(parsedDate.getTime()))
+      updatedAt = parsedDate.toISOString();
   }
-  function setPageTimestamp(route, markdown, timestamp, previousManifest) {
-    const contentHash = hashContent(markdown);
-    const existing = previousManifest.pages[route];
-    manifest.pages[route] = {
-      contentHash,
-      updatedAt: timestamp,
-      firstSeenAt: existing?.firstSeenAt || timestamp
+  await nuxt.hooks.callHook("ai-ready:page:markdown", { route, markdown, title, description, headings });
+  if (state.jsonlInitialized && state.pageDataPath) {
+    const pageData = {
+      route,
+      title,
+      description,
+      headings: flattenHeadings(headings),
+      updatedAt,
+      markdown
     };
-    if (debug) {
-      logger.debug(`Manual timestamp set for ${route}: ${timestamp}`);
+    await appendFile(state.pageDataPath, `${JSON.stringify(pageData)}
+`, "utf-8");
+  }
+  state.prerenderedRoutes.add(route);
+}
+async function crawlSitemapEntries(state, nuxt, nitro, entries) {
+  logger.debug(`Crawling ${entries.length} sitemap entries`);
+  let crawled = 0;
+  let skipped = 0;
+  for (const entry of entries) {
+    const loc = typeof entry === "string" ? entry : entry.loc;
+    const lastmod = typeof entry === "string" ? void 0 : entry.lastmod;
+    const route = loc.startsWith("http") ? new URL(loc).pathname : loc;
+    if (route.split("/").some((segment) => segment.startsWith("_"))) {
+      skipped++;
+      continue;
     }
+    if (state.prerenderedRoutes.has(route)) {
+      skipped++;
+      continue;
+    }
+    const mdRoute = route === "/" ? "/index.md" : `${route}.md`;
+    const mdUrl = withBase(mdRoute, nitro.options.baseURL);
+    logger.debug(`Fetching markdown for ${route} \u2192 ${mdUrl}`);
+    const res = await globalThis.$fetch(mdUrl, {
+      headers: { "x-nitro-prerender": mdRoute }
+    }).catch((err) => {
+      logger.debug(`Failed to fetch ${mdUrl}: ${err.message}`);
+      return null;
+    });
+    if (!res)
+      continue;
+    const parsed = JSON.parse(res);
+    await processMarkdownRoute(state, nuxt, route, parsed, lastmod);
+    crawled++;
   }
+  logger.debug(`Sitemap crawl complete: ${crawled} crawled, ${skipped} skipped (already indexed)`);
+  return crawled;
+}
+async function crawlSitemapContent(state, nuxt, nitro, sitemapContent) {
+  logger.debug(`Parsing sitemap XML (${sitemapContent.length} bytes)`);
+  const result = await parseSitemapXml(sitemapContent);
+  const urls = result?.urls || [];
+  logger.debug(`Found ${urls.length} URLs in sitemap`);
+  return crawlSitemapEntries(state, nuxt, nitro, urls);
+}
+function isNuxtGenerate() {
+  return process.argv.includes("generate") || process.env.NUXT_GENERATE === "true" || process.env.prerender === "true";
+}
+function resolveNitroPreset() {
+  return process.env.NITRO_PRESET || process.env.SERVER_PRESET;
+}
+function includesSitemapRoot(sitemapName, routes) {
+  return routes.some((r) => r === `/${sitemapName}` || r.startsWith(`/${sitemapName}/`));
+}
+function detectSitemapPrerender(sitemapName = "sitemap.xml") {
+  const nuxt = useNuxt();
+  const prerenderedRoutes = nuxt.options.nitro.prerender?.routes || [];
+  const hasSitemapModule = nuxt.options._installedModules?.some(
+    (m) => m.meta?.name === "@nuxtjs/sitemap"
+  );
+  let prerenderSitemap = hasSitemapModule || isNuxtGenerate() || includesSitemapRoot(sitemapName, prerenderedRoutes);
+  if (resolveNitroPreset() === "vercel-edge")
+    prerenderSitemap = true;
+  const hasPrerender = !!(nuxt.options.nitro.prerender?.routes?.length || nuxt.options.nitro.prerender?.crawlLinks);
+  const shouldHookIntoPrerender = prerenderSitemap || hasPrerender;
+  logger.debug(`Sitemap detection: module=${hasSitemapModule}, generate=${isNuxtGenerate()}, routes=${includesSitemapRoot(sitemapName, prerenderedRoutes)}`);
   return {
-    getManifest,
-    saveManifest,
-    hashContent,
-    updatePageHash,
-    setPageTimestamp
+    useSitemapHook: prerenderSitemap,
+    usePrerenderHook: shouldHookIntoPrerender && !prerenderSitemap
   };
 }
-
-function generateVectorId(route, chunkIdx) {
-  const hash = createHash("sha256").update(route).digest("hex").substring(0, 8);
-  return `${hash}-${chunkIdx}`;
-}
-async function updateFirstLine(filePath, newFirstLine) {
-  const fh = await open(filePath, "r+");
-  try {
-    const buffer = Buffer.alloc(1024);
-    await fh.read(buffer, 0, 1024, 0);
-    const content = buffer.toString("utf-8");
-    const firstLineEnd = content.indexOf("\n");
-    const oldFirstLine = content.substring(0, firstLineEnd);
-    const paddedLine = newFirstLine.padEnd(oldFirstLine.length, " ");
-    await fh.write(paddedLine, 0, "utf-8");
-  } finally {
-    await fh.close();
-  }
+async function prerenderRoute(nitro, route) {
+  const start = Date.now();
+  const encodedRoute = encodeURI(route);
+  const fetchUrl = withBase(encodedRoute, nitro.options.baseURL);
+  const res = await globalThis.$fetch.raw(fetchUrl, {
+    headers: { "x-nitro-prerender": encodedRoute },
+    retry: nitro.options.prerender.retry,
+    retryDelay: nitro.options.prerender.retryDelay
+  });
+  const filePath = join(nitro.options.output.publicDir, route);
+  await mkdir(dirname(filePath), { recursive: true });
+  const data = res._data;
+  if (data === void 0)
+    throw new Error(`No data returned from '${fetchUrl}'`);
+  await writeFile(filePath, data, "utf8");
+  const _route = {
+    route,
+    fileName: filePath,
+    generateTimeMS: Date.now() - start
+  };
+  nitro._prerenderedRoutes.push(_route);
+  return stat(filePath);
 }
-function setupPrerenderHandler(llmsTxtConfig, timestampsConfig) {
+function setupPrerenderHandler(pageDataPath, siteInfo, llmsTxtConfig) {
   const nuxt = useNuxt();
   nuxt.hooks.hook("nitro:init", async (nitro) => {
-    let writer = null;
-    let chunksStream = null;
-    let pagesStream = null;
-    let chunksProcessed = 0;
-    let pageCount = 0;
-    const startTime = Date.now();
-    const pagesChunksPath = join(nitro.options.output.publicDir, "llms-full.toon");
-    const pagesPath = join(nitro.options.output.publicDir, "llms.toon");
-    let contentHashManager = null;
-    let previousManifest = null;
-    if (timestampsConfig?.enabled) {
-      const manifestPath = join(
-        nuxt.options.rootDir,
-        timestampsConfig.manifestPath || "node_modules/.cache/nuxt-seo/ai-index/content-hashes.json"
-      );
-      contentHashManager = createContentHashManager({
-        storagePath: manifestPath,
-        debug: !!nuxt.options.debug
-      });
-    }
+    const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
+    const state = createCrawlerState(pageDataPath, llmsFullTxtPath, siteInfo, llmsTxtConfig);
+    let initPromise = null;
     nitro.hooks.hook("prerender:generate", async (route) => {
-      if (route.route === "/sitemap.xml" && contentHashManager) {
-        await contentHashManager.saveManifest();
-        logger.debug("Saved content hash manifest before sitemap generation");
-      }
-      if (!route.fileName?.endsWith(".md")) {
+      if (!route.fileName?.endsWith(".md"))
         return;
-      }
       let pageRoute = route.route.replace(/\.md$/, "");
-      if (pageRoute === "/index") {
+      if (pageRoute === "/index")
         pageRoute = "/";
-      }
-      if (!writer) {
-        const siteConfig = useSiteConfig();
-        const stream = createLlmsTxtStream({
-          siteName: siteConfig.name || siteConfig.url,
-          description: siteConfig.description,
-          origin: siteConfig.url,
-          generateFull: true,
-          outputDir: nitro.options.output.publicDir,
-          sections: llmsTxtConfig.sections,
-          notes: llmsTxtConfig.notes
-        });
-        writer = stream.getWriter();
-        if (contentHashManager && !previousManifest) {
-          previousManifest = await contentHashManager.getManifest();
-        }
-        mkdirSync(dirname(pagesChunksPath), { recursive: true });
-        mkdirSync(dirname(pagesPath), { recursive: true });
-        chunksStream = createWriteStream(pagesChunksPath, { encoding: "utf-8" });
-        chunksStream.write("pageChunks[999999]{id,route,content}:\n");
-        pagesStream = createWriteStream(pagesPath, { encoding: "utf-8" });
-        pagesStream.write("pages[999999]{route,title,description,headings,chunkIds,updatedAt}:\n");
-      }
-      const { markdown, chunks, title, description, headings, updatedAt: metaUpdatedAt } = JSON.parse(route.contents || "{}");
-      let pageTimestamp = {
-        updatedAt: (/* @__PURE__ */ new Date()).toISOString()
-      };
-      let usedMetaTimestamp = false;
+      const pageStartTime = Date.now();
+      if (!initPromise)
+        initPromise = initCrawler(state);
+      await initPromise;
+      const parsed = JSON.parse(route.contents || "{}");
+      const { markdown, title, description, headings, updatedAt: metaUpdatedAt } = parsed;
+      let updatedAt = (/* @__PURE__ */ new Date()).toISOString();
       if (metaUpdatedAt) {
         const parsedDate = new Date(metaUpdatedAt);
-        if (!Number.isNaN(parsedDate.getTime())) {
-          pageTimestamp.updatedAt = parsedDate.toISOString();
-          usedMetaTimestamp = true;
-          if (contentHashManager && previousManifest) {
-            contentHashManager.setPageTimestamp(pageRoute, markdown, pageTimestamp.updatedAt, previousManifest);
-          }
-        }
-      }
-      if (!usedMetaTimestamp && contentHashManager && previousManifest) {
-        pageTimestamp = contentHashManager.updatePageHash(
-          pageRoute,
-          markdown,
-          previousManifest
-        );
+        if (!Number.isNaN(parsedDate.getTime()))
+          updatedAt = parsedDate.toISOString();
       }
-      await writer.write({
-        url: pageRoute,
+      await nuxt.hooks.callHook("ai-ready:page:markdown", {
+        route: pageRoute,
+        markdown,
         title,
-        content: markdown,
-        metadata: {
-          description,
-          title
-        }
+        description,
+        headings
       });
-      pageCount++;
-      logger.debug(`Processing ${chunks.length} chunks for route: ${pageRoute}`);
-      const chunkIds = [];
-      for (let idx = 0; idx < chunks.length; idx++) {
-        const chunk = chunks[idx];
-        if (!chunk)
-          continue;
-        const chunkId = generateVectorId(pageRoute, idx);
-        chunkIds.push(chunkId);
-        const bulkChunk = {
-          id: chunkId,
-          route: pageRoute,
-          content: chunk.content
-        };
-        await nuxt.hooks.callHook("ai-ready:chunk", {
-          chunk: bulkChunk,
+      if (state.jsonlInitialized && state.pageDataPath) {
+        const pageData = {
           route: pageRoute,
           title,
           description,
-          headings: Object.entries(headings).flatMap(
-            ([tag, texts]) => texts.map((text) => ({ [tag]: text }))
-          )
-        });
-        if (chunksStream) {
-          const lines = Array.from(encodeLines({ pageChunks: [bulkChunk] }));
-          if (lines[1]) {
-            chunksStream.write(`${lines[1]}
-`);
-          }
-        }
-        chunksProcessed++;
+          headings: flattenHeadings(headings),
+          updatedAt,
+          markdown
+        };
+        await appendFile(state.pageDataPath, `${JSON.stringify(pageData)}
+`, "utf-8");
       }
-      logger.debug(`Completed ${chunks.length} chunks for ${pageRoute}`);
-      const pageDoc = {
-        route: pageRoute,
-        title,
-        description,
-        // Convert headings object to readable string format (h1:Title|h2:Subtitle,...)
-        headings: headings && Object.keys(headings).length ? Object.entries(headings).flatMap(
-          ([tag, texts]) => texts.map((text) => `${tag}:${text}`)
-        ).join("|") : "",
-        // Join chunkIds array to comma-separated string
-        chunkIds: chunkIds.join(","),
-        updatedAt: pageTimestamp.updatedAt
-      };
-      if (pagesStream) {
-        const lines = Array.from(encodeLines({ pages: [pageDoc] }));
-        if (lines[1]) {
-          pagesStream.write(`${lines[1]}
-`);
-        }
+      if (state.llmsFullTxtPath) {
+        const pageContent = formatPageForLlmsFullTxt(pageRoute, title, description, markdown, state.siteInfo?.url);
+        await appendFile(state.llmsFullTxtPath, pageContent, "utf-8");
       }
+      state.prerenderedRoutes.add(pageRoute);
       route.contents = markdown;
+      state.totalProcessingTime += Date.now() - pageStartTime;
     });
-    nitro.hooks.hook("prerender:done", async () => {
-      if (!writer) {
-        return;
-      }
-      await writer.close();
-      if (chunksStream) {
-        await new Promise((resolve, reject) => {
-          chunksStream.on("error", reject);
-          chunksStream.on("finish", resolve);
-          chunksStream.end();
-        });
-      }
-      if (pagesStream) {
-        await new Promise((resolve, reject) => {
-          pagesStream.on("error", reject);
-          pagesStream.on("finish", resolve);
-          pagesStream.end();
-        });
-      }
-      await updateFirstLine(pagesChunksPath, `pageChunks[${chunksProcessed}]{id,route,content}:`);
-      await updateFirstLine(pagesPath, `pages[${pageCount}]{route,title,description,headings,chunkIds,updatedAt}:`);
-      if (contentHashManager) {
-        await contentHashManager.saveManifest();
-        logger.debug("Saved content hash manifest");
-      }
-      logger.info(`Wrote llms-full.toon with ${chunksProcessed} chunks`);
-      logger.info(`Wrote llms.toon with ${pageCount} pages`);
-      const llmsTxtPath = join(nitro.options.output.publicDir, "llms.txt");
-      const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
-      const files = [
-        {
-          route: "/llms.txt",
-          fileName: llmsTxtPath,
-          generateTimeMS: 0
-        },
-        {
-          route: "/llms-full.txt",
-          fileName: llmsFullTxtPath,
-          generateTimeMS: 0
-        },
-        {
-          route: "/llms-full.toon",
-          fileName: pagesChunksPath,
-          generateTimeMS: 0
-        },
-        {
-          route: "/llms.toon",
-          fileName: pagesPath,
-          generateTimeMS: 0
-        }
-      ];
-      const [llmsStats, llmsFullStats, pagesChunksStats, pagesStats] = await Promise.all([
-        stat(llmsTxtPath),
-        stat(llmsFullTxtPath),
-        stat(pagesChunksPath),
-        stat(pagesPath)
-      ]);
-      nitro._prerenderedRoutes.push(...files);
-      const elapsed = Date.now() - startTime;
-      const llmsKb = (llmsStats.size / 1024).toFixed(2);
-      const llmsFullKb = (llmsFullStats.size / 1024).toFixed(2);
-      const pagesChunksKb = (pagesChunksStats.size / 1024).toFixed(2);
-      const pagesKb = (pagesStats.size / 1024).toFixed(2);
-      logger.info(`Generated llms.txt (${llmsKb}kb), llms-full.txt (${llmsFullKb}kb), llms-full.toon (${pagesChunksKb}kb), and llms.toon (${pagesKb}kb) from ${pageCount} pages (${chunksProcessed} chunks) in ${elapsed}ms`);
-    });
+    async function writeLlmsFiles() {
+      const llmsStats = await prerenderRoute(nitro, "/llms.txt");
+      const llmsFullStats = await stat(state.llmsFullTxtPath);
+      const kb = (b) => (b / 1024).toFixed(1);
+      const totalKb = kb(llmsStats.size + llmsFullStats.size);
+      const dim = (s) => colorize("dim", s);
+      const cyan = (s) => colorize("cyan", s);
+      const timeStr = state.totalProcessingTime >= 100 ? ` in ${cyan(`${(state.totalProcessingTime / 1e3).toFixed(1)}s`)}` : "";
+      logger.info(`Indexed ${cyan(String(state.prerenderedRoutes.size))} pages for llms.txt${timeStr} \u2192 ${cyan(`${totalKb}kb`)}`);
+      logger.info(dim(`  llms.txt: ${kb(llmsStats.size)}kb, llms-full.txt: ${kb(llmsFullStats.size)}kb`));
+    }
+    const { useSitemapHook, usePrerenderHook } = detectSitemapPrerender();
+    logger.debug(`Prerender hooks: sitemap=${useSitemapHook}, prerender=${usePrerenderHook}`);
+    if (useSitemapHook) {
+      nuxt.hooks.hook("sitemap:prerender:done", async (ctx) => {
+        if (!state.initialized)
+          return;
+        for (const sitemap of ctx.sitemaps)
+          await crawlSitemapContent(state, nuxt, nitro, sitemap.content);
+        await writeLlmsFiles();
+        state.prerenderedRoutes.clear();
+      });
+    } else if (usePrerenderHook) {
+      nitro.hooks.hook("prerender:done", async () => {
+        if (!state.initialized)
+          return;
+        const sitemapContent = await globalThis.$fetch("/sitemap.xml", {
+          headers: { "x-nitro-prerender": "/sitemap.xml" }
+        }).catch(() => null);
+        if (sitemapContent)
+          await crawlSitemapContent(state, nuxt, nitro, sitemapContent);
+        await writeLlmsFiles();
+        state.prerenderedRoutes.clear();
+      });
+    }
   });
 }
 
-function createPagesPromise(nuxt = useNuxt()) {
-  return new Promise((resolve) => {
-    nuxt.hooks.hook("modules:done", () => {
-      if (typeof nuxt.options.pages === "boolean" && !nuxt.options.pages || typeof nuxt.options.pages === "object" && !nuxt.options.pages.enabled) {
-        return resolve([]);
-      }
-      extendPages(resolve);
-    });
-  });
-}
-function flattenPages(pages, parent = "") {
-  return pages.flatMap((page) => {
-    const path = parent + page.path;
-    const current = { path, name: page.name, meta: page.meta };
-    return page.children?.length ? [current, ...flattenPages(page.children, path)] : [current];
-  });
-}
 const module$1 = defineNuxtModule({
   meta: {
     name: "nuxt-ai-ready",
@@ -424,6 +390,9 @@ const module$1 = defineNuxtModule({
     "@nuxtjs/robots": {
       version: ">=5.6.0"
     },
+    "@nuxtjs/sitemap": {
+      version: ">=7"
+    },
     "nuxt-site-config": {
       version: ">=3"
     },
@@ -444,10 +413,8 @@ const module$1 = defineNuxtModule({
         // 1 hour
         swr: true
       },
-      timestamps: {
-        enabled: false,
-        manifestPath: "node_modules/.cache/nuxt-seo/ai-index/content-hashes.json"
-      }
+      cacheMaxAgeSeconds: 600
+      // 10 minutes
     };
   },
   async setup(config, nuxt) {
@@ -470,16 +437,9 @@ const module$1 = defineNuxtModule({
     nuxt.options.nitro.scanDirs.push(
       resolve("./runtime/server/utils")
     );
-    const pagesPromise = createPagesPromise(nuxt);
-    nuxt.hooks.hook("nitro:config", (nitroConfig) => {
-      nitroConfig.virtual = nitroConfig.virtual || {};
-      nitroConfig.virtual["#ai-ready/routes.mjs"] = async () => {
-        const pages = await pagesPromise;
-        const routes = flattenPages(pages);
-        return `export default ${JSON.stringify(routes)}`;
-      };
-    });
     if (typeof config.contentSignal === "object") {
+      nuxt.options.robots = nuxt.options.robots || {};
+      nuxt.options.robots.groups = nuxt.options.robots.groups || [];
       nuxt.options.robots.groups.push({
         userAgent: "*",
         contentUsage: [`train-ai=${config.contentSignal.aiTrain ? "y" : "n"}`],
@@ -487,54 +447,52 @@ const module$1 = defineNuxtModule({
       });
     }
     addTypeTemplate({
-      filename: "module/nuxt-ai-ready.d.ts",
-      getContents: (data) => {
-        const typesPath = relative(resolve(data.nuxt.options.rootDir, data.nuxt.options.buildDir, "module"), resolve("runtime/types"));
-        const nitroTypes = `  interface NitroRuntimeHooks {
-    'ai-ready:markdown': (context: import('${typesPath}').MarkdownContext) => void | Promise<void>
-    'ai-ready:mdreamConfig': (config: import('mdream').HTMLToMarkdownOptions) => void | Promise<void>
-  }`;
-        return `// Generated by nuxt-ai-ready
+      filename: "types/nuxt-ai-ready.d.ts",
+      getContents: () => `// Generated by nuxt-ai-ready
+import type { MarkdownContext } from 'nuxt-ai-ready'
+import type { HTMLToMarkdownOptions } from 'mdream'
+
 declare module 'nitropack/types' {
-${nitroTypes}
+  interface NitroRuntimeHooks {
+    'ai-ready:markdown': (context: MarkdownContext) => void | Promise<void>
+    'ai-ready:mdreamConfig': (config: HTMLToMarkdownOptions) => void | Promise<void>
+  }
+}
+
+declare module '#ai-ready-virtual/read-page-data.mjs' {
+  export function readPageDataFromFilesystem(): Promise<Array<{
+    route: string
+    title: string
+    description: string
+    headings: string
+    updatedAt: string
+    markdown: string
+  }> | null>
 }
 
-declare module 'nitropack' {
-${nitroTypes}
+declare module '#ai-ready-virtual/page-data.mjs' {
+  export const pages: never[]
 }
 
 export {}
-`;
-      }
-    }, {
-      nitro: true
-    });
+`
+    }, { nitro: true });
     const defaultLlmsTxtSections = [];
-    const pagesRoute = withSiteUrl("llms.toon");
-    const pagesChunksRoute = withSiteUrl("llms-full.toon");
+    const llmsFullRoute = withSiteUrl("llms-full.txt");
     defaultLlmsTxtSections.push({
       title: "LLM Resources",
       links: [
         {
-          title: "Pages Minimal",
-          href: pagesRoute,
-          description: `Page-level metadata in TOON format (token-efficient JSON encoding, see https://toonformat.dev). Contains: route, title, description, headings, chunkIds. Use with llms-full.toon for complete content. Fields: { route, title, description, headings, chunkIds }.
-
-  <code lang="bash">curl "${pagesRoute}"</code>`
-        },
-        {
-          title: "Page Chunks",
-          href: pagesChunksRoute,
-          description: `Individual content chunks in TOON format for RAG/embeddings. Contains: id, route, content. Fields: { id, route, content }. Join with llms.toon using route to get title/description/headings metadata. Chunk index inferred from id suffix (e.g., "hash-0", "hash-1").
-
-  <code lang="bash">curl "${pagesChunksRoute}"</code>`
+          title: "Full Content",
+          href: llmsFullRoute,
+          description: "Complete page content in markdown format."
         }
       ]
     });
     const hasMCP = hasNuxtModule("@nuxtjs/mcp-toolkit");
     if (hasMCP) {
       nuxt.hook("mcp:definitions:paths", (paths) => {
-        const mcpRuntimeDir = resolve(`./runtime/server/mcp/${nuxt.options.dev ? "dev" : "prod"}`);
+        const mcpRuntimeDir = resolve(`./runtime/server/mcp`);
         const mcpConfig = config.mcp || {};
         if (mcpConfig.tools !== false)
           (paths.tools ||= []).push(`${mcpRuntimeDir}/tools`);
@@ -569,43 +527,74 @@ export {}
     await nuxt.callHook("ai-ready:llms-txt", llmsTxtPayload);
     mergedLlmsTxt.sections = llmsTxtPayload.sections;
     mergedLlmsTxt.notes = llmsTxtPayload.notes.length > 0 ? llmsTxtPayload.notes : void 0;
-    const timestampsManifestPath = config.timestamps?.enabled ? join(nuxt.options.rootDir, config.timestamps.manifestPath || "node_modules/.cache/nuxt-seo/ai-index/content-hashes.json") : void 0;
+    const prerenderCacheDir = join(nuxt.options.rootDir, "node_modules/.cache/nuxt-seo/ai-ready/routes");
+    const pageDataPath = join(nuxt.options.buildDir, ".data/ai-ready/page-data.jsonl");
+    nuxt.hooks.hook("nitro:config", (nitroConfig) => {
+      nitroConfig.virtual = nitroConfig.virtual || {};
+      nitroConfig.virtual["#ai-ready-virtual/read-page-data.mjs"] = `
+import { readFile } from 'node:fs/promises'
+
+export async function readPageDataFromFilesystem() {
+  if (!import.meta.prerender) {
+    return null
+  }
+  const data = await readFile(${JSON.stringify(pageDataPath)}, 'utf-8').catch(() => null)
+  if (!data) return []
+  return data.trim().split('\\n').filter(Boolean).map(line => JSON.parse(line))
+}
+`;
+      nitroConfig.virtual["#ai-ready-virtual/page-data.mjs"] = `export const pages = []`;
+    });
     nuxt.options.runtimeConfig["nuxt-ai-ready"] = {
       version: version || "0.0.0",
       debug: config.debug || false,
-      hasSitemap: hasNuxtModule("@nuxtjs/sitemap"),
       mdreamOptions: config.mdreamOptions || {},
       markdownCacheHeaders: defu(config.markdownCacheHeaders, {
         maxAge: 3600,
         swr: true
       }),
       llmsTxt: mergedLlmsTxt,
-      timestampsManifestPath
+      cacheMaxAgeSeconds: config.cacheMaxAgeSeconds ?? 600,
+      prerenderCacheDir
     };
-    if (config.timestamps?.enabled && hasNuxtModule("@nuxtjs/sitemap")) {
-      nuxt.hook("nitro:config", (nitroConfig) => {
-        nitroConfig.plugins = nitroConfig.plugins || [];
-        nitroConfig.plugins.push(resolve("./runtime/server/plugins/sitemap-lastmod"));
-      });
-    }
     addServerHandler({
       middleware: true,
-      handler: resolve("./runtime/server/middleware/mdream")
+      handler: resolve("./runtime/server/middleware/markdown.prerender")
+    });
+    addServerHandler({
+      middleware: true,
+      handler: resolve("./runtime/server/middleware/markdown")
     });
     if (nuxt.options.build) {
-      addPlugin({ mode: "server", src: resolve("./runtime/nuxt/plugins/prerender") });
+      addPlugin({
+        mode: "server",
+        src: resolve("./runtime/nuxt/plugins/md-hints.prerender")
+      });
     }
     addServerHandler({ route: "/llms.txt", handler: resolve("./runtime/server/routes/llms.txt.get") });
-    addServerHandler({ route: "/llms-full.txt", handler: resolve("./runtime/server/routes/llms.txt.get") });
+    addServerHandler({ route: "/llms-full.txt", handler: resolve("./runtime/server/routes/llms-full.txt.get") });
     const isStatic = nuxt.options.nitro.static || nuxt.options._generate || false;
-    if (isStatic || nuxt.options.nitro.prerender?.routes?.length) {
-      setupPrerenderHandler(mergedLlmsTxt, config.timestamps);
+    const hasPrerenderedRoutes = nuxt.options.nitro.prerender?.routes?.length;
+    const isSPA = nuxt.options.ssr === false;
+    if (!nuxt.options.dev && !nuxt.options._prepare) {
+      if (isSPA && !hasPrerenderedRoutes) {
+        logger.warn("SPA mode detected without prerendering. llms-full.txt will not be generated.");
+        logger.warn("For full functionality, enable SSR or prerender routes.");
+      } else if (!isStatic && !hasPrerenderedRoutes) {
+        logger.info("SSR-only mode: llms-full.txt requires prerendering. Runtime markdown conversion available.");
+      }
+    }
+    if (isStatic || hasPrerenderedRoutes) {
+      const siteConfig = useSiteConfig();
+      setupPrerenderHandler(pageDataPath, {
+        name: siteConfig.name,
+        url: siteConfig.url,
+        description: siteConfig.description
+      }, mergedLlmsTxt);
     }
     nuxt.options.nitro.routeRules = nuxt.options.nitro.routeRules || {};
     nuxt.options.nitro.routeRules["/llms.txt"] = { headers: { "Content-Type": "text/plain; charset=utf-8" } };
     nuxt.options.nitro.routeRules["/llms-full.txt"] = { headers: { "Content-Type": "text/plain; charset=utf-8" } };
-    nuxt.options.nitro.routeRules["/llms.toon"] = { headers: { "Content-Type": "text/toon; charset=utf-8" } };
-    nuxt.options.nitro.routeRules["/llms-full.toon"] = { headers: { "Content-Type": "text/toon; charset=utf-8" } };
     nuxt.hooks.hook("nitro:build:before", (nitro) => {
       nitro.hooks.hook("compiled", async () => {
         const headersPath = join(nitro.options.output.publicDir, "_headers");