nuxt-ai-ready 3.1.0 → 3.3.0

package/dist/chunks/agent-skills.mjs

@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto';
 import { realpath, readFile, readdir } from 'node:fs/promises';
 import { join, resolve, isAbsolute, relative, sep } from 'node:path';
 import { parseDocument } from 'yaml';
-import { A as AGENT_SKILLS_SCHEMA } from '../shared/nuxt-ai-ready.Dmvy7CIJ.mjs';
+import { A as AGENT_SKILLS_SCHEMA } from '../shared/nuxt-ai-ready.BwnSMqRn.mjs';
 import 'node:module';
 import 'node:url';
 import '@nuxt/kit';
@@ -11,6 +11,7 @@ import 'nuxt-site-config/kit';
 import 'nuxtseo-shared/kit';
 import '../shared/nuxt-ai-ready.B3hAuOLQ.mjs';
 import 'node:fs';
+import '../../dist/runtime/llms-txt-files.js';
 import '../../dist/runtime/route-path.js';
 import '../../dist/runtime/server/utils/sitemap-md.js';
 import '../../dist/runtime/cache-control.js';

package/dist/chunks/prerender.mjs

@@ -1,14 +1,16 @@
-import { mkdir, writeFile, appendFile, stat, readdir } from 'node:fs/promises';
-import { join, dirname, relative, resolve } from 'node:path';
+import { mkdir, writeFile, appendFile, readFile, stat, readdir } from 'node:fs/promises';
+import { resolve, relative, join, dirname } from 'node:path';
 import { useNuxt, getNitroVersion, hasNuxtModule, resolveFiles } from '@nuxt/kit';
 import { colorize } from 'consola/utils';
 import { resolveLocaleFromRoute } from 'nuxtseo-shared/i18n-runtime';
 import { createPrerenderFetch } from 'nuxtseo-shared/prerender';
 import { collectSitemap } from 'sitemapd/parse';
-import { withLeadingSlash, joinURL, withBase } from 'ufo';
-import { l as logger, M as MARKDOWN_LINK_AVAILABILITY_FILE } from '../shared/nuxt-ai-ready.Dmvy7CIJ.mjs';
-import { normalizePagePath, toMarkdownPath } from '../../dist/runtime/markdown-path.js';
-import { toDeployedRoute, toLogicalRoute } from '../../dist/runtime/route-path.js';
+import { parseURL, withLeadingSlash, joinURL, withBase } from 'ufo';
+import { l as logger, M as MARKDOWN_LINK_AVAILABILITY_FILE } from '../shared/nuxt-ai-ready.BwnSMqRn.mjs';
+import { toLogicalRoute, toDeployedRoute } from '../../dist/runtime/route-path.js';
+import { insertLlmsTxtDirective } from '../../dist/runtime/llms-txt-directive.js';
+import { absoluteSiteUrl } from '../../dist/runtime/markdown-links.js';
+import { normalizePagePath, hasMarkdownTwin, toMarkdownPath } from '../../dist/runtime/markdown-path.js';
 import { initSchema, computeContentHash, insertPage, queryAllPages, exportDbDump } from '../../dist/runtime/server/db/shared.js';
 import { buildLlmsFullTxtHeader, formatPageForLlmsFullTxt } from '../../dist/runtime/server/utils/llms-full.js';
 import { resolvePageUpdatedAt } from '../../dist/runtime/server/utils/page-updated-at.js';
@@ -21,6 +23,7 @@ import 'nuxt-site-config/kit';
 import 'nuxtseo-shared/kit';
 import '../shared/nuxt-ai-ready.B3hAuOLQ.mjs';
 import 'node:fs';
+import '../../dist/runtime/llms-txt-files.js';
 import '../../dist/runtime/cache-control.js';
 import '../../dist/runtime/server/utils/discovery-response.js';
 import 'confbox';
@@ -29,6 +32,33 @@ import '../../dist/runtime/server/utils/content-negotiation.js';
 import 'pathe';
 import 'nuxtseo-shared/i18n';
 
+function parsePrerenderRoute(input, baseURL) {
+  let pathname = parseURL(input).pathname;
+  try {
+    pathname = decodeURIComponent(pathname);
+  } catch {
+    return { _tag: "Invalid" };
+  }
+  while (true) {
+    if (pathname.includes("\\") || [...pathname].some((character) => character.charCodeAt(0) < 32 || character.charCodeAt(0) === 127) || pathname.split("/").some((segment) => segment === "." || segment === ".."))
+      return { _tag: "Invalid" };
+    const decoded = pathname.replace(/%([\da-f]{2})/gi, (_, byte) => String.fromCharCode(Number.parseInt(byte, 16)));
+    if (decoded === pathname)
+      break;
+    pathname = decoded;
+  }
+  return { _tag: "Valid", route: toLogicalRoute(input, baseURL) };
+}
+function resolvePrerenderFile(root, route) {
+  if (parsePrerenderRoute(route, "/")._tag === "Invalid")
+    throw new Error(`Unsafe prerender file path: ${route}`);
+  const destination = resolve(root, `.${route.startsWith("/") ? route : `/${route}`}`);
+  const child = relative(resolve(root), destination);
+  if (child === ".." || child.startsWith("../") || child.startsWith("..\\"))
+    throw new Error(`Prerender file path escapes its root: ${route}`);
+  return destination;
+}
+
 const PRERENDER_PAGE_TIMEOUT = 3e4;
 const RE_HTML_MD_EXT = /\.(html|md)$/;
 const RE_INDEX_SUFFIX = /\/index$/;
@@ -184,7 +214,32 @@ async function processMarkdownRoute(state, nuxt, route, parsed, lastmod, options
 async function processSitemapEntry(state, nuxt, nitro, entry) {
   const loc = typeof entry === "string" ? entry : entry.loc;
   const lastmod = typeof entry === "string" ? void 0 : entry.lastmod;
-  const route = toLogicalRoute(loc, nitro.options.baseURL);
+  const parsedRoute = parsePrerenderRoute(loc, nitro.options.baseURL);
+  if (parsedRoute._tag === "Invalid") {
+    logger.warn(`Skipping unsafe sitemap URL: ${loc}`);
+    return { crawled: false, skipped: true };
+  }
+  const route = parsedRoute.route;
+  if (!hasMarkdownTwin(route))
+    return { crawled: false, skipped: true };
+  if (nitro.options.static) {
+    for (const asset of nitro.options.publicAssets) {
+      const assetBase = asset.baseURL || "/";
+      if (!route.startsWith(assetBase))
+        continue;
+      const assetRoute = route.slice(assetBase.length);
+      for (const htmlRoute of [`${assetRoute}.html`, `${assetRoute}/index.html`]) {
+        const htmlPath = resolvePrerenderFile(asset.dir, htmlRoute);
+        const existing = await stat(htmlPath).catch((error) => {
+          if (error.code === "ENOENT")
+            return void 0;
+          throw error;
+        });
+        if (existing?.isFile())
+          return { crawled: false, skipped: true };
+      }
+    }
+  }
   if (route.split("/").some((segment) => segment.startsWith("_"))) {
     return { crawled: false, skipped: true };
   }
@@ -193,12 +248,24 @@ async function processSitemapEntry(state, nuxt, nitro, entry) {
   }
   const mdRoute = toMarkdownPath(route);
   const mdUrl = toDeployedRoute(mdRoute, nitro.options.baseURL);
+  const candidates = [route, toDeployedRoute(route, nitro.options.baseURL), mdRoute, mdUrl];
+  const ignored = nitro.options.prerender.ignore.some((pattern) => candidates.some((path) => {
+    if (typeof pattern === "string")
+      return path.startsWith(pattern);
+    if (typeof pattern === "function")
+      return pattern(path) === true;
+    return pattern.test(path);
+  }));
+  if (ignored)
+    return { crawled: false, skipped: true };
   logger.debug(`Fetching markdown for ${route} \u2192 ${mdUrl}`);
   const res = await crawlerFetch(state)(mdUrl, {
     headers: { "x-nitro-prerender": mdRoute },
     retry: 0,
     signal: AbortSignal.timeout(PRERENDER_PAGE_TIMEOUT)
   }).catch((err) => {
+    if (err?.response?.status === 404 || err?.response?.status === 410)
+      state.errorRoutes.add(route);
     if (err?.name === "TimeoutError" || err?.name === "AbortError")
       logger.warn(`Timeout (${PRERENDER_PAGE_TIMEOUT}ms) fetching markdown for ${route}`);
     else
@@ -215,6 +282,29 @@ async function processSitemapEntry(state, nuxt, nitro, entry) {
     return { crawled: false, skipped: false };
   }
   await processMarkdownRoute(state, nuxt, route, parsed, lastmod, { skipLlmsFullTxt: true });
+  if (nitro.options.static) {
+    for (const asset of nitro.options.publicAssets) {
+      const assetBase = asset.baseURL || "/";
+      if (!mdRoute.startsWith(assetBase))
+        continue;
+      const source = resolvePrerenderFile(asset.dir, mdRoute.slice(assetBase.length));
+      const existing = await stat(source).catch((error) => {
+        if (error.code === "ENOENT")
+          return void 0;
+        throw error;
+      });
+      if (existing?.isFile())
+        return { crawled: true, skipped: false };
+    }
+    const generated = { route: mdRoute, fileName: mdRoute, contents: JSON.stringify(parsed) };
+    await nitro.hooks.callHook("prerender:generate", generated, nitro);
+    if (generated.skip || generated.error)
+      return { crawled: true, skipped: false };
+    const filePath = resolvePrerenderFile(nitro.options.output.publicDir, mdRoute);
+    await mkdir(dirname(filePath), { recursive: true });
+    await writeFile(filePath, generated.contents || "", "utf8");
+    nitro._prerenderedRoutes.push(generated);
+  }
   return { crawled: true, skipped: false };
 }
 async function crawlSitemapEntries(state, nuxt, nitro, entries) {
@@ -235,18 +325,72 @@ async function crawlSitemapEntries(state, nuxt, nitro, entries) {
   logger.debug(`Sitemap crawl complete: ${crawled} crawled, ${skipped} skipped`);
   return crawled;
 }
-async function crawlSitemapContent(state, nuxt, nitro, sitemapContent) {
+async function crawlSitemapContent(state, nuxt, nitro, sitemapContent, seen = /* @__PURE__ */ new Set(), depth = 0, documents = /* @__PURE__ */ new Map()) {
   logger.debug(`Parsing sitemap XML (${sitemapContent.length} bytes)`);
   const result = await collectSitemap(sitemapContent);
-  if (result._tag !== "document" || result.document._tag !== "urlset") {
+  if (result._tag !== "document") {
     const issues = result.issues.map((issue) => issue.message).join("; ");
     logger.debug(`Skipping sitemap: ${issues || "document is not a URL set"}`);
     return 0;
   }
+  if (result.document._tag === "index") {
+    if (depth >= 3 || seen.size >= 100) {
+      logger.warn("Sitemap crawl reached its document limit.");
+      return 0;
+    }
+    let crawled = 0;
+    const origin = new URL(state.siteInfo?.url || "http://localhost").origin;
+    for (const entry of result.document.entries) {
+      if (parsePrerenderRoute(entry.loc, nitro.options.baseURL)._tag === "Invalid")
+        continue;
+      const url = new URL(entry.loc, origin);
+      if (url.origin !== origin || seen.has(url.href))
+        continue;
+      if (seen.size >= 100) {
+        logger.warn("Sitemap crawl reached its document limit.");
+        break;
+      }
+      seen.add(url.href);
+      const content = documents.get(toLogicalRoute(url.pathname, nitro.options.baseURL))?.() ?? await crawlerFetch(state)(`${url.pathname}${url.search}`, {
+        headers: { "x-nitro-prerender": url.pathname },
+        retry: 0,
+        signal: AbortSignal.timeout(PRERENDER_PAGE_TIMEOUT)
+      }).catch((error) => {
+        logger.warn(`Sitemap crawl failed for ${url.pathname}: ${String(error)}`);
+        return null;
+      });
+      if (content)
+        crawled += await crawlSitemapContent(state, nuxt, nitro, content, seen, depth + 1, documents);
+    }
+    return crawled;
+  }
   const urls = result.document.entries;
   logger.debug(`Found ${urls.length} URLs in sitemap`);
   return crawlSitemapEntries(state, nuxt, nitro, urls);
 }
+async function crawlConfiguredSitemaps(state, nuxt, nitro, sitemap, documents = /* @__PURE__ */ new Map()) {
+  const names = state.llmsTxtConfig?.sitemaps;
+  const singleName = (sitemap.sitemapName || "sitemap.xml").replace(/\.xml$/, "");
+  const roots = sitemap.sitemaps && typeof sitemap.sitemaps === "object" ? Object.entries(sitemap.sitemaps).flatMap(([key, definition]) => {
+    const name = (definition.sitemapName || key).replace(/\.xml$/, "");
+    if (key === "index" || names !== void 0 && !names.includes(name))
+      return [];
+    const route = definition._route || definition.route;
+    return [route ? toLogicalRoute(route, nitro.options.baseURL) : joinURL(sitemap.sitemapsPathPrefix ?? "/__sitemap__/", `${name}.xml`)];
+  }) : names !== void 0 && !names.includes(sitemap.sitemaps ? "chunks" : singleName) ? [] : [sitemap.sitemaps ? "/sitemap_index.xml" : `/${sitemap.sitemapName || "sitemap.xml"}`];
+  for (const root of roots) {
+    const content = documents.get(root)?.() ?? await crawlerFetch(state)(toDeployedRoute(root, nitro.options.baseURL), {
+      headers: { "x-nitro-prerender": root },
+      retry: 0,
+      signal: AbortSignal.timeout(PRERENDER_PAGE_TIMEOUT)
+    }).catch((error) => {
+      logger.warn(`Sitemap crawl failed for ${root}: ${String(error)}`);
+      return null;
+    });
+    if (content)
+      await crawlSitemapContent(state, nuxt, nitro, content, /* @__PURE__ */ new Set(), 0, documents);
+  }
+}
 function isNuxtGenerate() {
   return process.argv.includes("generate") || process.env.NUXT_GENERATE === "true" || process.env.prerender === "true";
 }
@@ -272,6 +416,7 @@ function detectSitemapPrerender(sitemapName = "sitemap.xml") {
   };
 }
 async function prerenderRoute(nitro, route, fetch) {
+  const filePath = resolvePrerenderFile(nitro.options.output.publicDir, route);
   const start = Date.now();
   const encodedRoute = encodeURI(route);
   const fetchUrl = withBase(encodedRoute, nitro.options.baseURL);
@@ -280,7 +425,6 @@ async function prerenderRoute(nitro, route, fetch) {
     retry: nitro.options.prerender.retry,
     retryDelay: nitro.options.prerender.retryDelay
   });
-  const filePath = join(nitro.options.output.publicDir, route);
   logger.debug(`Creating directory for prerender: ${dirname(filePath)}`);
   await mkdir(dirname(filePath), { recursive: true });
   const data = res._data;
@@ -302,7 +446,7 @@ async function prerenderRoute(nitro, route, fetch) {
 function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras) {
   const nuxt = useNuxt();
   nuxt.hooks.hook("nitro:init", async (nitro) => {
-    const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
+    const llmsFullTxtPath = options.llmsFullTxt === false ? void 0 : join(nitro.options.output.publicDir, "llms-full.txt");
     const state = createCrawlerState(
       dbPath,
       llmsFullTxtPath,
@@ -346,8 +490,10 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras)
         initPromise = initCrawler(state);
       await initPromise;
       const parsed = JSON.parse(route.contents || "{}");
-      await processMarkdownRoute(state, nuxt, pageRoute, parsed);
-      route.contents = options.sitemapMd === false ? parsed.markdown : appendSitemapSection(parsed.markdown, toDeployedRoute(SITEMAP_MD_ROUTE, nitro.options.baseURL));
+      if (!state.prerenderedRoutes.has(pageRoute))
+        await processMarkdownRoute(state, nuxt, pageRoute, parsed);
+      const withSitemap = options.sitemapMd === false ? parsed.markdown : appendSitemapSection(parsed.markdown, absoluteSiteUrl(toDeployedRoute(SITEMAP_MD_ROUTE, nitro.options.baseURL), siteInfo?.url));
+      route.contents = llmsTxtConfig?.directive?.markdown === false ? withSitemap : insertLlmsTxtDirective(withSitemap, absoluteSiteUrl(toDeployedRoute("/llms.txt", nitro.options.baseURL), siteInfo?.url));
       route.contentType = "text/markdown; charset=utf-8";
       state.totalProcessingTime += Date.now() - pageStartTime;
     });
@@ -409,10 +555,12 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras)
       }
       if (state.dbPath && state.llmsTxtConfig?.markdownLinks) {
         const availabilityFile = join(dirname(state.dbPath), MARKDOWN_LINK_AVAILABILITY_FILE);
+        const options2 = nitro.options;
+        const ignoreBaseDir = options2.srcDir || (typeof options2.serverDir === "string" ? options2.serverDir : void 0) || options2.rootDir;
         const paths = await findOutputMarkdownPaths(
           nitro.options.output.publicDir,
           nitro.options.baseURL,
-          nitro.options.srcDir,
+          ignoreBaseDir,
           nitro.options.ignore,
           nitro.options.publicAssets
         );
@@ -421,20 +569,46 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras)
           paths
         }), "utf-8");
       }
-      const llmsStats = await prerenderRoute(nitro, "/llms.txt", crawlerFetch(state));
-      const llmsFullStats = await stat(state.llmsFullTxtPath);
-      nitro._prerenderedRoutes.push({ route: "/llms-full.txt", fileName: "/llms-full.txt" });
+      let llmsSize = 0;
+      if (options.llmsTxt !== false) {
+        const seen = /* @__PURE__ */ new Set();
+        const pending = ["/llms.txt"];
+        while (pending.length) {
+          const route = pending.shift();
+          if (seen.has(route))
+            continue;
+          seen.add(route);
+          const stats = await prerenderRoute(nitro, route, crawlerFetch(state));
+          llmsSize += stats.size;
+          const body = await readFile(resolvePrerenderFile(nitro.options.output.publicDir, route), "utf-8");
+          for (const match of body.matchAll(/\]\(([^)]+)\)/g)) {
+            const origin = new URL(siteInfo?.url || "http://localhost").origin;
+            const parsedTarget = parsePrerenderRoute(match[1], nitro.options.baseURL);
+            if (parsedTarget._tag === "Invalid")
+              continue;
+            const target = new URL(match[1], origin);
+            if (target.origin !== origin)
+              continue;
+            const path = parsedTarget.route;
+            if (path.startsWith("/llms/") && path.endsWith(".txt"))
+              pending.push(path);
+          }
+        }
+      }
+      const llmsFullStats = state.llmsFullTxtPath ? await stat(state.llmsFullTxtPath) : { size: 0 };
+      if (state.llmsFullTxtPath)
+        nitro._prerenderedRoutes.push({ route: "/llms-full.txt", fileName: "/llms-full.txt" });
       if (options.sitemapMd !== false) {
         const sitemapMdStats = await prerenderRoute(nitro, SITEMAP_MD_ROUTE, crawlerFetch(state));
         logger.debug(`Wrote sitemap.md (${(sitemapMdStats.size / 1024).toFixed(1)}kb)`);
       }
       const kb = (b) => (b / 1024).toFixed(1);
-      const totalKb = kb(llmsStats.size + llmsFullStats.size);
+      const totalKb = kb(llmsSize + llmsFullStats.size);
       const dim = (s) => colorize("dim", s);
       const cyan = (s) => colorize("cyan", s);
       const timeStr = state.totalProcessingTime >= 100 ? ` in ${cyan(`${(state.totalProcessingTime / 1e3).toFixed(1)}s`)}` : "";
       logger.info(`Indexed ${cyan(String(state.prerenderedRoutes.size))} pages for llms.txt${timeStr} \u2192 ${cyan(`${totalKb}kb`)}`);
-      logger.info(dim(`  llms.txt: ${kb(llmsStats.size)}kb, llms-full.txt: ${kb(llmsFullStats.size)}kb`));
+      logger.info(dim(`  llms.txt files: ${kb(llmsSize)}kb, llms-full.txt: ${kb(llmsFullStats.size)}kb`));
     }
     const { useSitemapHook, usePrerenderHook } = detectSitemapPrerender();
     logger.debug(`Prerender hooks: sitemap=${useSitemapHook}, prerender=${usePrerenderHook}`);
@@ -442,8 +616,8 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras)
       nuxt.hooks.hook("sitemap:prerender:done", async (ctx) => (async () => {
         if (!state.initialized)
           return;
-        for (const sitemap of ctx.sitemaps)
-          await crawlSitemapContent(state, nuxt, nitro, sitemap.content);
+        const documents = new Map(ctx.sitemaps.map((sitemap) => [toLogicalRoute(sitemap.name, nitro.options.baseURL), () => sitemap.content]));
+        await crawlConfiguredSitemaps(state, nuxt, nitro, ctx.options, documents);
         await writeLlmsFiles();
         state.prerenderedRoutes.clear();
         if (state.db)
@@ -453,14 +627,7 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras)
       nitro.hooks.hook("prerender:done", async () => (async () => {
         if (!state.initialized)
           return;
-        const sitemapContent = await crawlerFetch(state)(toDeployedRoute("/sitemap.xml", nitro.options.baseURL), {
-          headers: { "x-nitro-prerender": "/sitemap.xml" },
-          signal: AbortSignal.timeout(PRERENDER_PAGE_TIMEOUT)
-        }).catch(() => {
-          return null;
-        });
-        if (sitemapContent)
-          await crawlSitemapContent(state, nuxt, nitro, sitemapContent);
+        await crawlConfiguredSitemaps(state, nuxt, nitro, nuxt.options.sitemap || {});
         await writeLlmsFiles();
         state.prerenderedRoutes.clear();
         if (state.db)

package/dist/module.d.mts

@@ -101,6 +101,7 @@ interface ModulePublicRuntimeConfig {
     version: string;
     sitemapMd: boolean;
     describedby: boolean;
+    llmsFullTxt?: boolean;
     mdreamOptions: ModuleOptions['mdreamOptions'];
     markdownCacheHeaders: Required<NonNullable<ModuleOptions['markdownCacheHeaders']>>;
     database: ResolvedDatabase;

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": "^4.6.0 || ^5.0.0-0"
   },
   "configKey": "aiReady",
-  "version": "3.1.0",
+  "version": "3.3.0",
   "builder": {
     "@nuxt/module-builder": "1.0.3",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -7,8 +7,9 @@ import '@nuxt/kit';
 import 'defu';
 import 'nuxt-site-config/kit';
 import 'nuxtseo-shared/kit';
-export { m as default } from './shared/nuxt-ai-ready.Dmvy7CIJ.mjs';
+export { m as default } from './shared/nuxt-ai-ready.BwnSMqRn.mjs';
 import './shared/nuxt-ai-ready.B3hAuOLQ.mjs';
+import '../dist/runtime/llms-txt-files.js';
 import '../dist/runtime/route-path.js';
 import '../dist/runtime/server/utils/sitemap-md.js';
 import '../dist/runtime/server/utils/discovery-response.js';

package/dist/runtime/llms-txt-directive.d.ts

@@ -0,0 +1,15 @@
+/** The line that points a reader of one page at the whole site's index. */
+export declare function llmsTxtDirective(llmsTxtUrl: string): string;
+/**
+ * Add the llms.txt directive near the top of a page's Markdown, after any
+ * frontmatter so YAML parsers still find it first. An agent that lands on one
+ * page then knows where the full index is.
+ */
+export declare function insertLlmsTxtDirective(markdown: string, llmsTxtUrl: string): string;
+/**
+ * The HTML equivalent: a link at the top of the body, hidden visually but
+ * kept for screen readers and agents that read the raw HTML.
+ */
+export declare function llmsTxtDirectiveHtml(llmsTxtPath: string): string;
+/** Remove the injected HTML link, so a page's Markdown carries the directive once. */
+export declare function stripLlmsTxtDirectiveHtml(html: string): string;

package/dist/runtime/llms-txt-directive.js

@@ -0,0 +1,23 @@
+const RE_FRONTMATTER = /^\uFEFF?---[\t ]*\r?\n(?:[\s\S]*?\r?\n)?---[\t ]*(?:\r?\n|$)/;
+export function llmsTxtDirective(llmsTxtUrl) {
+  return `> Index of every page on this site: [llms.txt](${llmsTxtUrl})`;
+}
+export function insertLlmsTxtDirective(markdown, llmsTxtUrl) {
+  const directive = llmsTxtDirective(llmsTxtUrl);
+  const frontmatter = markdown.match(RE_FRONTMATTER)?.[0] ?? "";
+  const body = markdown.slice(frontmatter.length).replace(/^(?:\r?\n)+/, "");
+  if (body === directive || body.startsWith(`${directive}
+`) || body.startsWith(`${directive}\r
+`))
+    return markdown;
+  return `${frontmatter}${frontmatter ? "\n" : ""}${directive}
+
+${body}`;
+}
+const RE_DIRECTIVE_HTML = /<a [^>]*data-llms-txt-directive[^>]*>[^<]*<\/a>/g;
+export function llmsTxtDirectiveHtml(llmsTxtPath) {
+  return `<a href="${llmsTxtPath}" tabindex="-1" style="position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0" data-llms-txt-directive>Index of every page on this site: llms.txt</a>`;
+}
+export function stripLlmsTxtDirectiveHtml(html) {
+  return html.replace(RE_DIRECTIVE_HTML, "");
+}

package/dist/runtime/llms-txt-files.d.ts

@@ -0,0 +1,30 @@
+import type { LlmsTxtConfig, LlmsTxtSection } from './types.js';
+export interface LlmsTxtPage {
+    pathname: string;
+    href?: string;
+    title?: string;
+    description?: string;
+    locale?: string;
+}
+export declare function llmsTxtSectionPath(section: LlmsTxtSection): string;
+export type ParsedLlmsTxtConfig = {
+    _tag: 'Valid';
+    config: LlmsTxtConfig;
+} | {
+    _tag: 'Invalid';
+    message: string;
+};
+/** Parse the public configuration once, before Nitro registers routes. */
+export declare function parseLlmsTxtConfig(input: unknown): ParsedLlmsTxtConfig;
+interface LlmsTxtFilesInput {
+    rootContent?: string;
+    header: string;
+    config: LlmsTxtConfig;
+    sections: LlmsTxtSection[];
+    pages: LlmsTxtPage[];
+    sitemapPages: ReadonlyMap<string, LlmsTxtPage[]>;
+    resolveUrl: (path: string) => string;
+}
+/** A bounded tree of file lists. Every page appears once; every file is reachable from the root. */
+export declare function createLlmsTxtFiles(input: LlmsTxtFilesInput): Map<string, string>;
+export {};

package/dist/runtime/llms-txt-files.js

@@ -0,0 +1,141 @@
+import { formatLlmsTxtPageLink, normalizeLlmsTxtConfig } from "./llms-txt-format.js";
+const FILE_PATH = /^\/llms\/[a-z0-9][\w-]*\.txt$/i;
+const SITEMAP_NAME = /^[a-z0-9][\w-]*$/i;
+export function llmsTxtSectionPath(section) {
+  const name = Array.isArray(section.sitemap) ? section.sitemap[0] : section.sitemap;
+  return section.path || `/llms/${name}.txt`;
+}
+export function parseLlmsTxtConfig(input) {
+  const invalid = (message) => ({ _tag: "Invalid", message });
+  if (!input || typeof input !== "object" || Array.isArray(input))
+    return invalid("llmsTxt must be an object or false.");
+  const config = input;
+  if (config.pages !== void 0 && config.pages !== false && config.pages !== "sitemap" && config.pages !== "all")
+    return invalid("llmsTxt.pages must be sitemap, all, or false.");
+  if (config.maxLength !== void 0 && (!Number.isSafeInteger(config.maxLength) || config.maxLength < 1024))
+    return invalid("llmsTxt.maxLength must be an integer of at least 1024.");
+  if (config.sitemaps !== void 0 && (!Array.isArray(config.sitemaps) || config.sitemaps.some((name) => typeof name !== "string" || !SITEMAP_NAME.test(name))))
+    return invalid("llmsTxt.sitemaps must contain sitemap names.");
+  if (config.sections !== void 0 && !Array.isArray(config.sections))
+    return invalid("llmsTxt.sections must be an array.");
+  const paths = /* @__PURE__ */ new Set();
+  for (const section of config.sections || []) {
+    if (!section || typeof section !== "object" || typeof section.title !== "string")
+      return invalid("Each llmsTxt section needs a title.");
+    if (section.sitemap === void 0) {
+      if (section.path !== void 0)
+        return invalid("A section path requires a sitemap.");
+      continue;
+    }
+    const names = Array.isArray(section.sitemap) ? section.sitemap : [section.sitemap];
+    if (!names.length || names.some((name) => typeof name !== "string" || !SITEMAP_NAME.test(name)))
+      return invalid("Each section sitemap must contain a sitemap name.");
+    const path = llmsTxtSectionPath(section);
+    if (!FILE_PATH.test(path))
+      return invalid("A section path must use /llms/<name>.txt.");
+    if (paths.has(path))
+      return invalid(`Duplicate llmsTxt section path: ${path}.`);
+    paths.add(path);
+  }
+  return { _tag: "Valid", config };
+}
+export function createLlmsTxtFiles(input) {
+  const files = /* @__PURE__ */ new Map();
+  const maxLength = input.config.maxLength ?? 8e4;
+  const link = (title, path, description) => formatLlmsTxtPageLink({ pathname: path, href: input.resolveUrl(path), title, description });
+  const render = (header, title, lines) => `${header.trim()}
+
+## ${title.replace(/\s+/g, " ").trim()}
+
+${lines.join("\n")}
+`;
+  function tree(path, prefix, header, title, lines, level = 0) {
+    const body = render(header, title, lines);
+    if (body.length <= maxLength) {
+      files.set(path, body);
+      return;
+    }
+    const overhead = render(header, title, []).length;
+    const chunks = [];
+    let chunk = [];
+    let length = overhead;
+    for (const line of lines) {
+      if (overhead + line.length + 1 > maxLength)
+        throw new Error("An llms.txt entry exceeds llmsTxt.maxLength. Shorten the entry or increase the limit.");
+      if (length + line.length + 1 > maxLength) {
+        chunks.push(chunk);
+        chunk = [];
+        length = overhead;
+      }
+      chunk.push(line);
+      length += line.length + 1;
+    }
+    if (chunk.length)
+      chunks.push(chunk);
+    if (chunks.length >= lines.length)
+      throw new Error("llmsTxt.maxLength cannot fit an index. Increase the limit.");
+    const links = chunks.map((entries, index) => {
+      const child = `/llms/_${prefix}-${level}-${index + 1}.txt`;
+      files.set(child, render(header, title, entries));
+      return link(`${title} ${index + 1}`, child);
+    });
+    tree(path, prefix, header, title, links, level + 1);
+  }
+  const sections = [];
+  const assigned = /* @__PURE__ */ new Set();
+  for (const [index, section] of input.sections.entries()) {
+    if (!section.sitemap) {
+      sections.push(section);
+      continue;
+    }
+    const names = Array.isArray(section.sitemap) ? section.sitemap : [section.sitemap];
+    const pages2 = /* @__PURE__ */ new Map();
+    for (const name of names) {
+      for (const page of input.sitemapPages.get(name) || []) {
+        if (!assigned.has(page.pathname))
+          pages2.set(page.pathname, page);
+      }
+    }
+    const entries = [...(section.links || []).map((entry) => formatLlmsTxtPageLink({ pathname: entry.href, ...entry })), ...[...pages2.values()].map(formatLlmsTxtPageLink)];
+    if (!entries.length)
+      continue;
+    for (const pathname of pages2.keys())
+      assigned.add(pathname);
+    const path = llmsTxtSectionPath(section);
+    const description = Array.isArray(section.description) ? section.description.join(" ") : section.description;
+    const preamble = normalizeLlmsTxtConfig({ notes: description });
+    tree(path, `section-${index + 1}`, `${input.header.trim()}
+
+${preamble}
+
+${link("llms.txt", "/llms.txt")}`, section.title, entries);
+    sections.push({ ...section, sitemap: void 0, path: void 0, links: [{ title: section.title, href: input.resolveUrl(path), description }] });
+  }
+  const rootHeader = `${input.header.trim()}
+
+${normalizeLlmsTxtConfig({ ...input.config, sections })}${input.rootContent ? `
+
+${input.rootContent}` : ""}`;
+  const pages = input.pages.filter((page) => !assigned.has(page.pathname)).map(formatLlmsTxtPageLink);
+  if (!pages.length && rootHeader.length + 1 <= maxLength) {
+    files.set("/llms.txt", `${rootHeader.trim()}
+`);
+  } else {
+    const inline = render(rootHeader, "Pages", pages);
+    if (inline.length <= maxLength) {
+      files.set("/llms.txt", inline);
+    } else {
+      tree("/llms/_pages.txt", "pages", input.header, "Pages", pages);
+      const root = `${rootHeader.trim()}
+
+## Pages
+
+${link("Pages", "/llms/_pages.txt")}
+`;
+      if (root.length > maxLength)
+        throw new Error("llms.txt sections exceed llmsTxt.maxLength. Shorten the sections or increase the limit.");
+      files.set("/llms.txt", root);
+    }
+  }
+  return files;
+}

package/dist/runtime/llms-txt-utils.d.ts

@@ -1,4 +1,5 @@
 import type { H3Event } from '#nuxtseo/h3';
-import { normalizeLlmsTxtConfig } from './llms-txt-format.js';
-export { normalizeLlmsTxtConfig };
+export { normalizeLlmsTxtConfig } from './llms-txt-format.js';
+/** One discovery snapshot supplies the root and every bounded child file. */
+export declare function buildLlmsTxtFiles(event: H3Event): Promise<Record<string, string>>;
 export declare function buildLlmsTxt(event: H3Event): Promise<string>;