nuxt-ai-ready 3.1.0 → 3.3.0

package/dist/runtime/server/utils/markdown-request.js

@@ -1,7 +1,7 @@
 import { negotiateContent, parseAcceptHeader } from "@mdream/js/negotiate";
 import { getBotInfo } from "@nuxtjs/robots/util";
 import { getHeaders } from "#nuxtseo/h3";
-import { isReservedPath } from "../../markdown-path.js";
+import { hasMarkdownTwin, isReservedPath } from "../../markdown-path.js";
 const RE_MD_EXT = /\.md$/;
 export function toMarkdownRequest(event, isPrerender = !!import.meta.prerender) {
   return { path: event.path, headers: getHeaders(event), isPrerender };
@@ -38,10 +38,7 @@ export function getRequestRenderInfo(request, mode = { _tag: "runtime", contentN
   const isExplicit = originalPath.endsWith(".md");
   if (isPrerender && !isExplicit)
     return null;
-  const lastSegment = originalPath.split("/").pop() || "";
-  const hasExtension = lastSegment.includes(".");
-  const extension = hasExtension ? lastSegment.substring(lastSegment.lastIndexOf(".")) : "";
-  if (hasExtension && extension !== ".md")
+  if (!isExplicit && !hasMarkdownTwin(originalPath))
     return null;
   const negotiation = isPrerender ? "markdown" : mode.contentNegotiation ? negotiateRequestRepresentation(request, mode.botNegotiation) : "html";
   if (isExplicit) {

package/dist/runtime/server/utils/sitemap.d.ts

@@ -53,5 +53,6 @@ export declare function fetchSitemapByRoute(event: H3Event | undefined, route: s
  * Fetch all URLs from all sitemaps (or single sitemap if not multi-sitemap)
  * Used for backwards compatibility and llms.txt generation
  */
+export declare function fetchSitemapUrlGroups(event: H3Event, names?: readonly string[]): Promise<Map<string, SitemapUrl[]>>;
 export declare function fetchSitemapUrls(event: H3Event): Promise<SitemapUrl[]>;
 export {};

package/dist/runtime/server/utils/sitemap.js

@@ -1,5 +1,7 @@
 import { createSitemapReader } from "sitemapd";
 import { parseSitemap } from "sitemapd/parse";
+import staticSitemapConfig from "#ai-ready-virtual/sitemap-config.mjs";
+import { getRequestURL } from "#nuxtseo/h3";
 import { useRuntimeConfig } from "#nuxtseo/nitro";
 import { logger } from "../logger.js";
 import { getCfEnv, hasAssets } from "./cloudflare.js";
@@ -19,17 +21,18 @@ const SITEMAP_READER_LIMITS = {
 const MAX_SITEMAP_CRAWL_ROUNDS = 10;
 export function getSitemapsFromConfig(event) {
   const runtimeConfig = useRuntimeConfig(event);
-  const sitemapConfig = runtimeConfig.sitemap;
+  const sitemapConfig = { ...staticSitemapConfig, ...runtimeConfig.sitemap };
   if (!sitemapConfig?.sitemaps)
     return [];
   const sitemaps = [];
   for (const [key, sitemap] of Object.entries(sitemapConfig.sitemaps)) {
     if (key === "index")
       continue;
-    if (sitemap._route) {
+    const route = sitemap._route || sitemap.route || (key === "chunks" ? sitemapConfig.sitemaps.index?._route || sitemapConfig.sitemaps.index?.route : void 0);
+    if (route) {
       sitemaps.push({
-        name: sitemap.sitemapName || key,
-        route: sitemap._route
+        name: (sitemap.sitemapName || key).replace(/\.xml$/, ""),
+        route: route.startsWith("/") ? route : `/${route}`
       });
     }
   }
@@ -305,6 +308,7 @@ export async function crawlSitemapByRoute(event, input) {
   const usePublicAsset = config.sitemapPrerendered && hasAssets(event);
   const configuredSiteUrl = createUniversalContext(event).siteUrl ?? SITEMAP_FALLBACK_ORIGIN;
   const siteUrl = parseAbsoluteUrl(configuredSiteUrl);
+  const requestOrigin = import.meta.dev && event ? getRequestURL(event, { xForwardedHost: true, xForwardedProto: true }).origin : void 0;
   if (siteUrl._tag === "error") {
     return {
       _tag: "failed",
@@ -346,7 +350,7 @@ export async function crawlSitemapByRoute(event, input) {
       const target = parseAbsoluteUrl(request.url);
       if (target._tag === "error")
         return { _tag: "deny", reason: `Invalid sitemap target: ${request.url}` };
-      if (target.value.origin !== siteUrl.value.origin) {
+      if (target.value.origin !== siteUrl.value.origin && target.value.origin !== requestOrigin) {
         return {
           _tag: "deny",
           reason: `Sitemap target origin ${target.value.origin} differs from ${siteUrl.value.origin}`
@@ -433,17 +437,21 @@ export async function fetchSitemapByRoute(event, route) {
   const result = await crawlSitemapByRoute(event, { route });
   return result._tag === "complete" ? { urls: result.urls } : { urls: result.urls, error: result.error };
 }
-export async function fetchSitemapUrls(event) {
+export async function fetchSitemapUrlGroups(event, names) {
   const sitemaps = getSitemapsFromConfig(event);
+  const groups = /* @__PURE__ */ new Map();
   if (sitemaps.length > 0) {
     logger.debug(`[sitemap] Multi-sitemap mode: ${sitemaps.length} sitemaps`);
-    const allUrls = [];
     for (const sitemap of sitemaps) {
+      if (names !== void 0 && !names.includes(sitemap.name))
+        continue;
       const { urls: urls2 } = await fetchSitemapByRoute(event, sitemap.route);
-      allUrls.push(...urls2);
+      groups.set(sitemap.name, [...groups.get(sitemap.name) || [], ...urls2]);
     }
-    return allUrls;
+    return groups;
   }
+  if (names !== void 0 && !names.includes("sitemap"))
+    return groups;
   logger.debug(`[sitemap] Single sitemap mode: /sitemap.xml`);
   const { urls, error } = await fetchSitemapByRoute(event, "/sitemap.xml");
   if (error) {
@@ -451,5 +459,9 @@ export async function fetchSitemapUrls(event) {
   } else if (urls.length === 0) {
     logger.warn("Sitemap is empty - add routes to sitemap or configure sitemap.sources in nuxt.config");
   }
-  return urls;
+  groups.set("sitemap", urls);
+  return groups;
+}
+export async function fetchSitemapUrls(event) {
+  return [...(await fetchSitemapUrlGroups(event)).values()].flat();
 }

package/dist/runtime/server/utils.js

@@ -1,5 +1,6 @@
 import { htmlToMarkdown } from "mdream";
 import { useNitroApp } from "#nuxtseo/nitro";
+import { stripLlmsTxtDirectiveHtml } from "../llms-txt-directive.js";
 import { hoistMetaDescription } from "./utils/frontmatter.js";
 const RE_NBSP = /\u00A0/g;
 function normalizeWhitespace(text) {
@@ -75,7 +76,7 @@ export async function convertHtmlToMarkdown(html, url, mdreamOptions, opts) {
   await nitroApp.hooks.callHook("ai-ready:mdreamConfig", options);
   const context = {
     html,
-    markdown: hoistMetaDescription(htmlToMarkdown(html, options)),
+    markdown: hoistMetaDescription(htmlToMarkdown(stripLlmsTxtDirectiveHtml(html), options)),
     route: opts.hooks.route,
     title: meta.title,
     description: meta.description,

package/dist/runtime/types.d.ts

@@ -185,7 +185,9 @@ export interface ModuleOptions {
     /**
      * Structured llms.txt configuration
      */
-    llmsTxt?: LlmsTxtConfig;
+    llmsTxt?: LlmsTxtConfig | false;
+    /** Generate and serve llms-full.txt. Set false to omit the export and its discovery link. */
+    llmsFullTxt?: boolean;
     /**
      * Generate a Markdown sitemap at `/sitemap.md` from indexed pages and end
      * every Markdown page with a `## Sitemap` section linking to it.
@@ -512,6 +514,10 @@ export interface LlmsTxtLink {
  * Section in llms.txt
  */
 export interface LlmsTxtSection {
+    /** Generate a separate file from these sitemap names. Chunked sitemaps retain their configured name. */
+    sitemap?: string | string[];
+    /** Generated file path. Defaults to /llms/<sitemap>.txt. */
+    path?: string;
     /** The title of the section */
     title: string;
     /** Description rendered in the heading-free preamble, or in each optional link note (can be array for multiple paragraphs) */
@@ -525,13 +531,35 @@ export interface LlmsTxtSection {
  * Structured llms.txt configuration
  */
 export interface LlmsTxtConfig {
+    /** Page admission. Sitemap URLs are authoritative by default. Use all to include stored pages outside the sitemap. */
+    pages?: 'sitemap' | 'all' | false;
+    /** Only read these sitemap names. An empty array disables sitemap page discovery. */
+    sitemaps?: string[];
+    /** Maximum characters per generated file. Large lists become linked files. Minimum 1024. Default 80000. */
+    maxLength?: number;
     /**
      * Rewrite automatically generated page links to their Markdown representation
      * when the final static output contains the file or the deployment retains
      * the runtime Markdown handler.
-     * @default false
+     * @default true
      */
     markdownLinks?: boolean;
+    /**
+     * Point each page at llms.txt, so an agent that lands on one page finds the
+     * whole site's index.
+     */
+    directive?: {
+        /**
+         * Add a blockquote line after the frontmatter of every page's `.md` response.
+         * @default true
+         */
+        markdown?: boolean;
+        /**
+         * Add a visually hidden link to `/llms.txt` at the top of every HTML page's body.
+         * @default false
+         */
+        html?: boolean;
+    };
     /** The sections of the documentation */
     sections?: LlmsTxtSection[];
     /** Additional heading-free preamble notes rendered before file-list sections */

package/dist/shared/{nuxt-ai-ready.Dmvy7CIJ.mjs → nuxt-ai-ready.BwnSMqRn.mjs}

@@ -1,5 +1,5 @@
 import { createHash, randomBytes } from 'node:crypto';
-import { readFile, mkdir, writeFile, access } from 'node:fs/promises';
+import { readFile, access, mkdir, writeFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import { dirname, basename, join, resolve, relative } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
@@ -8,6 +8,7 @@ import defu$1, { defu } from 'defu';
 import { useSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
 import { resolveNuxtContentVersion, renderNitroTypeAugmentations, setupRuntimeAliases, setupNitroRuntimeCompatibility } from 'nuxtseo-shared/kit';
 import { r as readPackageMetadata, a as resolvePackageMetadataPath } from './nuxt-ai-ready.B3hAuOLQ.mjs';
+import { parseLlmsTxtConfig } from '../../dist/runtime/llms-txt-files.js';
 import { toDeployedRoute } from '../../dist/runtime/route-path.js';
 import { SITEMAP_MD_ROUTE } from '../../dist/runtime/server/utils/sitemap-md.js';
 import { publicCacheControl, mergeVaryHeader } from '../../dist/runtime/cache-control.js';
@@ -210,7 +211,11 @@ export {}
   });
   addTemplate({
     filename: "types/ai-ready-virtual.d.ts",
-    getContents: () => `declare module '#ai-ready-virtual/read-page-data.mjs' {
+    getContents: () => `declare module '#ai-ready-virtual/sitemap-config.mjs' {
+  const config: { sitemaps?: Record<string, { sitemapName?: string, route?: string, _route?: string }> }
+  export default config
+}
+declare module '#ai-ready-virtual/read-page-data.mjs' {
   export function readMarkdownLinkAvailabilityFromFilesystem(): Promise<{
     runtimeMarkdownAvailable: boolean
     paths: string[]
@@ -666,18 +671,42 @@ function cloudflareStaticFallbackPaths(existing, pages, environments = []) {
   const patterns = [...existing || [], ...environments.flatMap((routing) => Array.isArray(routing) ? routing : [])];
   return pagePaths(pages).filter((path) => !patterns.some((pattern) => matchesPattern(path, pattern.startsWith("!") ? pattern.slice(1) : pattern)));
 }
-function cloudflareNegotiationPatterns(existing, pages) {
-  if (existing === true || existing === false || pages.length === 0)
-    return existing;
-  const patterns = [...existing || []];
-  for (const path of cloudflareStaticFallbackPaths(existing, pages)) {
-    if (!patterns.includes(path))
-      patterns.push(path);
+function parentDirectories(path) {
+  const segments = path.split("/").filter(Boolean);
+  return segments.slice(0, -1).map((_, index) => `/${segments.slice(0, index + 1).join("/")}/`);
+}
+function fitWorkerFirstBudget(paths, budget) {
+  let remaining = [...paths];
+  const globs = [];
+  while (remaining.length + globs.length > budget) {
+    const counts = /* @__PURE__ */ new Map();
+    for (const path of remaining) {
+      for (const directory2 of parentDirectories(path))
+        counts.set(directory2, (counts.get(directory2) ?? 0) + 1);
+    }
+    let best;
+    for (const [directory2, count] of counts) {
+      if (!best || count > best[1] || count === best[1] && directory2.length > best[0].length)
+        best = [directory2, count];
+    }
+    if (!best || best[1] < 2)
+      return void 0;
+    const [directory] = best;
+    globs.push(`${directory}*`);
+    remaining = remaining.filter((path) => !path.startsWith(directory) || path === directory);
   }
-  if (patterns.length > MAX_WORKER_FIRST_PATTERNS) {
-    throw new Error("Cloudflare Markdown negotiation exceeds 100 worker-first patterns. Configure broader assets.run_worker_first patterns, or disable aiReady.contentNegotiation.");
+  return [...remaining, ...globs];
+}
+function cloudflareWorkerFirstAdditions(existing, pages) {
+  if (existing === true || existing === false || pages.length === 0)
+    return [];
+  const patterns = existing || [];
+  const paths = cloudflareStaticFallbackPaths(existing, pages).filter((path) => !patterns.includes(path));
+  const additions = fitWorkerFirstBudget(paths, MAX_WORKER_FIRST_PATTERNS - patterns.length);
+  if (!additions) {
+    throw new Error("Cloudflare Markdown negotiation exceeds 100 worker-first patterns, even with directory globs. Configure broader assets.run_worker_first patterns, or disable aiReady.contentNegotiation.");
   }
-  return patterns;
+  return additions;
 }
 
 const CACHE_GUIDE = "https://nuxtseo.com/docs/ai-ready/guides/markdown#cache-safety";
@@ -1219,6 +1248,14 @@ function createMcpServerCardEtag(card) {
   return `"${createHash("sha256").update(JSON.stringify(card)).digest("hex")}"`;
 }
 
+function opaqueJsonModule(exports) {
+  const encode = (value) => JSON.stringify(Buffer.from(JSON.stringify(value), "utf8").toString("base64"));
+  return [
+    "const decode = value => JSON.parse(new TextDecoder().decode(Uint8Array.from(atob(value), char => char.charCodeAt(0))))",
+    ...Object.entries(exports).map(([name, value]) => `export const ${name} = decode(${encode(value)})`)
+  ].join("\n");
+}
+
 function escapeRegExp(value) {
   return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
@@ -1808,7 +1845,7 @@ ${details}`);
     const llmsFullRoute = withSiteUrl("llms-full.txt", { withBase: true });
     defaultLlmsTxtSections.push({
       title: "LLM Resources",
-      links: [
+      links: config.llmsFullTxt === false ? [] : [
         {
           title: "Full Content",
           href: llmsFullRoute,
@@ -1821,21 +1858,39 @@ ${details}`);
       defaultLlmsTxtSections.push({
         title: "Agent Skills",
         description: `Skills an agent can install from this site. The index at ${indexUrl} carries a sha256 digest for each one.`,
-        links: agentSkillsResult.links.map((link) => ({
+        links: [{ title: "Agent Skills index", href: indexUrl, description: "Every published skill and its sha256 digest." }, ...agentSkillsResult.links.map((link) => ({
           title: link.name,
           href: link.source === "local" ? withSiteUrl(link.href.slice(1), { withBase: true }) : resolveExternalSkillUrl(link.href, indexUrl),
           description: link.description
-        }))
+        }))]
       });
     }
+    const directiveConfig = config.llmsTxt === false ? void 0 : config.llmsTxt?.directive;
+    const llmsTxtDirective = {
+      markdown: directiveConfig?.markdown !== false,
+      html: directiveConfig?.html === true
+    };
+    const parsedLlmsTxt = parseLlmsTxtConfig(config.llmsTxt || {});
+    if (parsedLlmsTxt._tag === "Invalid")
+      throw new Error(`[nuxt-ai-ready] ${parsedLlmsTxt.message}`);
+    const publicLlmsTxt = await Promise.all(getLayerDirectories(nuxt).map(
+      (layer) => access(join(layer.public, "llms.txt")).then(() => true).catch((error) => {
+        if (error.code === "ENOENT")
+          return false;
+        throw error;
+      })
+    ));
+    const generateLlmsTxt = config.llmsTxt !== false && !publicLlmsTxt.some(Boolean);
     const mergedLlmsTxt = config.llmsTxt ? {
-      markdownLinks: config.llmsTxt.markdownLinks ?? false,
+      ...parsedLlmsTxt.config,
+      markdownLinks: config.llmsTxt.markdownLinks ?? true,
+      directive: llmsTxtDirective,
       sections: [
         ...defaultLlmsTxtSections,
         ...config.llmsTxt.sections || []
       ],
       notes: config.llmsTxt.notes
-    } : { markdownLinks: false, sections: defaultLlmsTxtSections };
+    } : { markdownLinks: true, directive: llmsTxtDirective, sections: defaultLlmsTxtSections };
     const llmsTxtPayload = {
       sections: mergedLlmsTxt.sections || [],
       notes: typeof mergedLlmsTxt.notes === "string" ? [mergedLlmsTxt.notes] : mergedLlmsTxt.notes || []
@@ -1843,6 +1898,9 @@ ${details}`);
     await nuxt.callHook("ai-ready:llms-txt", llmsTxtPayload);
     mergedLlmsTxt.sections = llmsTxtPayload.sections;
     mergedLlmsTxt.notes = llmsTxtPayload.notes.length > 0 ? llmsTxtPayload.notes : void 0;
+    const parsedHookLlmsTxt = parseLlmsTxtConfig(mergedLlmsTxt);
+    if (parsedHookLlmsTxt._tag === "Invalid")
+      throw new Error(`[nuxt-ai-ready] ${parsedHookLlmsTxt.message}`);
     const prerenderCacheDir = join(nuxt.options.rootDir, "node_modules/.cache/nuxt-seo/ai-ready/routes");
     const buildDbPath = join(nuxt.options.buildDir, ".data/ai-ready/build.db");
     const runtimeSyncConfig = typeof config.runtimeSync === "object" ? config.runtimeSync : {};
@@ -1859,6 +1917,8 @@ ${details}`);
       if (apiCatalogRegistered)
         return;
       apiCatalogRegistered = true;
+      const resources = mergedLlmsTxt.sections?.find((section) => section.title === "LLM Resources");
+      resources?.links?.push({ title: "API catalog", href: catalog.href, description: "The public API and service descriptions." });
       addServerHandler({ route: API_CATALOG_PATH, handler: resolve$1("./runtime/server/routes/api-catalog"), lazy: true });
       const apiCatalogLink = `<${catalog.href}>; rel="api-catalog"`;
       extendRouteRules("/**", {
@@ -2160,8 +2220,7 @@ export { unavailable as computeLocaleAlternates, unavailable as localePath, unav
 `;
       };
       nitroConfig.virtual["#ai-ready-virtual/site-tools.mjs"] = `export default ${JSON.stringify(siteToolsConfig)}`;
-      nitroConfig.virtual["#ai-ready-virtual/agent-skills.mjs"] = agentSkillsResult._tag === "Enabled" ? `export const agentSkillsIndex = ${JSON.stringify(agentSkillsResult.index)}
-export const localAgentSkillArtifacts = ${JSON.stringify(agentSkillsResult.localArtifacts)}` : "export const agentSkillsIndex = null\nexport const localAgentSkillArtifacts = {}";
+      nitroConfig.virtual["#ai-ready-virtual/agent-skills.mjs"] = agentSkillsResult._tag === "Enabled" ? opaqueJsonModule({ agentSkillsIndex: agentSkillsResult.index, localAgentSkillArtifacts: agentSkillsResult.localArtifacts }) : "export const agentSkillsIndex = null\nexport const localAgentSkillArtifacts = {}";
       const markdownLinkAvailabilityPath = join(dirname(buildDbPath), MARKDOWN_LINK_AVAILABILITY_FILE);
       nitroConfig.virtual["#ai-ready-virtual/read-page-data.mjs"] = createBuildPageDataVirtual({
         buildDbPath,
@@ -2237,6 +2296,7 @@ export function trackDrizzleWork(event, work) { return work }
         cron: !!config.cron
       })}`;
       nitroConfig.virtual["#ai-ready-virtual/content-lookup.mjs"] = contentLookupModule(contentSource);
+      nitroConfig.virtual["#ai-ready-virtual/sitemap-config.mjs"] = `export { default } from '#sitemap-virtual/static-config.mjs'`;
     });
     nuxt.options.runtimeConfig["nuxt-ai-ready"] = {
       version: version || "0.0.0",
@@ -2252,6 +2312,7 @@ export function trackDrizzleWork(event, work) { return work }
         swr: true
       }),
       llmsTxt: mergedLlmsTxt,
+      llmsFullTxt: config.llmsFullTxt !== false,
       llmsTxtCacheSeconds: config.llmsTxtCacheSeconds ?? 600,
       prerenderCacheDir,
       database,
@@ -2293,14 +2354,20 @@ export function trackDrizzleWork(event, work) { return work }
     }
     addNitroPlugin(resolve$1(getNitroVersion(nuxt) === 3 ? "./runtime/server/plugins/link-header-nitro3" : "./runtime/server/plugins/link-header"));
     addNitroPlugin(resolve$1("./runtime/server/plugins/markdown-negotiation"));
+    if (llmsTxtDirective.html)
+      addNitroPlugin(resolve$1("./runtime/server/plugins/llms-txt-directive-html"));
     addPlugin({
       mode: "server",
       src: resolve$1("./runtime/app/plugins/md-alternate.server")
     });
-    addServerHandler({ route: "/llms.txt", handler: resolve$1("./runtime/server/routes/llms.txt.get"), lazy: true });
-    addServerHandler({ route: "/llms-full.txt", handler: resolve$1("./runtime/server/routes/llms-full.txt.get"), lazy: true });
+    if (generateLlmsTxt) {
+      for (const route of ["/llms.txt", "/llms/**"])
+        addServerHandler({ route, method: void 0, handler: resolve$1("./runtime/server/routes/llms.txt.get"), lazy: true });
+    }
+    if (config.llmsFullTxt !== false)
+      addServerHandler({ route: "/llms-full.txt", method: void 0, handler: resolve$1("./runtime/server/routes/llms-full.txt.get"), lazy: true });
     if (config.sitemapMd !== false) {
-      addServerHandler({ route: SITEMAP_MD_ROUTE, handler: resolve$1("./runtime/server/routes/sitemap.md.get"), lazy: true });
+      addServerHandler({ route: SITEMAP_MD_ROUTE, method: void 0, handler: resolve$1("./runtime/server/routes/sitemap.md.get"), lazy: true });
     }
     if (agentSkillsResult._tag === "Enabled") {
       addServerHandler({ route: AGENT_SKILLS_INDEX_ROUTE, handler: resolve$1("./runtime/server/routes/agent-skills-index"), lazy: true });
@@ -2376,7 +2443,7 @@ export function trackDrizzleWork(event, work) { return work }
     }
     if (prerenderEnabled) {
       const { setupPrerenderHandler } = await import('../chunks/prerender.mjs');
-      setupPrerenderHandler(config, buildDbPath, {
+      setupPrerenderHandler({ ...config, llmsTxt: generateLlmsTxt ? config.llmsTxt : false }, buildDbPath, {
         name: siteConfig.name,
         url: siteConfig.url ? withSiteUrl("/", { withBase: true }) : void 0,
         description: siteConfig.description
@@ -2399,7 +2466,7 @@ export function trackDrizzleWork(event, work) { return work }
         icon: "carbon:ai-label"
       }, resolve$1, nuxt);
     }
-    for (const route of ["/llms.txt", "/llms-full.txt"]) {
+    for (const route of [generateLlmsTxt && "/llms.txt", generateLlmsTxt && "/llms/**", config.llmsFullTxt !== false && "/llms-full.txt"].filter((route2) => Boolean(route2))) {
       extendRouteRules(route, { headers: { "Content-Type": "text/plain; charset=utf-8" } });
     }
     if (config.sitemapMd !== false) {
@@ -2446,13 +2513,12 @@ export function trackDrizzleWork(event, work) { return work }
           const assets = wrangler.assets ||= {};
           const fileRouting = await readCloudflareAssetRouting(nitro.options.rootDir);
           const routing = defu$1({ value: assets.run_worker_first }, { value: fileRouting.root }).value;
-          cloudflareNegotiationPatterns(routing, pages);
+          const additions = cloudflareWorkerFirstAdditions(routing, pages);
           const environmentConfig = wrangler;
           const configuredEnvironments = Object.values(environmentConfig.env || {}).flatMap((env) => env.assets?.run_worker_first === void 0 ? [] : [env.assets.run_worker_first]);
           const fallbackPaths = cloudflareStaticFallbackPaths(routing, pages, [...fileRouting.environments, ...configuredEnvironments]);
-          const addedPaths = cloudflareStaticFallbackPaths(routing, pages);
-          if (addedPaths.length > 0)
-            assets.run_worker_first = [...Array.isArray(assets.run_worker_first) ? assets.run_worker_first : [], ...addedPaths];
+          if (additions.length > 0)
+            assets.run_worker_first = [...Array.isArray(assets.run_worker_first) ? assets.run_worker_first : [], ...additions];
           nitro.options.runtimeConfig["nuxt-ai-ready"].cloudflarePrerenderedPages = fallbackPaths;
         }
         const plan = planStaticMarkdownHeaderRules(

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "3.1.0",
+  "version": "3.3.0",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",

package/skills/nuxt-ai-ready/SKILL.md

@@ -31,7 +31,7 @@ A page visit in dev or production does not index anything.
 
 With no `aiReady` config, the module adds:
 
-- `/llms.txt`: site name, description, resource links, and a `## Pages` list. In dev it lists sitemap routes without titles.
+- `/llms.txt`: site name, description, resource links, and current sitemap pages. Stored pages supply metadata. Large lists split under `/llms/`.
 - `/llms-full.txt`: the full Markdown of each prerendered page. Only prerendered pages go in the static file.
 - A `.md` twin per page. `/about` becomes `/about.md`, and `/` becomes `/index.md`. Paths under `/api` and `/_` get none.
 - Accept negotiation: a request that prefers `text/markdown` gets a `307` to the `.md` URL. Responses send `Vary: Accept`. AI bot user agents get Markdown only with `botNegotiation: true`.
@@ -58,6 +58,32 @@ export default defineNuxtConfig({
 })
 ```
 
+Publish separate discovery files from named sitemaps. Use module configuration before writing sitemap and database middleware:
+
+```ts
+export default defineNuxtConfig({
+  aiReady: {
+    llmsFullTxt: false,
+    llmsTxt: {
+      markdownLinks: true,
+      sitemaps: ['pages', 'skills', 'demos'],
+      sections: [
+        { title: 'Skills', sitemap: 'skills', path: '/llms/skills.txt' },
+        { title: 'Demos', sitemap: 'demos', path: '/llms/demos.txt' },
+      ],
+    },
+  },
+})
+```
+
+The root links each section file. Chunked sitemaps keep their configured name.
+Every file stays within `maxLength`, default 80,000 characters. Static builds emit every linked child.
+GET and HEAD work on every generated discovery file. Empty sections produce no link or file.
+
+For manual discovery, place `public/llms.txt`. The module preserves it and keeps Markdown conversion enabled.
+If a server handler owns discovery, set `llmsTxt: false`.
+Use `llmsFullTxt: false` instead of redirecting the full export to the index.
+
 Remove an element from the Markdown on every path, build and runtime. Put it in `mdreamOptions`; the defaults `minimal` and `clean` stay on:
 
 ```ts
@@ -92,9 +118,12 @@ Runtime indexing, database drivers, Cloudflare D1, and the server query helpers:
 
 ## Traps
 
+- **Discovery is not authorization.** Selected sitemaps exclude stale rows from `llms.txt`; they do not delete stored content or protect other APIs.
+- **Custom route policies must allow Markdown twins.** If middleware rejects `/skills/planning.md`, allow it whenever `/skills/planning` is allowed.
 - **The same edit in both hooks applies twice.** During prerendering the Nitro `ai-ready:page:markdown` hook runs, then the Nuxt hook of the same name. Put an edit in one of them, not both.
 - **Content source pages skip conversion hooks.** A Nuxt Content page ignores `mdreamOptions` and both Nitro hooks.
 - **Dev has no page data.** Query helpers, MCP tools, and search return empty results in `nuxt dev`. Test with `nuxt build` and `node .output/server/index.mjs`.
+- **Cloudflare static pages negotiate only with `contentNegotiation: true`.** The default leaves prerendered HTML on the asset service, so `Accept: text/markdown` gets HTML there. With `true`, prerendered pages go into `assets.run_worker_first`: exact paths up to Cloudflare's 100 entries, directory globs such as `/docs/*` past that. More than 100 top-level pages fail the build; set broader patterns yourself.
 - **Response caching turns negotiation off.** An ISR, `swr`, or `cache` route rule without `varies: ['accept']` serves HTML to Markdown clients. With `botNegotiation: true`, the rule needs `varies: ['accept', 'sec-fetch-dest', 'user-agent']`. The `.md` URL still works. For a CDN that caches by URL only, set `contentNegotiation: false`.
 - **Emptying `ctx.markdown` does not hide a page.** It drops the body from `llms-full.txt`. The URL stays in `llms.txt` and in the index. Use `sitemap.exclude` for discovery, and access control for private pages.
 - **Root `skills/` folders require explicit publication.** Automatic discovery scans public directories only. Set `agentSkills.dir: 'skills'` to publish root skills intentionally.
@@ -125,7 +154,13 @@ Runtime indexing, database drivers, Cloudflare D1, and the server query helpers:
 
 ## Config
 
-- `llmsTxt.markdownLinks` (`false`): link page entries in `llms.txt` to their `.md` twins.
+- `llmsTxt` (`object`): generated discovery. Set `false` for application-owned server routes. A public `llms.txt` disables generation automatically.
+- `llmsTxt.pages` (`'sitemap'`): sitemap admission. Use `'all'` for stored pages outside the sitemap, or `false` for configured sections only.
+- `llmsTxt.sitemaps`: selected sitemap names. An empty array stops sitemap discovery.
+- `llmsTxt.maxLength` (`80000`): maximum characters per file. Minimum `1024`. Larger lists split without dropping links.
+- `llmsFullTxt` (`true`): generate the full export and advertise its link. Set `false` to omit both.
+- `llmsTxt.directive` (`{ markdown: true, html: false }`): a `> Index of every page on this site: [llms.txt](...)` line after the frontmatter of each page's `.md`; `html: true` adds a visually hidden `/llms.txt` link at the top of each HTML body.
+- `llmsTxt.markdownLinks` (`true`): link page entries in `llms.txt` to their `.md` twins. `false` links canonical URLs. With `site.url` set, page links in `llms.txt`, `sitemap.md`, and `.md` bodies are absolute.
 - `describedby` (`true`): the `describedby` link tag and header.
 - `tools.<listPages|searchPages|getPageMarkdown>.<mcp|webmcp>.enabled`: attach a built in tool to one transport only.
 - `autoI18n` (`true`): with `@nuxtjs/i18n`, adds hreflang `Link` headers and a languages section in `llms.txt`.