nuxt-ai-ready 3.2.0 → 3.3.1

package/dist/runtime/server/utils/llms-full.js

@@ -1,15 +1,41 @@
 import { joinURL } from "ufo";
+import { isMap, isScalar, parseDocument, stringify } from "yaml";
 import { normalizeLlmsTxtConfig } from "../../llms-txt-format.js";
-const RE_FRONTMATTER = /^---\r?\n[\s\S]*?\r?\n---\r?\n*/;
+import { escapeMarkdownHtml } from "../../markdown-text.js";
+const RE_FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---\r?\n*/;
 const RE_INLINE_WHITESPACE = /\s+/g;
+function imageMetadataFence(frontmatter) {
+  const document = parseDocument(frontmatter);
+  const meta = document.get("meta", true);
+  if (document.errors.length || !isMap(meta))
+    return;
+  const images = {};
+  for (const { key, value } of meta.items) {
+    if (!isScalar(key) || typeof key.value !== "string" || !isScalar(value))
+      continue;
+    if (key.value !== "og:image" && !key.value.startsWith("og:image:") && key.value !== "twitter:image" && !key.value.startsWith("twitter:image:"))
+      continue;
+    if (typeof value.value === "string" || typeof value.value === "number" && Number.isFinite(value.value))
+      images[key.value] = value.value;
+  }
+  if (!Object.keys(images).length)
+    return;
+  return `\`\`\`yaml
+${stringify({ meta: images }, { lineWidth: 0 }).trimEnd()}
+\`\`\``;
+}
 export function formatPageForLlmsFullTxt(route, title, description, markdown, siteUrl) {
   const canonicalUrl = siteUrl ? joinURL(siteUrl, route) : route;
-  const pageTitle = (title && title !== route ? title : route).trim().replace(RE_INLINE_WHITESPACE, " ");
-  const pageDescription = description.trim().replace(RE_INLINE_WHITESPACE, " ");
+  const pageTitle = escapeMarkdownHtml((title && title !== route ? title : route).trim().replace(RE_INLINE_WHITESPACE, " "));
+  const pageDescription = escapeMarkdownHtml(description.trim().replace(RE_INLINE_WHITESPACE, " "));
   const content = markdown.replace(RE_FRONTMATTER, "").trim();
   const parts = ["---", "", `- **Page:** ${pageTitle}`, `- **Source:** ${canonicalUrl}`];
   if (pageDescription)
     parts.push(`- **Description:** ${pageDescription}`);
+  const frontmatter = markdown.match(RE_FRONTMATTER)?.[1];
+  const images = frontmatter && imageMetadataFence(frontmatter);
+  if (images)
+    parts.push("", images);
   parts.push("");
   if (content) {
     parts.push(content);
@@ -19,9 +45,9 @@ export function formatPageForLlmsFullTxt(route, title, description, markdown, si
 `;
 }
 export function buildLlmsFullTxtHeader(siteInfo, llmsTxtConfig) {
-  const parts = [`# ${siteInfo?.name || siteInfo?.url || "Site"}`];
+  const parts = [`# ${escapeMarkdownHtml(siteInfo?.name || siteInfo?.url || "Site")}`];
   if (siteInfo?.description)
-    parts.push("", `> ${siteInfo.description}`);
+    parts.push("", `> ${escapeMarkdownHtml(siteInfo.description)}`);
   if (siteInfo?.url)
     parts.push("", `Canonical Origin: ${siteInfo.url}`);
   if (llmsTxtConfig) {

package/dist/runtime/server/utils/markdown-request.js

@@ -1,7 +1,7 @@
 import { negotiateContent, parseAcceptHeader } from "@mdream/js/negotiate";
 import { getBotInfo } from "@nuxtjs/robots/util";
 import { getHeaders } from "#nuxtseo/h3";
-import { isReservedPath } from "../../markdown-path.js";
+import { hasMarkdownTwin, isReservedPath } from "../../markdown-path.js";
 const RE_MD_EXT = /\.md$/;
 export function toMarkdownRequest(event, isPrerender = !!import.meta.prerender) {
   return { path: event.path, headers: getHeaders(event), isPrerender };
@@ -38,10 +38,7 @@ export function getRequestRenderInfo(request, mode = { _tag: "runtime", contentN
   const isExplicit = originalPath.endsWith(".md");
   if (isPrerender && !isExplicit)
     return null;
-  const lastSegment = originalPath.split("/").pop() || "";
-  const hasExtension = lastSegment.includes(".");
-  const extension = hasExtension ? lastSegment.substring(lastSegment.lastIndexOf(".")) : "";
-  if (hasExtension && extension !== ".md")
+  if (!isExplicit && !hasMarkdownTwin(originalPath))
     return null;
   const negotiation = isPrerender ? "markdown" : mode.contentNegotiation ? negotiateRequestRepresentation(request, mode.botNegotiation) : "html";
   if (isExplicit) {

package/dist/runtime/server/utils/negotiation-response.js

@@ -52,6 +52,9 @@ export function setUncacheableHeaders(event) {
 }
 export function setMarkdownHeaders(event, ctx, sourceHeaders) {
   setHeader(event, "content-type", "text/markdown; charset=utf-8");
+  const sourcePolicy = sourceHeaders?.get("content-signal");
+  if (sourcePolicy !== null && sourcePolicy !== void 0 && getResponseHeader(event, "content-signal") === void 0)
+    setHeader(event, "content-signal", sourcePolicy);
   const vary = [getResponseHeader(event, "vary"), sourceHeaders?.get("vary")].filter(Boolean).join(", ");
   setHeader(event, "vary", mergeVaryHeader(vary, "Accept"));
   setLinkHeader(event, ctx, "markdown");

package/dist/runtime/server/utils/runCron.js

@@ -170,7 +170,7 @@ async function pingSitemap(event, config, sitemapIntervalMinutes, debug) {
     route: nextSitemap.route,
     state: nextSitemap.crawlState
   });
-  const routes = [...mapSitemapRoutes(result.urls).entries()].map(([route, url]) => ({ route, url: url.loc }));
+  const routes = [...mapSitemapRoutes(result.urls).entries()].map(([route, url]) => ({ route, url: url.loc, lastmod: url.lastmod }));
   if (routes.length > 0)
     await seedRoutes(event, routes, { refreshWindowMs });
   if (result._tag === "failed") {

package/dist/runtime/server/utils/sitemap-lastmod.d.ts

@@ -0,0 +1,4 @@
+/** Mark dates added by this module without changing the public sitemap date. */
+export declare function markIndexedSitemapLastmods(xml: string, enrichedUrls: ReadonlySet<string>): string;
+/** The source crawl must never treat our indexed date as a CMS change. */
+export declare function stripIndexedSitemapLastmods(body: Uint8Array): Uint8Array;

package/dist/runtime/server/utils/sitemap-lastmod.js

@@ -0,0 +1,19 @@
+const INDEXED_LASTMOD_MARKER = "<!-- nuxt-ai-ready:indexed-lastmod -->";
+export function markIndexedSitemapLastmods(xml, enrichedUrls) {
+  const escapedUrls = new Set([...enrichedUrls].map((url) => url.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&apos;")));
+  return xml.replace(/<url>([\s\S]*?)<\/url>/g, (entry, body) => {
+    const loc = body.match(/<loc>([^<]*)<\/loc>/)?.[1];
+    return loc && escapedUrls.has(loc) ? entry.replace("<lastmod>", `${INDEXED_LASTMOD_MARKER}<lastmod>`) : entry;
+  });
+}
+export function stripIndexedSitemapLastmods(body) {
+  let xml;
+  try {
+    xml = new TextDecoder("utf-8", { fatal: true }).decode(body);
+  } catch {
+    return body;
+  }
+  if (!xml.includes(INDEXED_LASTMOD_MARKER))
+    return body;
+  return new TextEncoder().encode(xml.replace(/<!-- nuxt-ai-ready:indexed-lastmod -->\s*<lastmod>[^<]*<\/lastmod>/g, ""));
+}

package/dist/runtime/server/utils/sitemap-md.js

@@ -1,11 +1,12 @@
 import { toMarkdownPath } from "../../markdown-path.js";
+import { escapeMarkdownHtml } from "../../markdown-text.js";
 import { toDeployedRoute } from "../../route-path.js";
 export const SITEMAP_MD_ROUTE = "/sitemap.md";
 function topSegment(route) {
   return route.split("/").filter(Boolean)[0] || "";
 }
 function escapeTitle(title) {
-  return title.replaceAll("[", "\\[").replaceAll("]", "\\]");
+  return escapeMarkdownHtml(title).replaceAll("[", "\\[").replaceAll("]", "\\]");
 }
 function formatEntry(entry, resolveHref) {
   const title = escapeTitle(entry.title?.trim() || entry.route);
@@ -35,7 +36,7 @@ export function buildSitemapMd(entries, options = {}) {
       return 1;
     return a.localeCompare(b);
   });
-  const parts = [`# ${options.siteName || "Site"} Sitemap`, "", "All pages in Markdown format."];
+  const parts = [`# ${escapeMarkdownHtml(options.siteName || "Site")} Sitemap`, "", "All pages in Markdown format."];
   for (const key of keys) {
     parts.push("", `## ${key || "Root"}`);
     for (const entry of groups.get(key))

package/dist/runtime/server/utils/sitemap.d.ts

@@ -53,5 +53,6 @@ export declare function fetchSitemapByRoute(event: H3Event | undefined, route: s
  * Fetch all URLs from all sitemaps (or single sitemap if not multi-sitemap)
  * Used for backwards compatibility and llms.txt generation
  */
+export declare function fetchSitemapUrlGroups(event: H3Event, names?: readonly string[]): Promise<Map<string, SitemapUrl[]>>;
 export declare function fetchSitemapUrls(event: H3Event): Promise<SitemapUrl[]>;
 export {};

package/dist/runtime/server/utils/sitemap.js

@@ -1,10 +1,13 @@
 import { createSitemapReader } from "sitemapd";
 import { parseSitemap } from "sitemapd/parse";
+import staticSitemapConfig from "#ai-ready-virtual/sitemap-config.mjs";
+import { getRequestURL } from "#nuxtseo/h3";
 import { useRuntimeConfig } from "#nuxtseo/nitro";
 import { logger } from "../logger.js";
 import { getCfEnv, hasAssets } from "./cloudflare.js";
 import { createUniversalContext } from "./context.js";
 import { fetchRawWithEvent } from "./fetch.js";
+import { stripIndexedSitemapLastmods } from "./sitemap-lastmod.js";
 const FETCH_TIMEOUT = 15e3;
 const SITEMAP_FALLBACK_ORIGIN = "http://localhost";
 const SITEMAP_READER_LIMITS = {
@@ -19,17 +22,18 @@ const SITEMAP_READER_LIMITS = {
 const MAX_SITEMAP_CRAWL_ROUNDS = 10;
 export function getSitemapsFromConfig(event) {
   const runtimeConfig = useRuntimeConfig(event);
-  const sitemapConfig = runtimeConfig.sitemap;
+  const sitemapConfig = { ...staticSitemapConfig, ...runtimeConfig.sitemap };
   if (!sitemapConfig?.sitemaps)
     return [];
   const sitemaps = [];
   for (const [key, sitemap] of Object.entries(sitemapConfig.sitemaps)) {
     if (key === "index")
       continue;
-    if (sitemap._route) {
+    const route = sitemap._route || sitemap.route || (key === "chunks" ? sitemapConfig.sitemaps.index?._route || sitemapConfig.sitemaps.index?.route : void 0);
+    if (route) {
       sitemaps.push({
-        name: sitemap.sitemapName || key,
-        route: sitemap._route
+        name: (sitemap.sitemapName || key).replace(/\.xml$/, ""),
+        route: route.startsWith("/") ? route : `/${route}`
       });
     }
   }
@@ -218,11 +222,12 @@ function createLocalSitemapLoader(event, usePublicAsset, loadedBodies) {
         bytesRead: read.result.bytesRead
       };
     }
-    loadedBodies.set(request.url, read.result.body);
+    const sourceBody = stripIndexedSitemapLastmods(read.result.body);
+    loadedBodies.set(request.url, sourceBody);
     return {
       _tag: "body",
       url: request.url,
-      body: read.result.body
+      body: sourceBody
     };
   };
 }
@@ -305,6 +310,7 @@ export async function crawlSitemapByRoute(event, input) {
   const usePublicAsset = config.sitemapPrerendered && hasAssets(event);
   const configuredSiteUrl = createUniversalContext(event).siteUrl ?? SITEMAP_FALLBACK_ORIGIN;
   const siteUrl = parseAbsoluteUrl(configuredSiteUrl);
+  const requestOrigin = import.meta.dev && event ? getRequestURL(event, { xForwardedHost: true, xForwardedProto: true }).origin : void 0;
   if (siteUrl._tag === "error") {
     return {
       _tag: "failed",
@@ -346,7 +352,7 @@ export async function crawlSitemapByRoute(event, input) {
       const target = parseAbsoluteUrl(request.url);
       if (target._tag === "error")
         return { _tag: "deny", reason: `Invalid sitemap target: ${request.url}` };
-      if (target.value.origin !== siteUrl.value.origin) {
+      if (target.value.origin !== siteUrl.value.origin && target.value.origin !== requestOrigin) {
         return {
           _tag: "deny",
           reason: `Sitemap target origin ${target.value.origin} differs from ${siteUrl.value.origin}`
@@ -433,17 +439,21 @@ export async function fetchSitemapByRoute(event, route) {
   const result = await crawlSitemapByRoute(event, { route });
   return result._tag === "complete" ? { urls: result.urls } : { urls: result.urls, error: result.error };
 }
-export async function fetchSitemapUrls(event) {
+export async function fetchSitemapUrlGroups(event, names) {
   const sitemaps = getSitemapsFromConfig(event);
+  const groups = /* @__PURE__ */ new Map();
   if (sitemaps.length > 0) {
     logger.debug(`[sitemap] Multi-sitemap mode: ${sitemaps.length} sitemaps`);
-    const allUrls = [];
     for (const sitemap of sitemaps) {
+      if (names !== void 0 && !names.includes(sitemap.name))
+        continue;
       const { urls: urls2 } = await fetchSitemapByRoute(event, sitemap.route);
-      allUrls.push(...urls2);
+      groups.set(sitemap.name, [...groups.get(sitemap.name) || [], ...urls2]);
     }
-    return allUrls;
+    return groups;
   }
+  if (names !== void 0 && !names.includes("sitemap"))
+    return groups;
   logger.debug(`[sitemap] Single sitemap mode: /sitemap.xml`);
   const { urls, error } = await fetchSitemapByRoute(event, "/sitemap.xml");
   if (error) {
@@ -451,5 +461,9 @@ export async function fetchSitemapUrls(event) {
   } else if (urls.length === 0) {
     logger.warn("Sitemap is empty - add routes to sitemap or configure sitemap.sources in nuxt.config");
   }
-  return urls;
+  groups.set("sitemap", urls);
+  return groups;
+}
+export async function fetchSitemapUrls(event) {
+  return [...(await fetchSitemapUrlGroups(event)).values()].flat();
 }

package/dist/runtime/server/utils.js

@@ -3,11 +3,57 @@ import { useNitroApp } from "#nuxtseo/nitro";
 import { stripLlmsTxtDirectiveHtml } from "../llms-txt-directive.js";
 import { hoistMetaDescription } from "./utils/frontmatter.js";
 const RE_NBSP = /\u00A0/g;
+const JSON_LD_MAX_BYTES = 65536;
+const JSON_LD_MAX_BLOCKS = 16;
+const IMAGE_META_FIELDS = [
+  "og:image",
+  "og:image:url",
+  "og:image:secure_url",
+  "og:image:type",
+  "og:image:width",
+  "og:image:height",
+  "og:image:alt",
+  "twitter:image",
+  "twitter:image:src",
+  "twitter:image:alt"
+];
+function parseJsonLdBlock(text, remainingBytes) {
+  if (text.length > remainingBytes)
+    return { _tag: "Skipped" };
+  const content = text.trim();
+  if (!content)
+    return { _tag: "Skipped" };
+  let value;
+  try {
+    value = JSON.parse(text);
+  } catch {
+    return { _tag: "Skipped" };
+  }
+  const isObject = (item) => typeof item === "object" && item !== null && !Array.isArray(item);
+  if (!isObject(value) && !(Array.isArray(value) && value.every(isObject)))
+    return { _tag: "Skipped" };
+  const markdown = `\`\`\`json
+${content.replace(RE_NBSP, "\\u00a0")}
+\`\`\``;
+  const bytes = new TextEncoder().encode(markdown).byteLength + 2;
+  if (bytes > remainingBytes)
+    return { _tag: "Skipped" };
+  return { _tag: "Preserved", markdown, bytes };
+}
 function normalizeWhitespace(text) {
   return text.replace(RE_NBSP, " ");
 }
 function buildMdreamOptions(url, mdreamOptions, meta, extractUpdatedAt, additionalFields) {
   const extraction = {
+    "script[type]": (el) => {
+      if (el.attributes.type?.trim().toLowerCase() !== "application/ld+json" || meta.jsonLd.blocks.length >= JSON_LD_MAX_BLOCKS)
+        return;
+      const block = parseJsonLdBlock(el.textContent, JSON_LD_MAX_BYTES - meta.jsonLd.bytes);
+      if (block._tag === "Preserved") {
+        meta.jsonLd.blocks.push(block.markdown);
+        meta.jsonLd.bytes += block.bytes;
+      }
+    },
     'meta[name="keywords"]': (el) => {
       meta.metaKeywords = el.attributes.content || "";
     },
@@ -30,6 +76,7 @@ function buildMdreamOptions(url, mdreamOptions, meta, extractUpdatedAt, addition
   };
   const frontmatter = {
     additionalFields,
+    metaFields: [...IMAGE_META_FIELDS],
     onExtract: (fm) => {
       if (fm.title)
         meta.title = fm.title;
@@ -70,13 +117,14 @@ export function extractLastUpdated(html) {
   return void 0;
 }
 export async function convertHtmlToMarkdown(html, url, mdreamOptions, opts) {
-  const meta = { title: "", description: "", metaKeywords: "", headings: [], textContent: [] };
+  const meta = { title: "", description: "", metaKeywords: "", headings: [], textContent: [], jsonLd: { blocks: [], bytes: 0 } };
   const options = buildMdreamOptions(url, mdreamOptions, meta, opts.extractUpdatedAt ?? false, opts.additionalFrontmatter ?? {});
   const nitroApp = useNitroApp();
   await nitroApp.hooks.callHook("ai-ready:mdreamConfig", options);
+  const convertedMarkdown = hoistMetaDescription(htmlToMarkdown(stripLlmsTxtDirectiveHtml(html), options));
   const context = {
     html,
-    markdown: hoistMetaDescription(htmlToMarkdown(stripLlmsTxtDirectiveHtml(html), options)),
+    markdown: [convertedMarkdown, ...meta.jsonLd.blocks].join("\n\n"),
     route: opts.hooks.route,
     title: meta.title,
     description: meta.description,

package/dist/runtime/types.d.ts

@@ -185,7 +185,9 @@ export interface ModuleOptions {
     /**
      * Structured llms.txt configuration
      */
-    llmsTxt?: LlmsTxtConfig;
+    llmsTxt?: LlmsTxtConfig | false;
+    /** Generate and serve llms-full.txt. Set false to omit the export and its discovery link. */
+    llmsFullTxt?: boolean;
     /**
      * Generate a Markdown sitemap at `/sitemap.md` from indexed pages and end
      * every Markdown page with a `## Sitemap` section linking to it.
@@ -512,6 +514,10 @@ export interface LlmsTxtLink {
  * Section in llms.txt
  */
 export interface LlmsTxtSection {
+    /** Generate a separate file from these sitemap names. Chunked sitemaps retain their configured name. */
+    sitemap?: string | string[];
+    /** Generated file path. Defaults to /llms/<sitemap>.txt. */
+    path?: string;
     /** The title of the section */
     title: string;
     /** Description rendered in the heading-free preamble, or in each optional link note (can be array for multiple paragraphs) */
@@ -525,6 +531,12 @@ export interface LlmsTxtSection {
  * Structured llms.txt configuration
  */
 export interface LlmsTxtConfig {
+    /** Page admission. Sitemap URLs are authoritative by default. Use all to include stored pages outside the sitemap. */
+    pages?: 'sitemap' | 'all' | false;
+    /** Only read these sitemap names. An empty array disables sitemap page discovery. */
+    sitemaps?: string[];
+    /** Maximum characters per generated file. Large lists become linked files. Minimum 1024. Default 80000. */
+    maxLength?: number;
     /**
      * Rewrite automatically generated page links to their Markdown representation
      * when the final static output contains the file or the deployment retains

package/dist/shared/{nuxt-ai-ready.DH9mSayy.mjs → nuxt-ai-ready.CKXM1jnE.mjs}

@@ -1,5 +1,5 @@
 import { createHash, randomBytes } from 'node:crypto';
-import { readFile, mkdir, writeFile, access } from 'node:fs/promises';
+import { readFile, access, mkdir, writeFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import { dirname, basename, join, resolve, relative } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
@@ -8,6 +8,7 @@ import defu$1, { defu } from 'defu';
 import { useSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
 import { resolveNuxtContentVersion, renderNitroTypeAugmentations, setupRuntimeAliases, setupNitroRuntimeCompatibility } from 'nuxtseo-shared/kit';
 import { r as readPackageMetadata, a as resolvePackageMetadataPath } from './nuxt-ai-ready.B3hAuOLQ.mjs';
+import { parseLlmsTxtConfig } from '../../dist/runtime/llms-txt-files.js';
 import { toDeployedRoute } from '../../dist/runtime/route-path.js';
 import { SITEMAP_MD_ROUTE } from '../../dist/runtime/server/utils/sitemap-md.js';
 import { publicCacheControl, mergeVaryHeader } from '../../dist/runtime/cache-control.js';
@@ -210,7 +211,11 @@ export {}
   });
   addTemplate({
     filename: "types/ai-ready-virtual.d.ts",
-    getContents: () => `declare module '#ai-ready-virtual/read-page-data.mjs' {
+    getContents: () => `declare module '#ai-ready-virtual/sitemap-config.mjs' {
+  const config: { sitemaps?: Record<string, { sitemapName?: string, route?: string, _route?: string }> }
+  export default config
+}
+declare module '#ai-ready-virtual/read-page-data.mjs' {
   export function readMarkdownLinkAvailabilityFromFilesystem(): Promise<{
     runtimeMarkdownAvailable: boolean
     paths: string[]
@@ -1840,7 +1845,7 @@ ${details}`);
     const llmsFullRoute = withSiteUrl("llms-full.txt", { withBase: true });
     defaultLlmsTxtSections.push({
       title: "LLM Resources",
-      links: [
+      links: config.llmsFullTxt === false ? [] : [
         {
           title: "Full Content",
           href: llmsFullRoute,
@@ -1853,18 +1858,31 @@ ${details}`);
       defaultLlmsTxtSections.push({
         title: "Agent Skills",
         description: `Skills an agent can install from this site. The index at ${indexUrl} carries a sha256 digest for each one.`,
-        links: agentSkillsResult.links.map((link) => ({
+        links: [{ title: "Agent Skills index", href: indexUrl, description: "Every published skill and its sha256 digest." }, ...agentSkillsResult.links.map((link) => ({
           title: link.name,
           href: link.source === "local" ? withSiteUrl(link.href.slice(1), { withBase: true }) : resolveExternalSkillUrl(link.href, indexUrl),
           description: link.description
-        }))
+        }))]
       });
     }
+    const directiveConfig = config.llmsTxt === false ? void 0 : config.llmsTxt?.directive;
     const llmsTxtDirective = {
-      markdown: config.llmsTxt?.directive?.markdown !== false,
-      html: config.llmsTxt?.directive?.html === true
+      markdown: directiveConfig?.markdown !== false,
+      html: directiveConfig?.html === true
     };
+    const parsedLlmsTxt = parseLlmsTxtConfig(config.llmsTxt || {});
+    if (parsedLlmsTxt._tag === "Invalid")
+      throw new Error(`[nuxt-ai-ready] ${parsedLlmsTxt.message}`);
+    const publicLlmsTxt = await Promise.all(getLayerDirectories(nuxt).map(
+      (layer) => access(join(layer.public, "llms.txt")).then(() => true).catch((error) => {
+        if (error.code === "ENOENT")
+          return false;
+        throw error;
+      })
+    ));
+    const generateLlmsTxt = config.llmsTxt !== false && !publicLlmsTxt.some(Boolean);
     const mergedLlmsTxt = config.llmsTxt ? {
+      ...parsedLlmsTxt.config,
       markdownLinks: config.llmsTxt.markdownLinks ?? true,
       directive: llmsTxtDirective,
       sections: [
@@ -1880,6 +1898,9 @@ ${details}`);
     await nuxt.callHook("ai-ready:llms-txt", llmsTxtPayload);
     mergedLlmsTxt.sections = llmsTxtPayload.sections;
     mergedLlmsTxt.notes = llmsTxtPayload.notes.length > 0 ? llmsTxtPayload.notes : void 0;
+    const parsedHookLlmsTxt = parseLlmsTxtConfig(mergedLlmsTxt);
+    if (parsedHookLlmsTxt._tag === "Invalid")
+      throw new Error(`[nuxt-ai-ready] ${parsedHookLlmsTxt.message}`);
     const prerenderCacheDir = join(nuxt.options.rootDir, "node_modules/.cache/nuxt-seo/ai-ready/routes");
     const buildDbPath = join(nuxt.options.buildDir, ".data/ai-ready/build.db");
     const runtimeSyncConfig = typeof config.runtimeSync === "object" ? config.runtimeSync : {};
@@ -1896,6 +1917,8 @@ ${details}`);
       if (apiCatalogRegistered)
         return;
       apiCatalogRegistered = true;
+      const resources = mergedLlmsTxt.sections?.find((section) => section.title === "LLM Resources");
+      resources?.links?.push({ title: "API catalog", href: catalog.href, description: "The public API and service descriptions." });
       addServerHandler({ route: API_CATALOG_PATH, handler: resolve$1("./runtime/server/routes/api-catalog"), lazy: true });
       const apiCatalogLink = `<${catalog.href}>; rel="api-catalog"`;
       extendRouteRules("/**", {
@@ -2273,6 +2296,7 @@ export function trackDrizzleWork(event, work) { return work }
         cron: !!config.cron
       })}`;
       nitroConfig.virtual["#ai-ready-virtual/content-lookup.mjs"] = contentLookupModule(contentSource);
+      nitroConfig.virtual["#ai-ready-virtual/sitemap-config.mjs"] = `export { default } from '#sitemap-virtual/static-config.mjs'`;
     });
     nuxt.options.runtimeConfig["nuxt-ai-ready"] = {
       version: version || "0.0.0",
@@ -2288,6 +2312,7 @@ export function trackDrizzleWork(event, work) { return work }
         swr: true
       }),
       llmsTxt: mergedLlmsTxt,
+      llmsFullTxt: config.llmsFullTxt !== false,
       llmsTxtCacheSeconds: config.llmsTxtCacheSeconds ?? 600,
       prerenderCacheDir,
       database,
@@ -2335,10 +2360,13 @@ export function trackDrizzleWork(event, work) { return work }
       mode: "server",
       src: resolve$1("./runtime/app/plugins/md-alternate.server")
     });
-    addServerHandler({ route: "/llms.txt", handler: resolve$1("./runtime/server/routes/llms.txt.get"), lazy: true });
-    addServerHandler({ route: "/llms-full.txt", handler: resolve$1("./runtime/server/routes/llms-full.txt.get"), lazy: true });
+    if (generateLlmsTxt) {
+      addServerHandler({ middleware: true, method: void 0, handler: resolve$1("./runtime/server/routes/llms.txt.get"), lazy: true });
+    }
+    if (config.llmsFullTxt !== false)
+      addServerHandler({ route: "/llms-full.txt", method: void 0, handler: resolve$1("./runtime/server/routes/llms-full.txt.get"), lazy: true });
     if (config.sitemapMd !== false) {
-      addServerHandler({ route: SITEMAP_MD_ROUTE, handler: resolve$1("./runtime/server/routes/sitemap.md.get"), lazy: true });
+      addServerHandler({ route: SITEMAP_MD_ROUTE, method: void 0, handler: resolve$1("./runtime/server/routes/sitemap.md.get"), lazy: true });
     }
     if (agentSkillsResult._tag === "Enabled") {
       addServerHandler({ route: AGENT_SKILLS_INDEX_ROUTE, handler: resolve$1("./runtime/server/routes/agent-skills-index"), lazy: true });
@@ -2414,7 +2442,7 @@ export function trackDrizzleWork(event, work) { return work }
     }
     if (prerenderEnabled) {
       const { setupPrerenderHandler } = await import('../chunks/prerender.mjs');
-      setupPrerenderHandler(config, buildDbPath, {
+      setupPrerenderHandler({ ...config, llmsTxt: generateLlmsTxt ? config.llmsTxt : false }, buildDbPath, {
         name: siteConfig.name,
         url: siteConfig.url ? withSiteUrl("/", { withBase: true }) : void 0,
         description: siteConfig.description
@@ -2437,7 +2465,7 @@ export function trackDrizzleWork(event, work) { return work }
         icon: "carbon:ai-label"
       }, resolve$1, nuxt);
     }
-    for (const route of ["/llms.txt", "/llms-full.txt"]) {
+    for (const route of [generateLlmsTxt && "/llms.txt", generateLlmsTxt && "/llms/**", config.llmsFullTxt !== false && "/llms-full.txt"].filter((route2) => Boolean(route2))) {
       extendRouteRules(route, { headers: { "Content-Type": "text/plain; charset=utf-8" } });
     }
     if (config.sitemapMd !== false) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "3.2.0",
+  "version": "3.3.1",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -71,7 +71,7 @@
     }
   },
   "dependencies": {
-    "@mdream/js": "^2.0.1",
+    "@mdream/js": "^2.0.3",
     "@nuxt/kit": "^4.6.0",
     "@nuxtjs/robots": "^7.0.0",
     "@nuxtjs/sitemap": "^9.0.0",
@@ -79,7 +79,7 @@
     "confbox": "^0.3.1",
     "consola": "^3.4.2",
     "defu": "^6.1.7",
-    "mdream": "^2.0.1",
+    "mdream": "^2.0.3",
     "nuxt-site-config": "^5.0.0",
     "nuxtseo-shared": "^6.0.0",
     "ohash": "^2.0.12",

package/skills/nuxt-ai-ready/SKILL.md

@@ -31,7 +31,7 @@ A page visit in dev or production does not index anything.
 
 With no `aiReady` config, the module adds:
 
-- `/llms.txt`: site name, description, resource links, and a `## Pages` list. In dev it lists sitemap routes without titles.
+- `/llms.txt`: site name, description, resource links, and current sitemap pages. Stored pages supply metadata. Large lists split under `/llms/`.
 - `/llms-full.txt`: the full Markdown of each prerendered page. Only prerendered pages go in the static file.
 - A `.md` twin per page. `/about` becomes `/about.md`, and `/` becomes `/index.md`. Paths under `/api` and `/_` get none.
 - Accept negotiation: a request that prefers `text/markdown` gets a `307` to the `.md` URL. Responses send `Vary: Accept`. AI bot user agents get Markdown only with `botNegotiation: true`.
@@ -58,6 +58,32 @@ export default defineNuxtConfig({
 })
 ```
 
+Publish separate discovery files from named sitemaps. Use module configuration before writing sitemap and database middleware:
+
+```ts
+export default defineNuxtConfig({
+  aiReady: {
+    llmsFullTxt: false,
+    llmsTxt: {
+      markdownLinks: true,
+      sitemaps: ['pages', 'skills', 'demos'],
+      sections: [
+        { title: 'Skills', sitemap: 'skills', path: '/llms/skills.txt' },
+        { title: 'Demos', sitemap: 'demos', path: '/llms/demos.txt' },
+      ],
+    },
+  },
+})
+```
+
+The root links each section file. Chunked sitemaps keep their configured name.
+Every file stays within `maxLength`, default 80,000 characters. Static builds emit every linked child.
+GET and HEAD work on every generated discovery file. Empty sections produce no link or file.
+
+For manual discovery, place `public/llms.txt`. The module preserves it and keeps Markdown conversion enabled.
+If a server handler owns discovery, set `llmsTxt: false`.
+Use `llmsFullTxt: false` instead of redirecting the full export to the index.
+
 Remove an element from the Markdown on every path, build and runtime. Put it in `mdreamOptions`; the defaults `minimal` and `clean` stay on:
 
 ```ts
@@ -92,6 +118,8 @@ Runtime indexing, database drivers, Cloudflare D1, and the server query helpers:
 
 ## Traps
 
+- **Discovery is not authorization.** Selected sitemaps exclude stale rows from `llms.txt`; they do not delete stored content or protect other APIs.
+- **Custom route policies must allow Markdown twins.** If middleware rejects `/skills/planning.md`, allow it whenever `/skills/planning` is allowed.
 - **The same edit in both hooks applies twice.** During prerendering the Nitro `ai-ready:page:markdown` hook runs, then the Nuxt hook of the same name. Put an edit in one of them, not both.
 - **Content source pages skip conversion hooks.** A Nuxt Content page ignores `mdreamOptions` and both Nitro hooks.
 - **Dev has no page data.** Query helpers, MCP tools, and search return empty results in `nuxt dev`. Test with `nuxt build` and `node .output/server/index.mjs`.
@@ -126,6 +154,11 @@ Runtime indexing, database drivers, Cloudflare D1, and the server query helpers:
 
 ## Config
 
+- `llmsTxt` (`object`): generated discovery. Set `false` for application-owned server routes. A public `llms.txt` disables generation automatically.
+- `llmsTxt.pages` (`'sitemap'`): sitemap admission. Use `'all'` for stored pages outside the sitemap, or `false` for configured sections only.
+- `llmsTxt.sitemaps`: selected sitemap names. An empty array stops sitemap discovery.
+- `llmsTxt.maxLength` (`80000`): maximum characters per file. Minimum `1024`. Larger lists split without dropping links.
+- `llmsFullTxt` (`true`): generate the full export and advertise its link. Set `false` to omit both.
 - `llmsTxt.directive` (`{ markdown: true, html: false }`): a `> Index of every page on this site: [llms.txt](...)` line after the frontmatter of each page's `.md`; `html: true` adds a visually hidden `/llms.txt` link at the top of each HTML body.
 - `llmsTxt.markdownLinks` (`true`): link page entries in `llms.txt` to their `.md` twins. `false` links canonical URLs. With `site.url` set, page links in `llms.txt`, `sitemap.md`, and `.md` bodies are absolute.
 - `describedby` (`true`): the `describedby` link tag and header.