nuxt-ai-ready 3.2.0 → 3.3.1

package/dist/chunks/agent-skills.mjs

@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto';
 import { realpath, readFile, readdir } from 'node:fs/promises';
 import { join, resolve, isAbsolute, relative, sep } from 'node:path';
 import { parseDocument } from 'yaml';
-import { A as AGENT_SKILLS_SCHEMA } from '../shared/nuxt-ai-ready.DH9mSayy.mjs';
+import { A as AGENT_SKILLS_SCHEMA } from '../shared/nuxt-ai-ready.CKXM1jnE.mjs';
 import 'node:module';
 import 'node:url';
 import '@nuxt/kit';
@@ -11,6 +11,7 @@ import 'nuxt-site-config/kit';
 import 'nuxtseo-shared/kit';
 import '../shared/nuxt-ai-ready.B3hAuOLQ.mjs';
 import 'node:fs';
+import '../../dist/runtime/llms-txt-files.js';
 import '../../dist/runtime/route-path.js';
 import '../../dist/runtime/server/utils/sitemap-md.js';
 import '../../dist/runtime/cache-control.js';

package/dist/chunks/prerender.mjs

@@ -1,16 +1,16 @@
-import { mkdir, writeFile, appendFile, stat, readdir } from 'node:fs/promises';
-import { join, dirname, relative, resolve } from 'node:path';
+import { mkdir, writeFile, appendFile, readFile, stat, readdir } from 'node:fs/promises';
+import { resolve, relative, join, dirname } from 'node:path';
 import { useNuxt, getNitroVersion, hasNuxtModule, resolveFiles } from '@nuxt/kit';
 import { colorize } from 'consola/utils';
 import { resolveLocaleFromRoute } from 'nuxtseo-shared/i18n-runtime';
 import { createPrerenderFetch } from 'nuxtseo-shared/prerender';
 import { collectSitemap } from 'sitemapd/parse';
-import { withLeadingSlash, joinURL, withBase } from 'ufo';
-import { l as logger, M as MARKDOWN_LINK_AVAILABILITY_FILE } from '../shared/nuxt-ai-ready.DH9mSayy.mjs';
+import { parseURL, withLeadingSlash, joinURL, withBase } from 'ufo';
+import { l as logger, M as MARKDOWN_LINK_AVAILABILITY_FILE } from '../shared/nuxt-ai-ready.CKXM1jnE.mjs';
+import { toLogicalRoute, toDeployedRoute } from '../../dist/runtime/route-path.js';
 import { insertLlmsTxtDirective } from '../../dist/runtime/llms-txt-directive.js';
 import { absoluteSiteUrl } from '../../dist/runtime/markdown-links.js';
-import { normalizePagePath, toMarkdownPath } from '../../dist/runtime/markdown-path.js';
-import { toDeployedRoute, toLogicalRoute } from '../../dist/runtime/route-path.js';
+import { normalizePagePath, hasMarkdownTwin, toMarkdownPath } from '../../dist/runtime/markdown-path.js';
 import { initSchema, computeContentHash, insertPage, queryAllPages, exportDbDump } from '../../dist/runtime/server/db/shared.js';
 import { buildLlmsFullTxtHeader, formatPageForLlmsFullTxt } from '../../dist/runtime/server/utils/llms-full.js';
 import { resolvePageUpdatedAt } from '../../dist/runtime/server/utils/page-updated-at.js';
@@ -23,6 +23,7 @@ import 'nuxt-site-config/kit';
 import 'nuxtseo-shared/kit';
 import '../shared/nuxt-ai-ready.B3hAuOLQ.mjs';
 import 'node:fs';
+import '../../dist/runtime/llms-txt-files.js';
 import '../../dist/runtime/cache-control.js';
 import '../../dist/runtime/server/utils/discovery-response.js';
 import 'confbox';
@@ -31,6 +32,33 @@ import '../../dist/runtime/server/utils/content-negotiation.js';
 import 'pathe';
 import 'nuxtseo-shared/i18n';
 
+function parsePrerenderRoute(input, baseURL) {
+  let pathname = parseURL(input).pathname;
+  try {
+    pathname = decodeURIComponent(pathname);
+  } catch {
+    return { _tag: "Invalid" };
+  }
+  while (true) {
+    if (pathname.includes("\\") || [...pathname].some((character) => character.charCodeAt(0) < 32 || character.charCodeAt(0) === 127) || pathname.split("/").some((segment) => segment === "." || segment === ".."))
+      return { _tag: "Invalid" };
+    const decoded = pathname.replace(/%([\da-f]{2})/gi, (_, byte) => String.fromCharCode(Number.parseInt(byte, 16)));
+    if (decoded === pathname)
+      break;
+    pathname = decoded;
+  }
+  return { _tag: "Valid", route: toLogicalRoute(input, baseURL) };
+}
+function resolvePrerenderFile(root, route) {
+  if (parsePrerenderRoute(route, "/")._tag === "Invalid")
+    throw new Error(`Unsafe prerender file path: ${route}`);
+  const destination = resolve(root, `.${route.startsWith("/") ? route : `/${route}`}`);
+  const child = relative(resolve(root), destination);
+  if (child === ".." || child.startsWith("../") || child.startsWith("..\\"))
+    throw new Error(`Prerender file path escapes its root: ${route}`);
+  return destination;
+}
+
 const PRERENDER_PAGE_TIMEOUT = 3e4;
 const RE_HTML_MD_EXT = /\.(html|md)$/;
 const RE_INDEX_SUFFIX = /\/index$/;
@@ -186,7 +214,32 @@ async function processMarkdownRoute(state, nuxt, route, parsed, lastmod, options
 async function processSitemapEntry(state, nuxt, nitro, entry) {
   const loc = typeof entry === "string" ? entry : entry.loc;
   const lastmod = typeof entry === "string" ? void 0 : entry.lastmod;
-  const route = toLogicalRoute(loc, nitro.options.baseURL);
+  const parsedRoute = parsePrerenderRoute(loc, nitro.options.baseURL);
+  if (parsedRoute._tag === "Invalid") {
+    logger.warn(`Skipping unsafe sitemap URL: ${loc}`);
+    return { crawled: false, skipped: true };
+  }
+  const route = parsedRoute.route;
+  if (!hasMarkdownTwin(route))
+    return { crawled: false, skipped: true };
+  if (nitro.options.static) {
+    for (const asset of nitro.options.publicAssets) {
+      const assetBase = asset.baseURL || "/";
+      if (!route.startsWith(assetBase))
+        continue;
+      const assetRoute = route.slice(assetBase.length);
+      for (const htmlRoute of [`${assetRoute}.html`, `${assetRoute}/index.html`]) {
+        const htmlPath = resolvePrerenderFile(asset.dir, htmlRoute);
+        const existing = await stat(htmlPath).catch((error) => {
+          if (error.code === "ENOENT")
+            return void 0;
+          throw error;
+        });
+        if (existing?.isFile())
+          return { crawled: false, skipped: true };
+      }
+    }
+  }
   if (route.split("/").some((segment) => segment.startsWith("_"))) {
     return { crawled: false, skipped: true };
   }
@@ -195,12 +248,24 @@ async function processSitemapEntry(state, nuxt, nitro, entry) {
   }
   const mdRoute = toMarkdownPath(route);
   const mdUrl = toDeployedRoute(mdRoute, nitro.options.baseURL);
+  const candidates = [route, toDeployedRoute(route, nitro.options.baseURL), mdRoute, mdUrl];
+  const ignored = nitro.options.prerender.ignore.some((pattern) => candidates.some((path) => {
+    if (typeof pattern === "string")
+      return path.startsWith(pattern);
+    if (typeof pattern === "function")
+      return pattern(path) === true;
+    return pattern.test(path);
+  }));
+  if (ignored)
+    return { crawled: false, skipped: true };
   logger.debug(`Fetching markdown for ${route} \u2192 ${mdUrl}`);
   const res = await crawlerFetch(state)(mdUrl, {
     headers: { "x-nitro-prerender": mdRoute },
     retry: 0,
     signal: AbortSignal.timeout(PRERENDER_PAGE_TIMEOUT)
   }).catch((err) => {
+    if (err?.response?.status === 404 || err?.response?.status === 410)
+      state.errorRoutes.add(route);
     if (err?.name === "TimeoutError" || err?.name === "AbortError")
       logger.warn(`Timeout (${PRERENDER_PAGE_TIMEOUT}ms) fetching markdown for ${route}`);
     else
@@ -217,6 +282,29 @@ async function processSitemapEntry(state, nuxt, nitro, entry) {
     return { crawled: false, skipped: false };
   }
   await processMarkdownRoute(state, nuxt, route, parsed, lastmod, { skipLlmsFullTxt: true });
+  if (nitro.options.static) {
+    for (const asset of nitro.options.publicAssets) {
+      const assetBase = asset.baseURL || "/";
+      if (!mdRoute.startsWith(assetBase))
+        continue;
+      const source = resolvePrerenderFile(asset.dir, mdRoute.slice(assetBase.length));
+      const existing = await stat(source).catch((error) => {
+        if (error.code === "ENOENT")
+          return void 0;
+        throw error;
+      });
+      if (existing?.isFile())
+        return { crawled: true, skipped: false };
+    }
+    const generated = { route: mdRoute, fileName: mdRoute, contents: JSON.stringify(parsed) };
+    await nitro.hooks.callHook("prerender:generate", generated, nitro);
+    if (generated.skip || generated.error)
+      return { crawled: true, skipped: false };
+    const filePath = resolvePrerenderFile(nitro.options.output.publicDir, mdRoute);
+    await mkdir(dirname(filePath), { recursive: true });
+    await writeFile(filePath, generated.contents || "", "utf8");
+    nitro._prerenderedRoutes.push(generated);
+  }
   return { crawled: true, skipped: false };
 }
 async function crawlSitemapEntries(state, nuxt, nitro, entries) {
@@ -237,18 +325,79 @@ async function crawlSitemapEntries(state, nuxt, nitro, entries) {
   logger.debug(`Sitemap crawl complete: ${crawled} crawled, ${skipped} skipped`);
   return crawled;
 }
-async function crawlSitemapContent(state, nuxt, nitro, sitemapContent) {
+async function crawlSitemapContent(state, nuxt, nitro, sitemapContent, seen = /* @__PURE__ */ new Set(), depth = 0, documents = /* @__PURE__ */ new Map(), budget = { fetched: 0, warned: false }) {
   logger.debug(`Parsing sitemap XML (${sitemapContent.length} bytes)`);
   const result = await collectSitemap(sitemapContent);
-  if (result._tag !== "document" || result.document._tag !== "urlset") {
+  if (result._tag !== "document") {
     const issues = result.issues.map((issue) => issue.message).join("; ");
     logger.debug(`Skipping sitemap: ${issues || "document is not a URL set"}`);
     return 0;
   }
+  if (result.document._tag === "index") {
+    let crawled = 0;
+    const origin = new URL(state.siteInfo?.url || "http://localhost").origin;
+    for (const entry of result.document.entries) {
+      if (parsePrerenderRoute(entry.loc, nitro.options.baseURL)._tag === "Invalid")
+        continue;
+      const url = new URL(entry.loc, origin);
+      if (url.origin !== origin)
+        continue;
+      const documentKey = toLogicalRoute(url.pathname, nitro.options.baseURL);
+      const localDocument = documents.get(documentKey);
+      const identity = localDocument ? `local:${documentKey}` : url.href;
+      if (seen.has(identity))
+        continue;
+      if (!localDocument && (depth >= 3 || budget.fetched >= 100)) {
+        if (!budget.warned) {
+          logger.warn("Sitemap crawl reached its document limit.");
+          budget.warned = true;
+        }
+        continue;
+      }
+      seen.add(identity);
+      if (!localDocument)
+        budget.fetched++;
+      const content = localDocument?.() ?? await crawlerFetch(state)(`${url.pathname}${url.search}`, {
+        headers: { "x-nitro-prerender": url.pathname },
+        retry: 0,
+        signal: AbortSignal.timeout(PRERENDER_PAGE_TIMEOUT)
+      }).catch((error) => {
+        logger.warn(`Sitemap crawl failed for ${url.pathname}: ${String(error)}`);
+        return null;
+      });
+      if (content)
+        crawled += await crawlSitemapContent(state, nuxt, nitro, content, seen, depth + Number(!localDocument), documents, budget);
+    }
+    return crawled;
+  }
   const urls = result.document.entries;
   logger.debug(`Found ${urls.length} URLs in sitemap`);
   return crawlSitemapEntries(state, nuxt, nitro, urls);
 }
+async function crawlConfiguredSitemaps(state, nuxt, nitro, sitemap, documents = /* @__PURE__ */ new Map()) {
+  const names = state.llmsTxtConfig?.sitemaps;
+  const singleName = (sitemap.sitemapName || "sitemap.xml").replace(/\.xml$/, "");
+  const configured = sitemap.sitemaps && typeof sitemap.sitemaps === "object" ? sitemap.sitemaps : void 0;
+  const roots = configured ? Object.entries(configured).flatMap(([key, definition]) => {
+    const name = (definition.sitemapName || key).replace(/\.xml$/, "");
+    if (key === "index" || names !== void 0 && !names.includes(name))
+      return [];
+    const route = definition._route || definition.route || (key === "chunks" ? configured.index?._route || configured.index?.route : void 0);
+    return [route ? toLogicalRoute(route, nitro.options.baseURL) : joinURL(sitemap.sitemapsPathPrefix ?? "/__sitemap__/", `${name}.xml`)];
+  }) : names !== void 0 && !names.includes(sitemap.sitemaps ? "chunks" : singleName) ? [] : [sitemap.sitemaps ? "/sitemap_index.xml" : `/${sitemap.sitemapName || "sitemap.xml"}`];
+  for (const root of roots) {
+    const content = documents.get(root)?.() ?? await crawlerFetch(state)(toDeployedRoute(root, nitro.options.baseURL), {
+      headers: { "x-nitro-prerender": root },
+      retry: 0,
+      signal: AbortSignal.timeout(PRERENDER_PAGE_TIMEOUT)
+    }).catch((error) => {
+      logger.warn(`Sitemap crawl failed for ${root}: ${String(error)}`);
+      return null;
+    });
+    if (content)
+      await crawlSitemapContent(state, nuxt, nitro, content, /* @__PURE__ */ new Set(), 0, documents);
+  }
+}
 function isNuxtGenerate() {
   return process.argv.includes("generate") || process.env.NUXT_GENERATE === "true" || process.env.prerender === "true";
 }
@@ -274,6 +423,7 @@ function detectSitemapPrerender(sitemapName = "sitemap.xml") {
   };
 }
 async function prerenderRoute(nitro, route, fetch) {
+  const filePath = resolvePrerenderFile(nitro.options.output.publicDir, route);
   const start = Date.now();
   const encodedRoute = encodeURI(route);
   const fetchUrl = withBase(encodedRoute, nitro.options.baseURL);
@@ -282,7 +432,6 @@ async function prerenderRoute(nitro, route, fetch) {
     retry: nitro.options.prerender.retry,
     retryDelay: nitro.options.prerender.retryDelay
   });
-  const filePath = join(nitro.options.output.publicDir, route);
   logger.debug(`Creating directory for prerender: ${dirname(filePath)}`);
   await mkdir(dirname(filePath), { recursive: true });
   const data = res._data;
@@ -304,7 +453,7 @@ async function prerenderRoute(nitro, route, fetch) {
 function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras) {
   const nuxt = useNuxt();
   nuxt.hooks.hook("nitro:init", async (nitro) => {
-    const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
+    const llmsFullTxtPath = options.llmsFullTxt === false ? void 0 : join(nitro.options.output.publicDir, "llms-full.txt");
     const state = createCrawlerState(
       dbPath,
       llmsFullTxtPath,
@@ -348,7 +497,8 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras)
         initPromise = initCrawler(state);
       await initPromise;
       const parsed = JSON.parse(route.contents || "{}");
-      await processMarkdownRoute(state, nuxt, pageRoute, parsed);
+      if (!state.prerenderedRoutes.has(pageRoute))
+        await processMarkdownRoute(state, nuxt, pageRoute, parsed);
       const withSitemap = options.sitemapMd === false ? parsed.markdown : appendSitemapSection(parsed.markdown, absoluteSiteUrl(toDeployedRoute(SITEMAP_MD_ROUTE, nitro.options.baseURL), siteInfo?.url));
       route.contents = llmsTxtConfig?.directive?.markdown === false ? withSitemap : insertLlmsTxtDirective(withSitemap, absoluteSiteUrl(toDeployedRoute("/llms.txt", nitro.options.baseURL), siteInfo?.url));
       route.contentType = "text/markdown; charset=utf-8";
@@ -426,20 +576,46 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras)
           paths
         }), "utf-8");
       }
-      const llmsStats = await prerenderRoute(nitro, "/llms.txt", crawlerFetch(state));
-      const llmsFullStats = await stat(state.llmsFullTxtPath);
-      nitro._prerenderedRoutes.push({ route: "/llms-full.txt", fileName: "/llms-full.txt" });
+      let llmsSize = 0;
+      if (options.llmsTxt !== false) {
+        const seen = /* @__PURE__ */ new Set();
+        const pending = ["/llms.txt"];
+        while (pending.length) {
+          const route = pending.shift();
+          if (seen.has(route))
+            continue;
+          seen.add(route);
+          const stats = await prerenderRoute(nitro, route, crawlerFetch(state));
+          llmsSize += stats.size;
+          const body = await readFile(resolvePrerenderFile(nitro.options.output.publicDir, route), "utf-8");
+          for (const match of body.matchAll(/\]\(([^)]+)\)/g)) {
+            const origin = new URL(siteInfo?.url || "http://localhost").origin;
+            const parsedTarget = parsePrerenderRoute(match[1], nitro.options.baseURL);
+            if (parsedTarget._tag === "Invalid")
+              continue;
+            const target = new URL(match[1], origin);
+            if (target.origin !== origin)
+              continue;
+            const path = parsedTarget.route;
+            if (/^\/llms\/[^/]+\.txt$/i.test(path))
+              pending.push(path);
+          }
+        }
+      }
+      const llmsFullStats = state.llmsFullTxtPath ? await stat(state.llmsFullTxtPath) : { size: 0 };
+      if (state.llmsFullTxtPath)
+        nitro._prerenderedRoutes.push({ route: "/llms-full.txt", fileName: "/llms-full.txt" });
       if (options.sitemapMd !== false) {
         const sitemapMdStats = await prerenderRoute(nitro, SITEMAP_MD_ROUTE, crawlerFetch(state));
         logger.debug(`Wrote sitemap.md (${(sitemapMdStats.size / 1024).toFixed(1)}kb)`);
       }
       const kb = (b) => (b / 1024).toFixed(1);
-      const totalKb = kb(llmsStats.size + llmsFullStats.size);
+      const totalKb = kb(llmsSize + llmsFullStats.size);
       const dim = (s) => colorize("dim", s);
       const cyan = (s) => colorize("cyan", s);
       const timeStr = state.totalProcessingTime >= 100 ? ` in ${cyan(`${(state.totalProcessingTime / 1e3).toFixed(1)}s`)}` : "";
       logger.info(`Indexed ${cyan(String(state.prerenderedRoutes.size))} pages for llms.txt${timeStr} \u2192 ${cyan(`${totalKb}kb`)}`);
-      logger.info(dim(`  llms.txt: ${kb(llmsStats.size)}kb, llms-full.txt: ${kb(llmsFullStats.size)}kb`));
+      logger.info(dim(`  llms.txt files: ${kb(llmsSize)}kb, llms-full.txt: ${kb(llmsFullStats.size)}kb`));
     }
     const { useSitemapHook, usePrerenderHook } = detectSitemapPrerender();
     logger.debug(`Prerender hooks: sitemap=${useSitemapHook}, prerender=${usePrerenderHook}`);
@@ -447,8 +623,8 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras)
       nuxt.hooks.hook("sitemap:prerender:done", async (ctx) => (async () => {
         if (!state.initialized)
           return;
-        for (const sitemap of ctx.sitemaps)
-          await crawlSitemapContent(state, nuxt, nitro, sitemap.content);
+        const documents = new Map(ctx.sitemaps.map((sitemap) => [toLogicalRoute(sitemap.name, nitro.options.baseURL), () => sitemap.content]));
+        await crawlConfiguredSitemaps(state, nuxt, nitro, ctx.options, documents);
         await writeLlmsFiles();
         state.prerenderedRoutes.clear();
         if (state.db)
@@ -458,14 +634,7 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras)
       nitro.hooks.hook("prerender:done", async () => (async () => {
         if (!state.initialized)
           return;
-        const sitemapContent = await crawlerFetch(state)(toDeployedRoute("/sitemap.xml", nitro.options.baseURL), {
-          headers: { "x-nitro-prerender": "/sitemap.xml" },
-          signal: AbortSignal.timeout(PRERENDER_PAGE_TIMEOUT)
-        }).catch(() => {
-          return null;
-        });
-        if (sitemapContent)
-          await crawlSitemapContent(state, nuxt, nitro, sitemapContent);
+        await crawlConfiguredSitemaps(state, nuxt, nitro, nuxt.options.sitemap || {});
         await writeLlmsFiles();
         state.prerenderedRoutes.clear();
         if (state.db)
@@ -475,4 +644,4 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras)
   });
 }
 
-export { createPrerenderDatabase, detectSitemapPrerender, setupPrerenderHandler };
+export { crawlConfiguredSitemaps, crawlSitemapContent, createPrerenderDatabase, detectSitemapPrerender, setupPrerenderHandler };

package/dist/cli.mjs

@@ -10,6 +10,240 @@ import 'node:module';
 import 'node:path';
 import 'node:url';
 
+const capabilities = ["llms", "api-catalog", "ai-catalog", "mcp", "skills"];
+function publicUrl(input, base) {
+  if (!URL.canParse(input, base))
+    return { _tag: "Err", message: "Use an absolute HTTP or HTTPS URL." };
+  const url = new URL(input, base);
+  if (!["http:", "https:"].includes(url.protocol) || url.username || url.password)
+    return { _tag: "Err", message: "Use an HTTP or HTTPS URL without credentials." };
+  url.hash = "";
+  return { _tag: "Ok", value: url };
+}
+function parseAuditOptions(input) {
+  const url = publicUrl(input.url);
+  if (url._tag === "Err")
+    return url;
+  const base = input.base || "/";
+  if (!base.startsWith("/") || base.startsWith("//") || /[?#\\]/.test(base))
+    return { _tag: "Err", message: "Use an application base path such as /docs/." };
+  const selected = [...new Set((input.capabilities || "").split(",").map((value) => value.trim()).filter(Boolean))];
+  const unknown = selected.filter((value) => !capabilities.includes(value));
+  if (unknown.length)
+    return { _tag: "Err", message: `Unknown capabilities: ${unknown.join(", ")}. Use ${capabilities.join(", ")}.` };
+  return { _tag: "Ok", value: { url: url.value.href, base: `${new URL(base, url.value).pathname.replace(/\/+$/, "")}/`, capabilities: selected } };
+}
+async function readDocument(url, accept, dependencies) {
+  const maxBytes = dependencies.maxBytes ?? 2 * 1024 * 1024;
+  const signal = AbortSignal.timeout(dependencies.timeoutMs ?? 1e4);
+  return Promise.resolve().then(async () => {
+    let current = url;
+    for (let redirects = 0; redirects <= 5; redirects++) {
+      const response = await dependencies.fetch(current, { headers: { accept }, redirect: "manual", signal });
+      if ([301, 302, 303, 307, 308].includes(response.status)) {
+        await response.body?.cancel();
+        const location = response.headers.get("location");
+        const next = location ? publicUrl(location, current) : { _tag: "Err", message: "Redirect has no Location header." };
+        if (next._tag === "Err")
+          return next;
+        current = next.value.href;
+        continue;
+      }
+      if (Number(response.headers.get("content-length")) > maxBytes) {
+        await response.body?.cancel();
+        return { _tag: "Err", message: `Response exceeds the ${maxBytes} byte limit.` };
+      }
+      const reader = response.body?.getReader();
+      const decoder = new TextDecoder();
+      let body = "";
+      let bytes = 0;
+      if (reader) {
+        while (true) {
+          const chunk = await reader.read();
+          if (chunk.done)
+            break;
+          bytes += chunk.value.byteLength;
+          if (bytes > maxBytes) {
+            await reader.cancel();
+            return { _tag: "Err", message: `Response exceeds the ${maxBytes} byte limit.` };
+          }
+          body += decoder.decode(chunk.value, { stream: true });
+        }
+        body += decoder.decode();
+      }
+      return { _tag: "Ok", value: { url: current, status: response.status, headers: response.headers, body } };
+    }
+    return { _tag: "Err", message: "Response exceeds the five redirect limit." };
+  }).catch((error) => ({ _tag: "Err", message: `Request failed: ${error instanceof Error ? error.message : String(error)}` }));
+}
+function responseFailure(document) {
+  if (document._tag === "Err")
+    return document.message;
+  if (document.value.headers.get("cf-mitigated") === "challenge")
+    return "Cloudflare returned a challenge. Check crawler access rules.";
+  if (document.value.status < 200 || document.value.status >= 300)
+    return `HTTP ${document.value.status}. Check this public URL.`;
+}
+function isHtml(body) {
+  return /^\s*(?:<!doctype html|<html|<head|<body)/i.test(body);
+}
+function contentCheck(id, url, document, accepts, success, failure) {
+  const error = responseFailure(document);
+  if (error)
+    return { id, url, status: "fail", detail: error };
+  return document._tag === "Ok" && accepts(document.value) ? { id, url: document.value.url, status: "pass", detail: success } : { id, url, status: "fail", detail: failure };
+}
+function markdown(document) {
+  return /^text\/markdown(?:;|$)/i.test(document.headers.get("content-type") || "") && !!document.body.trim() && !isHtml(document.body);
+}
+function links(header) {
+  const source = header || "";
+  const entries = [];
+  let quoted = false;
+  let target = false;
+  let start = 0;
+  for (let i = 0; i < source.length; i++) {
+    const character = source[i];
+    if (character === '"' && source[i - 1] !== "\\")
+      quoted = !quoted;
+    if (!quoted) {
+      if (character === "<")
+        target = true;
+      if (character === ">")
+        target = false;
+      if (character === "," && !target) {
+        entries.push(source.slice(start, i));
+        start = i + 1;
+      }
+    }
+  }
+  entries.push(source.slice(start));
+  return entries.flatMap((entry) => {
+    const match = entry.match(/^\s*<([^>]+)>([\s\S]*)$/);
+    if (!match)
+      return [];
+    const parameter = (name) => match[2]?.match(new RegExp(`;\\s*${name}\\s*=\\s*(?:"([^"]*)"|([^;\\s]+))`, "i"));
+    const rel = parameter("rel");
+    const type = parameter("type");
+    return [{ href: match[1], rel: (rel?.[1] || rel?.[2] || "").split(/\s+/), type: type?.[1] || type?.[2] || "" }];
+  });
+}
+function policy(header) {
+  const signals = {};
+  for (const part of header.split(",")) {
+    const match = part.trim().match(/^(search|ai-input|ai-train)\s*=\s*(yes|no)$/);
+    if (!match)
+      return { _tag: "Err", message: "Content-Signal has an invalid name or value." };
+    const name = match[1];
+    const value = match[2];
+    if (signals[name] && signals[name] !== value)
+      return { _tag: "Err", message: `Content-Signal contains conflicting ${name} values.` };
+    signals[name] = value;
+  }
+  return { _tag: "Ok", value: signals };
+}
+function policyCheck(html, md, id = "content-signal") {
+  if (html._tag === "Err" || md._tag === "Err" || responseFailure(html) || responseFailure(md))
+    return { id, status: "skip", detail: "Policy comparison requires both public page responses." };
+  const source = html.value.headers.get("content-signal");
+  const converted = md.value.headers.get("content-signal");
+  if (!source && !converted)
+    return { id, status: "warn", detail: "No HTTP Content-Signal. Check whether this matches your content policy." };
+  const parsedSource = source ? policy(source) : void 0;
+  const parsedConverted = converted ? policy(converted) : void 0;
+  for (const parsed of [parsedSource, parsedConverted]) {
+    if (parsed?._tag === "Err")
+      return { id, status: "fail", detail: parsed.message };
+  }
+  if (parsedSource?._tag === "Ok" && (!parsedConverted || parsedConverted._tag !== "Ok" || Object.keys(parsedSource.value).length !== Object.keys(parsedConverted.value).length || Object.entries(parsedSource.value).some(([key, value]) => parsedConverted.value[key] !== value))) {
+    return { id, status: "fail", detail: "Markdown does not preserve the HTML Content-Signal policy." };
+  }
+  return { id, status: "pass", detail: `HTTP policy: ${converted || source}` };
+}
+function isRecord(value) {
+  return !!value && typeof value === "object" && !Array.isArray(value);
+}
+function parseJson(body) {
+  try {
+    return { _tag: "Ok", value: JSON.parse(body) };
+  } catch {
+    return { _tag: "Err", message: "Discovery response contains invalid JSON." };
+  }
+}
+function discoveryDocument(capability, document) {
+  if (capability === "llms")
+    return !isHtml(document.body) && /^#\s+\S/m.test(document.body);
+  if (!/^application\/(?:[\w.+-]+\+)?json(?:;|$)/i.test(document.headers.get("content-type") || ""))
+    return false;
+  const parsed = parseJson(document.body);
+  if (parsed._tag === "Err")
+    return false;
+  const value = parsed.value;
+  if (!isRecord(value))
+    return false;
+  switch (capability) {
+    case "api-catalog":
+      return Array.isArray(value.linkset) && value.linkset.length > 0 && value.linkset.every((entry) => isRecord(entry) && typeof entry.anchor === "string");
+    case "skills":
+      return Array.isArray(value.skills);
+    case "ai-catalog":
+      return value.specVersion === "1.0" && Array.isArray(value.entries) && value.entries.length > 0 && value.entries.every((entry) => isRecord(entry) && typeof entry.url === "string");
+    case "mcp":
+      return typeof value.name === "string" && typeof value.version === "string" && Array.isArray(value.remotes) && value.remotes.some((remote) => isRecord(remote) && remote.type === "streamable-http" && typeof remote.url === "string");
+  }
+}
+async function auditSite(options, dependencies) {
+  const [html, negotiated] = await Promise.all([
+    readDocument(options.url, "text/html", dependencies),
+    readDocument(options.url, "text/markdown", dependencies)
+  ]);
+  const pageUrl = html._tag === "Ok" && !responseFailure(html) ? html.value.url : options.url;
+  const page = new URL(pageUrl);
+  const site = new URL(options.base, page);
+  const advertised = html._tag === "Ok" ? links(html.value.headers.get("link")) : [];
+  const alternate = advertised.find((link) => link.rel.includes("alternate") && link.type.toLowerCase() === "text/markdown");
+  const linkedUrl = alternate ? publicUrl(alternate.href, pageUrl) : void 0;
+  const direct = new URL(pageUrl);
+  direct.pathname = `${direct.pathname.replace(/\/$/, "") || "/index"}.md`;
+  const robotsUrl = new URL("/robots.txt", page).href;
+  const [directMd, robots] = await Promise.all([
+    readDocument(direct.href, "text/markdown", dependencies),
+    readDocument(robotsUrl, "text/plain", dependencies)
+  ]);
+  const robotsSitemap = robots._tag === "Ok" ? robots.value.body.match(/^sitemap:\s*(\S+)/im)?.[1] : void 0;
+  const sitemapUrl = robotsSitemap ? publicUrl(robotsSitemap, robotsUrl) : publicUrl("sitemap.xml", site.href);
+  const sitemap = sitemapUrl._tag === "Ok" ? await readDocument(sitemapUrl.value.href, "application/xml, text/xml", dependencies) : sitemapUrl;
+  const missingUrl = new URL(".well-known/nuxt-ai-ready-audit-missing", site).href;
+  const missing = await readDocument(missingUrl, "text/markdown", dependencies);
+  const checks = [
+    contentCheck("html", options.url, html, (doc) => /^text\/html(?:;|$)/i.test(doc.headers.get("content-type") || "") && !!doc.body.trim(), "Public HTML is available.", "Expected a non-empty HTML page."),
+    contentCheck("markdown-negotiation", options.url, negotiated, markdown, "Accept: text/markdown returns Markdown.", "Expected a non-empty text/markdown response."),
+    contentCheck("markdown-url", direct.href, directMd, markdown, "The direct Markdown URL is available.", "Expected a non-empty text/markdown response."),
+    { id: "markdown-link", status: linkedUrl?._tag === "Ok" && linkedUrl.value.href === direct.href ? "pass" : "warn", detail: linkedUrl?._tag === "Ok" && linkedUrl.value.href === direct.href ? "The Link header advertises the direct Markdown URL." : alternate ? "Inspect the alternate Markdown Link target." : "No alternate Markdown Link header." },
+    contentCheck("robots", robotsUrl, robots, (doc) => !isHtml(doc.body) && /^user-agent:\s*\S/im.test(doc.body), "robots.txt is available. Inspect its crawler-specific rules.", "Expected robots.txt with a User-agent group."),
+    contentCheck("sitemap", sitemapUrl._tag === "Ok" ? sitemapUrl.value.href : site.href, sitemap, (doc) => /<(?:urlset|sitemapindex)(?:\s|>)/i.test(doc.body) && /<loc>\s*https?:\/\/[^<]+<\/loc>/i.test(doc.body), "A sitemap document is available.", "Expected a sitemap with absolute URL entries."),
+    policyCheck(html, negotiated),
+    policyCheck(html, directMd, "content-signal-markdown-url"),
+    { id: "unknown-route", url: missingUrl, status: missing._tag === "Ok" && [404, 410].includes(missing.value.status) ? "pass" : "fail", detail: missing._tag === "Err" ? missing.message : [404, 410].includes(missing.value.status) ? "Unknown routes return 404 or 410." : `Unknown route returned HTTP ${missing.value.status}. Check fallback routing.` }
+  ];
+  if (negotiated._tag === "Ok" && !responseFailure(negotiated)) {
+    const vary = (negotiated.value.headers.get("vary") || "").toLowerCase().split(",").map((value) => value.trim());
+    checks.push({ id: "vary-accept", status: vary.includes("accept") || vary.includes("*") ? "pass" : "fail", detail: vary.includes("accept") || vary.includes("*") ? "Vary separates HTML and Markdown cache entries." : "Set Vary: Accept on negotiated responses." });
+  }
+  const paths = { "llms": "llms.txt", "api-catalog": ".well-known/api-catalog", "ai-catalog": ".well-known/ai-catalog.json", "mcp": ".well-known/mcp/server-card.json", "skills": ".well-known/agent-skills/index.json" };
+  const discovery = await Promise.all(capabilities.map(async (capability) => {
+    const declaredLink = capability === "api-catalog" ? advertised.find((link) => link.rel.includes("api-catalog")) : void 0;
+    if (!options.capabilities.includes(capability) && !declaredLink)
+      return { id: capability, status: "skip", detail: "Not declared. Use --capabilities to check this feature." };
+    const url = publicUrl(declaredLink?.href || paths[capability], declaredLink ? pageUrl : site.href);
+    if (url._tag === "Err")
+      return { id: capability, status: "fail", detail: url.message };
+    const document = await readDocument(url.value.href, capability === "llms" ? "text/plain" : "application/json", dependencies);
+    return contentCheck(capability, url.value.href, document, (doc) => discoveryDocument(capability, doc), "The discovery document is available.", "The discovery document has an unexpected format.");
+  }));
+  return { schemaVersion: 1, url: options.url, checks: [...checks, ...discovery] };
+}
+
 async function getSecret(cwd) {
   const secretPath = join(cwd, "node_modules/.cache/nuxt/ai-ready/secret");
   if (!existsSync(secretPath)) {
@@ -53,6 +287,31 @@ const main = defineCommand({
     version: await readPackageMetadata(import.meta.url).then((p) => p.version || "0.0.0")
   },
   subCommands: {
+    audit: () => defineCommand({
+      meta: { name: "audit", description: "Check a public page and its agent discovery documents" },
+      args: {
+        url: { type: "positional", required: true, description: "Public page URL" },
+        base: { type: "string", default: "/", description: "Application base path, such as /docs/" },
+        capabilities: { type: "string", description: "Check declared features: llms,api-catalog,ai-catalog,mcp,skills" },
+        json: { type: "boolean", description: "Print structured JSON results" }
+      },
+      async run({ args }) {
+        const parsed = parseAuditOptions(args);
+        if (parsed._tag === "Err")
+          throw new Error(parsed.message);
+        const report = await auditSite(parsed.value, { fetch });
+        if (args.json) {
+          process.stdout.write(`${JSON.stringify(report, null, 2)}
+`);
+        } else {
+          consola.info(`Readiness audit: ${report.url}`);
+          for (const check of report.checks)
+            consola.log(`${check.status.toUpperCase()} ${check.id}: ${check.detail}`);
+        }
+        if (report.checks.some((check) => check.status === "fail"))
+          process.exitCode = 1;
+      }
+    }),
     status: () => defineCommand({
       meta: {
         name: "status",

package/dist/module.d.mts

@@ -101,6 +101,7 @@ interface ModulePublicRuntimeConfig {
     version: string;
     sitemapMd: boolean;
     describedby: boolean;
+    llmsFullTxt?: boolean;
     mdreamOptions: ModuleOptions['mdreamOptions'];
     markdownCacheHeaders: Required<NonNullable<ModuleOptions['markdownCacheHeaders']>>;
     database: ResolvedDatabase;

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": "^4.6.0 || ^5.0.0-0"
   },
   "configKey": "aiReady",
-  "version": "3.2.0",
+  "version": "3.3.1",
   "builder": {
     "@nuxt/module-builder": "1.0.3",
     "unbuild": "3.6.1"