nuxt-ai-ready 3.2.0 → 3.3.0

package/dist/shared/{nuxt-ai-ready.DH9mSayy.mjs → nuxt-ai-ready.BwnSMqRn.mjs}

@@ -1,5 +1,5 @@
 import { createHash, randomBytes } from 'node:crypto';
-import { readFile, mkdir, writeFile, access } from 'node:fs/promises';
+import { readFile, access, mkdir, writeFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import { dirname, basename, join, resolve, relative } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
@@ -8,6 +8,7 @@ import defu$1, { defu } from 'defu';
 import { useSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
 import { resolveNuxtContentVersion, renderNitroTypeAugmentations, setupRuntimeAliases, setupNitroRuntimeCompatibility } from 'nuxtseo-shared/kit';
 import { r as readPackageMetadata, a as resolvePackageMetadataPath } from './nuxt-ai-ready.B3hAuOLQ.mjs';
+import { parseLlmsTxtConfig } from '../../dist/runtime/llms-txt-files.js';
 import { toDeployedRoute } from '../../dist/runtime/route-path.js';
 import { SITEMAP_MD_ROUTE } from '../../dist/runtime/server/utils/sitemap-md.js';
 import { publicCacheControl, mergeVaryHeader } from '../../dist/runtime/cache-control.js';
@@ -210,7 +211,11 @@ export {}
   });
   addTemplate({
     filename: "types/ai-ready-virtual.d.ts",
-    getContents: () => `declare module '#ai-ready-virtual/read-page-data.mjs' {
+    getContents: () => `declare module '#ai-ready-virtual/sitemap-config.mjs' {
+  const config: { sitemaps?: Record<string, { sitemapName?: string, route?: string, _route?: string }> }
+  export default config
+}
+declare module '#ai-ready-virtual/read-page-data.mjs' {
   export function readMarkdownLinkAvailabilityFromFilesystem(): Promise<{
     runtimeMarkdownAvailable: boolean
     paths: string[]
@@ -1840,7 +1845,7 @@ ${details}`);
     const llmsFullRoute = withSiteUrl("llms-full.txt", { withBase: true });
     defaultLlmsTxtSections.push({
       title: "LLM Resources",
-      links: [
+      links: config.llmsFullTxt === false ? [] : [
         {
           title: "Full Content",
           href: llmsFullRoute,
@@ -1853,18 +1858,31 @@ ${details}`);
       defaultLlmsTxtSections.push({
         title: "Agent Skills",
         description: `Skills an agent can install from this site. The index at ${indexUrl} carries a sha256 digest for each one.`,
-        links: agentSkillsResult.links.map((link) => ({
+        links: [{ title: "Agent Skills index", href: indexUrl, description: "Every published skill and its sha256 digest." }, ...agentSkillsResult.links.map((link) => ({
           title: link.name,
           href: link.source === "local" ? withSiteUrl(link.href.slice(1), { withBase: true }) : resolveExternalSkillUrl(link.href, indexUrl),
           description: link.description
-        }))
+        }))]
       });
     }
+    const directiveConfig = config.llmsTxt === false ? void 0 : config.llmsTxt?.directive;
     const llmsTxtDirective = {
-      markdown: config.llmsTxt?.directive?.markdown !== false,
-      html: config.llmsTxt?.directive?.html === true
+      markdown: directiveConfig?.markdown !== false,
+      html: directiveConfig?.html === true
     };
+    const parsedLlmsTxt = parseLlmsTxtConfig(config.llmsTxt || {});
+    if (parsedLlmsTxt._tag === "Invalid")
+      throw new Error(`[nuxt-ai-ready] ${parsedLlmsTxt.message}`);
+    const publicLlmsTxt = await Promise.all(getLayerDirectories(nuxt).map(
+      (layer) => access(join(layer.public, "llms.txt")).then(() => true).catch((error) => {
+        if (error.code === "ENOENT")
+          return false;
+        throw error;
+      })
+    ));
+    const generateLlmsTxt = config.llmsTxt !== false && !publicLlmsTxt.some(Boolean);
     const mergedLlmsTxt = config.llmsTxt ? {
+      ...parsedLlmsTxt.config,
       markdownLinks: config.llmsTxt.markdownLinks ?? true,
       directive: llmsTxtDirective,
       sections: [
@@ -1880,6 +1898,9 @@ ${details}`);
     await nuxt.callHook("ai-ready:llms-txt", llmsTxtPayload);
     mergedLlmsTxt.sections = llmsTxtPayload.sections;
     mergedLlmsTxt.notes = llmsTxtPayload.notes.length > 0 ? llmsTxtPayload.notes : void 0;
+    const parsedHookLlmsTxt = parseLlmsTxtConfig(mergedLlmsTxt);
+    if (parsedHookLlmsTxt._tag === "Invalid")
+      throw new Error(`[nuxt-ai-ready] ${parsedHookLlmsTxt.message}`);
     const prerenderCacheDir = join(nuxt.options.rootDir, "node_modules/.cache/nuxt-seo/ai-ready/routes");
     const buildDbPath = join(nuxt.options.buildDir, ".data/ai-ready/build.db");
     const runtimeSyncConfig = typeof config.runtimeSync === "object" ? config.runtimeSync : {};
@@ -1896,6 +1917,8 @@ ${details}`);
       if (apiCatalogRegistered)
         return;
       apiCatalogRegistered = true;
+      const resources = mergedLlmsTxt.sections?.find((section) => section.title === "LLM Resources");
+      resources?.links?.push({ title: "API catalog", href: catalog.href, description: "The public API and service descriptions." });
       addServerHandler({ route: API_CATALOG_PATH, handler: resolve$1("./runtime/server/routes/api-catalog"), lazy: true });
       const apiCatalogLink = `<${catalog.href}>; rel="api-catalog"`;
       extendRouteRules("/**", {
@@ -2273,6 +2296,7 @@ export function trackDrizzleWork(event, work) { return work }
         cron: !!config.cron
       })}`;
       nitroConfig.virtual["#ai-ready-virtual/content-lookup.mjs"] = contentLookupModule(contentSource);
+      nitroConfig.virtual["#ai-ready-virtual/sitemap-config.mjs"] = `export { default } from '#sitemap-virtual/static-config.mjs'`;
     });
     nuxt.options.runtimeConfig["nuxt-ai-ready"] = {
       version: version || "0.0.0",
@@ -2288,6 +2312,7 @@ export function trackDrizzleWork(event, work) { return work }
         swr: true
       }),
       llmsTxt: mergedLlmsTxt,
+      llmsFullTxt: config.llmsFullTxt !== false,
       llmsTxtCacheSeconds: config.llmsTxtCacheSeconds ?? 600,
       prerenderCacheDir,
       database,
@@ -2335,10 +2360,14 @@ export function trackDrizzleWork(event, work) { return work }
       mode: "server",
       src: resolve$1("./runtime/app/plugins/md-alternate.server")
     });
-    addServerHandler({ route: "/llms.txt", handler: resolve$1("./runtime/server/routes/llms.txt.get"), lazy: true });
-    addServerHandler({ route: "/llms-full.txt", handler: resolve$1("./runtime/server/routes/llms-full.txt.get"), lazy: true });
+    if (generateLlmsTxt) {
+      for (const route of ["/llms.txt", "/llms/**"])
+        addServerHandler({ route, method: void 0, handler: resolve$1("./runtime/server/routes/llms.txt.get"), lazy: true });
+    }
+    if (config.llmsFullTxt !== false)
+      addServerHandler({ route: "/llms-full.txt", method: void 0, handler: resolve$1("./runtime/server/routes/llms-full.txt.get"), lazy: true });
     if (config.sitemapMd !== false) {
-      addServerHandler({ route: SITEMAP_MD_ROUTE, handler: resolve$1("./runtime/server/routes/sitemap.md.get"), lazy: true });
+      addServerHandler({ route: SITEMAP_MD_ROUTE, method: void 0, handler: resolve$1("./runtime/server/routes/sitemap.md.get"), lazy: true });
     }
     if (agentSkillsResult._tag === "Enabled") {
       addServerHandler({ route: AGENT_SKILLS_INDEX_ROUTE, handler: resolve$1("./runtime/server/routes/agent-skills-index"), lazy: true });
@@ -2414,7 +2443,7 @@ export function trackDrizzleWork(event, work) { return work }
     }
     if (prerenderEnabled) {
       const { setupPrerenderHandler } = await import('../chunks/prerender.mjs');
-      setupPrerenderHandler(config, buildDbPath, {
+      setupPrerenderHandler({ ...config, llmsTxt: generateLlmsTxt ? config.llmsTxt : false }, buildDbPath, {
         name: siteConfig.name,
         url: siteConfig.url ? withSiteUrl("/", { withBase: true }) : void 0,
         description: siteConfig.description
@@ -2437,7 +2466,7 @@ export function trackDrizzleWork(event, work) { return work }
         icon: "carbon:ai-label"
       }, resolve$1, nuxt);
     }
-    for (const route of ["/llms.txt", "/llms-full.txt"]) {
+    for (const route of [generateLlmsTxt && "/llms.txt", generateLlmsTxt && "/llms/**", config.llmsFullTxt !== false && "/llms-full.txt"].filter((route2) => Boolean(route2))) {
       extendRouteRules(route, { headers: { "Content-Type": "text/plain; charset=utf-8" } });
     }
     if (config.sitemapMd !== false) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "3.2.0",
+  "version": "3.3.0",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",

package/skills/nuxt-ai-ready/SKILL.md

@@ -31,7 +31,7 @@ A page visit in dev or production does not index anything.
 
 With no `aiReady` config, the module adds:
 
-- `/llms.txt`: site name, description, resource links, and a `## Pages` list. In dev it lists sitemap routes without titles.
+- `/llms.txt`: site name, description, resource links, and current sitemap pages. Stored pages supply metadata. Large lists split under `/llms/`.
 - `/llms-full.txt`: the full Markdown of each prerendered page. Only prerendered pages go in the static file.
 - A `.md` twin per page. `/about` becomes `/about.md`, and `/` becomes `/index.md`. Paths under `/api` and `/_` get none.
 - Accept negotiation: a request that prefers `text/markdown` gets a `307` to the `.md` URL. Responses send `Vary: Accept`. AI bot user agents get Markdown only with `botNegotiation: true`.
@@ -58,6 +58,32 @@ export default defineNuxtConfig({
 })
 ```
 
+Publish separate discovery files from named sitemaps. Use module configuration before writing sitemap and database middleware:
+
+```ts
+export default defineNuxtConfig({
+  aiReady: {
+    llmsFullTxt: false,
+    llmsTxt: {
+      markdownLinks: true,
+      sitemaps: ['pages', 'skills', 'demos'],
+      sections: [
+        { title: 'Skills', sitemap: 'skills', path: '/llms/skills.txt' },
+        { title: 'Demos', sitemap: 'demos', path: '/llms/demos.txt' },
+      ],
+    },
+  },
+})
+```
+
+The root links each section file. Chunked sitemaps keep their configured name.
+Every file stays within `maxLength`, default 80,000 characters. Static builds emit every linked child.
+GET and HEAD work on every generated discovery file. Empty sections produce no link or file.
+
+For manual discovery, place `public/llms.txt`. The module preserves it and keeps Markdown conversion enabled.
+If a server handler owns discovery, set `llmsTxt: false`.
+Use `llmsFullTxt: false` instead of redirecting the full export to the index.
+
 Remove an element from the Markdown on every path, build and runtime. Put it in `mdreamOptions`; the defaults `minimal` and `clean` stay on:
 
 ```ts
@@ -92,6 +118,8 @@ Runtime indexing, database drivers, Cloudflare D1, and the server query helpers:
 
 ## Traps
 
+- **Discovery is not authorization.** Selected sitemaps exclude stale rows from `llms.txt`; they do not delete stored content or protect other APIs.
+- **Custom route policies must allow Markdown twins.** If middleware rejects `/skills/planning.md`, allow it whenever `/skills/planning` is allowed.
 - **The same edit in both hooks applies twice.** During prerendering the Nitro `ai-ready:page:markdown` hook runs, then the Nuxt hook of the same name. Put an edit in one of them, not both.
 - **Content source pages skip conversion hooks.** A Nuxt Content page ignores `mdreamOptions` and both Nitro hooks.
 - **Dev has no page data.** Query helpers, MCP tools, and search return empty results in `nuxt dev`. Test with `nuxt build` and `node .output/server/index.mjs`.
@@ -126,6 +154,11 @@ Runtime indexing, database drivers, Cloudflare D1, and the server query helpers:
 
 ## Config
 
+- `llmsTxt` (`object`): generated discovery. Set `false` for application-owned server routes. A public `llms.txt` disables generation automatically.
+- `llmsTxt.pages` (`'sitemap'`): sitemap admission. Use `'all'` for stored pages outside the sitemap, or `false` for configured sections only.
+- `llmsTxt.sitemaps`: selected sitemap names. An empty array stops sitemap discovery.
+- `llmsTxt.maxLength` (`80000`): maximum characters per file. Minimum `1024`. Larger lists split without dropping links.
+- `llmsFullTxt` (`true`): generate the full export and advertise its link. Set `false` to omit both.
 - `llmsTxt.directive` (`{ markdown: true, html: false }`): a `> Index of every page on this site: [llms.txt](...)` line after the frontmatter of each page's `.md`; `html: true` adds a visually hidden `/llms.txt` link at the top of each HTML body.
 - `llmsTxt.markdownLinks` (`true`): link page entries in `llms.txt` to their `.md` twins. `false` links canonical URLs. With `site.url` set, page links in `llms.txt`, `sitemap.md`, and `.md` bodies are absolute.
 - `describedby` (`true`): the `describedby` link tag and header.