nuxt-ai-ready 3.2.0 → 3.3.0

package/dist/runtime/llms-txt-utils.js

@@ -1,23 +1,18 @@
 import { decodePath } from "ufo";
-import { getRequestHost } from "#nuxtseo/h3";
+import { createError, getRequestHost, getRequestURL } from "#nuxtseo/h3";
 import { useRuntimeConfig } from "#nuxtseo/nitro";
 import { getSiteConfig } from "#site-config/server/composables/getSiteConfig";
 import { withSiteTrailingSlash, withSiteUrl } from "#site-config/server/composables/utils";
-import { formatLlmsTxtPageLink, normalizeLlmsTxtConfig } from "./llms-txt-format.js";
-import { absoluteSiteUrl } from "./markdown-links.js";
+import { createLlmsTxtFiles } from "./llms-txt-files.js";
 import { hasMarkdownTwin, toMarkdownPath } from "./markdown-path.js";
 import { normalizePersistedRoute, toDeployedRoute, toLogicalRoute } from "./route-path.js";
-import { queryPages } from "./server/db/queries.js";
+import { queryPageMetadata, queryPages } from "./server/db/queries.js";
 import { logger } from "./server/logger.js";
-import { fetchSitemapUrls } from "./server/utils/sitemap.js";
-export { normalizeLlmsTxtConfig };
+import { fetchSitemapUrlGroups, getSitemapsFromConfig } from "./server/utils/sitemap.js";
+export { normalizeLlmsTxtConfig } from "./llms-txt-format.js";
 async function findMarkdownLinkAvailability() {
-  if (!import.meta.prerender) {
-    return {
-      runtimeMarkdownAvailable: true,
-      paths: /* @__PURE__ */ new Set()
-    };
-  }
+  if (!import.meta.prerender)
+    return { runtimeMarkdownAvailable: true, paths: /* @__PURE__ */ new Set() };
   const { readMarkdownLinkAvailabilityFromFilesystem } = await import("#ai-ready-virtual/read-page-data.mjs");
   const availability = await readMarkdownLinkAvailabilityFromFilesystem();
   return {
@@ -25,231 +20,97 @@ async function findMarkdownLinkAvailability() {
     paths: new Set(availability.paths.map(decodePath))
   };
 }
-function getGroupPrefix(url, depth) {
-  const segments = url.split("/").filter(Boolean);
-  if (segments.length === 0)
-    return "/";
-  if (depth === 1 || segments.length === 1)
-    return `/${segments[0]}`;
-  return `/${segments[0]}/${segments[1]}`;
-}
-function getPathSegments(pathname) {
-  return pathname.split("/").filter((s) => Boolean(s));
-}
-function analyzePageGroups(pages) {
-  const twoSegmentCount = /* @__PURE__ */ new Map();
-  const segmentHasNested = /* @__PURE__ */ new Map();
-  for (const page of pages) {
-    const prefix = getGroupPrefix(page.pathname, 2);
-    twoSegmentCount.set(prefix, (twoSegmentCount.get(prefix) || 0) + 1);
-    const segments = getPathSegments(page.pathname);
-    const firstSegment = segments[0] || "";
-    if (!segmentHasNested.has(firstSegment))
-      segmentHasNested.set(firstSegment, false);
-    if (segments.length > 1)
-      segmentHasNested.set(firstSegment, true);
-  }
-  return { twoSegmentCount, segmentHasNested };
-}
-function getPageGroupKey(pathname, { twoSegmentCount, segmentHasNested }) {
-  const segments = getPathSegments(pathname);
-  const firstSegment = segments[0] || "";
-  const twoSegPrefix = getGroupPrefix(pathname, 2);
-  const twoSegCount = twoSegmentCount.get(twoSegPrefix) || 0;
-  let groupKey = twoSegCount > 1 ? twoSegPrefix : `/${firstSegment}`;
-  if (segments.length <= 1 && !segmentHasNested.get(firstSegment))
-    groupKey = "";
-  return groupKey;
-}
-function sortPagesByPath(pages) {
-  const analysis = analyzePageGroups(pages);
-  return pages.sort((a, b) => {
-    const segmentsA = getPathSegments(a.pathname);
-    const segmentsB = getPathSegments(b.pathname);
-    const groupKeyA = getPageGroupKey(a.pathname, analysis);
-    const groupKeyB = getPageGroupKey(b.pathname, analysis);
-    if (groupKeyA === "" && groupKeyB !== "")
-      return -1;
-    if (groupKeyA !== "" && groupKeyB === "")
-      return 1;
-    if (groupKeyA !== groupKeyB)
-      return groupKeyA.localeCompare(groupKeyB);
-    if (segmentsA.length === 0)
-      return -1;
-    if (segmentsB.length === 0)
-      return 1;
-    const minLen = Math.min(segmentsA.length, segmentsB.length);
-    for (let i = 0; i < minLen; i++) {
-      const cmp = segmentsA[i].localeCompare(segmentsB[i]);
-      if (cmp !== 0)
-        return cmp;
-    }
-    return segmentsA.length - segmentsB.length;
-  });
-}
-function formatPagesWithGroups(pages) {
-  if (pages.length === 0)
-    return [];
-  const analysis = analyzePageGroups(pages);
-  const lines = [];
-  let currentGroup = "";
-  let segmentGroupIndex = 0;
-  let urlsInCurrentGroup = 0;
-  for (const page of pages) {
-    const groupKey = getPageGroupKey(page.pathname, analysis);
-    if (groupKey !== currentGroup) {
-      if (urlsInCurrentGroup > 0) {
-        const shouldAddBlankLine = segmentGroupIndex === 0 || segmentGroupIndex >= 1 && segmentGroupIndex <= 2 && urlsInCurrentGroup > 1;
-        if (shouldAddBlankLine)
-          lines.push("");
-      }
-      currentGroup = groupKey;
-      segmentGroupIndex++;
-      urlsInCurrentGroup = 0;
-    }
-    urlsInCurrentGroup++;
-    lines.push(formatLlmsTxtPageLink(page));
-  }
-  return lines;
-}
-export async function buildLlmsTxt(event) {
+export async function buildLlmsTxtFiles(event) {
   const runtimeConfig = useRuntimeConfig(event);
   const aiReadyConfig = runtimeConfig["nuxt-ai-ready"];
-  const sitemapConfig = runtimeConfig.sitemap;
   const siteConfig = getSiteConfig(event);
-  const llmsTxtConfig = aiReadyConfig.llmsTxt;
-  const i18n = aiReadyConfig.i18n;
-  const i18nRuntime = i18n ? { _tag: "enabled", config: i18n, ...await import("./llms-txt-i18n.js") } : { _tag: "disabled" };
-  const i18nContext = { host: getRequestHost(event, { xForwardedHost: true }) };
+  const config = aiReadyConfig.llmsTxt;
   const baseURL = runtimeConfig.app.baseURL;
-  const resolvePath = (path) => withSiteTrailingSlash(event, toDeployedRoute(path, baseURL));
   const resolveUrl = (path) => withSiteUrl(event, toDeployedRoute(path, baseURL), { withBase: true });
+  const resolvePageUrl = (path) => resolveUrl(withSiteTrailingSlash(event, path));
   const canonicalSiteUrl = siteConfig.url ? resolveUrl("/") : void 0;
-  const parts = [];
-  parts.push(`# ${siteConfig.name || canonicalSiteUrl}`);
-  if (siteConfig.description) {
-    parts.push(`
-> ${siteConfig.description}`);
-  }
-  if (canonicalSiteUrl) {
-    parts.push(`
-Canonical Origin: ${canonicalSiteUrl}`);
-  }
-  parts.push("");
-  const sections = llmsTxtConfig.sections?.map((section) => ({
-    ...section,
-    links: section.links ? [...section.links] : void 0
-  })) ?? [];
-  if (sections[0]?.links) {
-    const sitemapRoutes = sitemapConfig?.sitemaps ? Object.values(sitemapConfig.sitemaps).map((s) => s.sitemapName) : ["sitemap.xml"];
-    for (const name of sitemapRoutes) {
-      sections[0].links.push({
-        title: name,
-        href: resolveUrl(`/${name}`),
-        description: "XML sitemap for search engines and crawlers."
-      });
+  const i18n = aiReadyConfig.i18n;
+  const i18nRuntime = i18n ? { _tag: "enabled", config: i18n, ...await import("./llms-txt-i18n.js") } : { _tag: "disabled" };
+  const i18nContext = { host: getRequestHost(event, { xForwardedHost: true }) };
+  const parts = [`# ${(siteConfig.name || canonicalSiteUrl || "Site").replace(/\s+/g, " ").trim()}`];
+  if (siteConfig.description)
+    parts.push("", `> ${siteConfig.description.replace(/\s+/g, " ").trim()}`);
+  if (canonicalSiteUrl)
+    parts.push("", `Canonical Origin: ${canonicalSiteUrl}`);
+  const sections = config.sections?.map((section) => ({ ...section, links: section.links ? [...section.links] : void 0 })) ?? [];
+  const resources = sections.find((section) => section.title === "LLM Resources" && !section.sitemap);
+  if (resources) {
+    resources.links ||= [];
+    for (const sitemap of getSitemapsFromConfig(event).filter((sitemap2) => config.sitemaps === void 0 || config.sitemaps.includes(sitemap2.name))) {
+      resources.links.push({ title: sitemap.route.split("/").pop() || `${sitemap.name}.xml`, href: resolveUrl(toLogicalRoute(sitemap.route, baseURL)), description: "XML sitemap for search engines and crawlers." });
     }
-    sections[0].links.push({
-      title: "robots.txt",
-      href: resolveUrl("/robots.txt"),
-      description: "Crawler rules and permissions."
-    });
+    if (!getSitemapsFromConfig(event).length && (config.sitemaps === void 0 || config.sitemaps.includes("sitemap")))
+      resources.links.push({ title: "sitemap.xml", href: resolveUrl("/sitemap.xml"), description: "XML sitemap for search engines and crawlers." });
+    resources.links.push({ title: "robots.txt", href: resolveUrl("/robots.txt"), description: "Crawler rules and permissions." });
   }
-  const normalizedContent = normalizeLlmsTxtConfig({ ...llmsTxtConfig, sections });
-  if (normalizedContent) {
-    parts.push(normalizedContent);
-    parts.push("");
+  if (config.pages === false && !sections.some((section) => section.sitemap))
+    return Object.fromEntries(createLlmsTxtFiles({ header: parts.join("\n"), config, sections, pages: [], sitemapPages: /* @__PURE__ */ new Map(), resolveUrl }));
+  const groups = await fetchSitemapUrlGroups(event, config.sitemaps);
+  const allowedOrigin = canonicalSiteUrl ? new URL(canonicalSiteUrl).origin : void 0;
+  const requestOrigin = getRequestURL(event, { xForwardedHost: true, xForwardedProto: true }).origin;
+  const sitemapPaths = /* @__PURE__ */ new Map();
+  for (const [name, urls] of groups) {
+    const paths = /* @__PURE__ */ new Set();
+    for (const entry of urls) {
+      const url = URL.canParse(entry.loc, canonicalSiteUrl || "http://localhost") ? new URL(entry.loc, canonicalSiteUrl || "http://localhost") : void 0;
+      if (!url || url.search || url.hash || allowedOrigin && url.origin !== allowedOrigin && url.origin !== requestOrigin)
+        continue;
+      const path = toLogicalRoute(url.pathname, baseURL);
+      paths.add(path);
+    }
+    sitemapPaths.set(name, paths);
   }
-  const urlsPromise = fetchSitemapUrls(event);
-  let pages = [];
-  let errorPages = [];
-  try {
-    ;
-    [pages, errorPages] = await Promise.all([
-      queryPages(event),
-      queryPages(event, { where: { hasError: true } })
-    ]);
-  } catch (err) {
-    logger.warn(
-      `[ai-ready] Database unavailable for llms.txt, falling back to sitemap-only pages: ${err instanceof Error ? err.message : String(err)}`
-    );
+  const admitted = new Set([...sitemapPaths.values()].flatMap((paths) => [...paths]));
+  let stored = [];
+  if (aiReadyConfig.database?._tag !== "Disabled") {
+    const read = config.pages === "all" ? Promise.all([queryPages(event), queryPages(event, { where: { hasError: true } })]).then((results) => results.flat()) : queryPageMetadata(event, [...admitted].flatMap((path) => [path, toDeployedRoute(path, baseURL)]));
+    stored = await read.catch((error) => {
+      logger.warn(`[ai-ready] Database unavailable for llms.txt. Using sitemap metadata: ${error instanceof Error ? error.message : String(error)}`);
+      return [];
+    });
   }
-  const urls = await urlsPromise;
-  const sitemapPaths = urls.map((url) => toLogicalRoute(url.loc, baseURL));
-  const sitemapPathSet = new Set(sitemapPaths);
-  const seenPaths = /* @__PURE__ */ new Set();
-  const errorSet = new Set(errorPages.map((p) => normalizePersistedRoute(p.route, sitemapPathSet, baseURL)));
-  const prerendered = [];
-  for (const page of pages) {
-    const pathname = normalizePersistedRoute(page.route, sitemapPathSet, baseURL);
-    if (seenPaths.has(pathname))
+  const errorPaths = new Set(stored.filter((page) => page.isError).map((page) => normalizePersistedRoute(page.route, admitted, baseURL)));
+  const metadata = new Map(stored.map((page) => [normalizePersistedRoute(page.route, admitted, baseURL), page]));
+  const availability = config.markdownLinks ? await findMarkdownLinkAvailability() : void 0;
+  const href = (path) => {
+    const markdownPath = toMarkdownPath(path);
+    if (availability && (availability.paths.has(decodePath(toDeployedRoute(markdownPath, baseURL))) || availability.runtimeMarkdownAvailable && hasMarkdownTwin(path)))
+      return resolveUrl(markdownPath);
+    return resolvePageUrl(path);
+  };
+  const pageMap = /* @__PURE__ */ new Map();
+  const candidates = config.pages === "all" ? /* @__PURE__ */ new Set([...admitted, ...metadata.keys()]) : admitted;
+  for (const pathname of [...candidates].sort()) {
+    if (errorPaths.has(pathname))
       continue;
-    seenPaths.add(pathname);
-    prerendered.push({ pathname, title: page.title, description: page.description, locale: page.locale });
+    const page = metadata.get(pathname);
+    const locale = page?.locale || (i18nRuntime._tag === "enabled" ? i18nRuntime.resolveLocaleFromRoute(pathname, i18nRuntime.config, i18nContext).locale : void 0);
+    pageMap.set(pathname, { pathname, href: href(pathname), title: page?.title, description: page?.description, locale });
   }
-  const devModeHint = import.meta.dev && prerendered.length === 0 ? " (dev mode - run `nuxi generate` for page titles)" : "";
-  const other = [];
-  for (const pathname of sitemapPaths) {
-    if (!seenPaths.has(pathname) && !errorSet.has(pathname)) {
-      const locale = i18nRuntime._tag === "enabled" ? i18nRuntime.resolveLocaleFromRoute(pathname, i18nRuntime.config, i18nContext).locale : void 0;
-      other.push({ pathname, locale });
-      seenPaths.add(pathname);
-    }
-  }
-  let resolvePageHref = resolvePath;
-  if (llmsTxtConfig.markdownLinks) {
-    const markdownLinkAvailability = await findMarkdownLinkAvailability();
-    resolvePageHref = (pathname) => {
-      const deployedPathname = resolvePath(pathname);
-      const markdownPath = resolvePath(toMarkdownPath(pathname));
-      if (markdownLinkAvailability.paths.has(decodePath(markdownPath)) || markdownLinkAvailability.runtimeMarkdownAvailable && hasMarkdownTwin(pathname)) {
-        return markdownPath;
-      }
-      return deployedPathname;
-    };
-  }
-  const resolvePagePath = resolvePageHref;
-  resolvePageHref = (pathname) => absoluteSiteUrl(resolvePagePath(pathname), canonicalSiteUrl);
-  for (const page of [...prerendered, ...other])
-    page.href = resolvePageHref(page.pathname);
+  let rootContent;
   if (i18nRuntime._tag === "enabled") {
-    const pageCounts = /* @__PURE__ */ new Map();
-    for (const locale of i18nRuntime.config.locales) pageCounts.set(locale.code, 0);
-    for (const p of [...prerendered, ...other]) {
-      const code = p.locale || i18nRuntime.resolveLocaleFromRoute(p.pathname, i18nRuntime.config, i18nContext).locale;
-      pageCounts.set(code, (pageCounts.get(code) ?? 0) + 1);
+    const counts = /* @__PURE__ */ new Map();
+    for (const page of pageMap.values())
+      counts.set(page.locale, (counts.get(page.locale) || 0) + 1);
+    rootContent = i18nRuntime.formatAvailableLanguagesSection(i18nRuntime.config, counts, href, i18nContext).join("\n");
+    for (const [path, page] of pageMap) {
+      if (page.locale !== i18nRuntime.config.defaultLocale)
+        pageMap.delete(path);
     }
-    parts.push(...i18nRuntime.formatAvailableLanguagesSection(i18nRuntime.config, pageCounts, resolvePageHref, i18nContext));
-    parts.push("");
   }
-  const isDefaultLocale = (item) => {
-    if (i18nRuntime._tag === "disabled")
-      return true;
-    const code = item.locale || i18nRuntime.resolveLocaleFromRoute(item.pathname, i18nRuntime.config, i18nContext).locale;
-    return code === i18nRuntime.config.defaultLocale;
-  };
-  const filteredPrerendered = i18nRuntime._tag === "enabled" ? prerendered.filter(isDefaultLocale) : prerendered;
-  const filteredOther = i18nRuntime._tag === "enabled" ? other.filter(isDefaultLocale) : other;
-  const sortedPrerendered = sortPagesByPath(filteredPrerendered);
-  const sortedOther = sortPagesByPath(filteredOther);
-  if (sortedPrerendered.length > 0 && sortedOther.length > 0) {
-    parts.push(`## Prerendered Pages${devModeHint}
-`);
-    parts.push(...formatPagesWithGroups(sortedPrerendered));
-    parts.push("");
-    parts.push("## Other Pages\n");
-    parts.push(...formatPagesWithGroups(sortedOther));
-    parts.push("");
-  } else if (sortedPrerendered.length > 0) {
-    parts.push(`## Pages${devModeHint}
-`);
-    parts.push(...formatPagesWithGroups(sortedPrerendered));
-    parts.push("");
-  } else if (sortedOther.length > 0) {
-    parts.push(`## Pages${devModeHint}
-`);
-    parts.push(...formatPagesWithGroups(sortedOther));
-    parts.push("");
-  }
-  return parts.join("\n");
+  const sitemapPages = new Map([...sitemapPaths].map(([name, paths]) => [name, [...paths].flatMap((path) => pageMap.has(path) ? [pageMap.get(path)] : [])]));
+  return Object.fromEntries(createLlmsTxtFiles({ header: parts.join("\n"), config, sections, rootContent, pages: config.pages === false ? [] : [...pageMap.values()], sitemapPages, resolveUrl }));
+}
+export async function buildLlmsTxt(event) {
+  const files = await buildLlmsTxtFiles(event);
+  const path = toLogicalRoute(event.path.split("?")[0], useRuntimeConfig(event).app.baseURL);
+  const content = files[path];
+  if (content === void 0)
+    throw createError({ statusCode: 404, statusMessage: "llms.txt file not found" });
+  return content;
 }

package/dist/runtime/server/db/queries.d.ts

@@ -34,6 +34,9 @@ export interface PageEntry {
 export interface PageData extends PageEntry {
     markdown: string;
 }
+export type PageMetadata = Pick<PageEntry, 'route' | 'title' | 'description' | 'isError' | 'locale'>;
+/** Read only admitted page metadata. Each query fits D1's 100 parameter limit. */
+export declare function queryPageMetadata(event: AiReadyDatabaseEvent | undefined, routes: readonly string[]): Promise<PageMetadata[]>;
 export interface SearchResult {
     route: string;
     title: string;

package/dist/runtime/server/db/queries.js

@@ -80,6 +80,7 @@ async function getPrerenderDb() {
       const isErrorQuery = sql.includes("is_error = 1") || params.includes(1) && sql.includes("is_error");
       const excludeErrors = sql.includes("is_error = 0");
       const includeMarkdown = wantsMarkdown(sql);
+      const admittedRoutes = sql.includes("WHERE route IN (") ? new Set(params) : void 0;
       if (isErrorQuery) {
         return [...errorRoutes].map((route) => ({
           route,
@@ -94,8 +95,8 @@ async function getPrerenderDb() {
           locale: ""
         }));
       }
-      const filtered = excludeErrors ? pages.filter((p) => !errorRoutes.has(p.route)) : pages;
-      return filtered.map((p) => ({
+      const filtered = pages.filter((p) => (!excludeErrors || !errorRoutes.has(p.route)) && (!admittedRoutes || admittedRoutes.has(p.route)));
+      const rows = filtered.map((p) => ({
         route: p.route,
         title: p.title,
         description: p.description,
@@ -107,6 +108,13 @@ async function getPrerenderDb() {
         indexed: 1,
         locale: p.locale || ""
       }));
+      if (admittedRoutes && !excludeErrors) {
+        for (const route of errorRoutes) {
+          if (admittedRoutes.has(route))
+            rows.push({ route, title: "", description: "", headings: [], keywords: "[]", updated_at: "", is_error: 1, indexed: 0, locale: "" });
+        }
+      }
+      return rows;
     },
     first: async (sql, params = []) => {
       if (sql.includes("WHERE route = ?")) {
@@ -136,6 +144,24 @@ async function getPrerenderDb() {
     }
   };
 }
+export async function queryPageMetadata(event, routes) {
+  if (!routes.length)
+    return [];
+  const db = await getDb(event);
+  if (!db)
+    return [];
+  const unique = [...new Set(routes)];
+  const pages = [];
+  for (let i = 0; i < unique.length; i += 100) {
+    const batch = unique.slice(i, i + 100);
+    const rows = await db.all(
+      `SELECT route, title, description, is_error, locale FROM ai_ready_pages WHERE route IN (${batch.map(() => "?").join(", ")})`,
+      batch
+    );
+    pages.push(...rows.map((row) => ({ route: row.route, title: row.title, description: row.description, isError: row.is_error === 1, locale: row.locale || "" })));
+  }
+  return pages;
+}
 function buildWhereClause(where) {
   const conditions = [];
   const params = [];

package/dist/runtime/server/middleware/markdown.js

@@ -20,7 +20,7 @@ function notFoundMarkdown(ctx, canonicalUrl, build) {
     ``,
     `- [Sitemap](${resolveUrl("/sitemap.xml")})`,
     `- [llms.txt](${resolveUrl("/llms.txt")})`,
-    `- [llms-full.txt](${resolveUrl("/llms-full.txt")})`,
+    ...config.llmsFullTxt === false ? [] : [`- [llms-full.txt](${resolveUrl("/llms-full.txt")})`],
     ``
   ].join("\n");
   const i18n = config.i18n;

package/dist/runtime/server/routes/llms-full.txt.get.js

@@ -1,4 +1,4 @@
-import { eventHandler, sendIterable, setHeader, setResponseHeader } from "#nuxtseo/h3";
+import { assertMethod, eventHandler, sendIterable, setHeader, setResponseHeader } from "#nuxtseo/h3";
 import { useRuntimeConfig } from "#nuxtseo/nitro";
 import { getSiteConfig } from "#site-config/server/composables";
 import { withSiteUrl } from "#site-config/server/composables/utils";
@@ -7,6 +7,7 @@ import { toDeployedRoute } from "../../route-path.js";
 import { countPages, streamPages } from "../db/queries.js";
 import { buildLlmsFullTxtHeader, formatPageForLlmsFullTxt } from "../utils/llms-full.js";
 export default eventHandler(async (event) => {
+  assertMethod(event, "GET", true);
   if (import.meta.prerender) {
     setHeader(event, "Content-Type", "text/plain; charset=utf-8");
     return "# llms-full.txt\n\nThis file is generated during prerender.";

package/dist/runtime/server/routes/llms.txt.get.js

@@ -1,29 +1,37 @@
-import { eventHandler, setHeader } from "#nuxtseo/h3";
+import { assertMethod, createError, eventHandler, getRequestURL, setHeader } from "#nuxtseo/h3";
 import { defineCachedFunction, useRuntimeConfig } from "#nuxtseo/nitro";
 import { publicCacheControl } from "../../cache-control.js";
-import { buildLlmsTxt } from "../../llms-txt-utils.js";
+import { buildLlmsTxtFiles } from "../../llms-txt-utils.js";
+import { toLogicalRoute } from "../../route-path.js";
 const cachedBuilders = /* @__PURE__ */ new Map();
 function getBuildLlmsTxtCached(maxAge) {
   let cached = cachedBuilders.get(maxAge);
   if (!cached) {
-    cached = defineCachedFunction(buildLlmsTxt, {
-      name: "llms-txt",
+    cached = defineCachedFunction(buildLlmsTxtFiles, {
+      name: "llms-txt-files",
       group: "ai-ready",
       maxAge,
-      swr: true
+      swr: true,
+      // Share one discovery snapshot between root and child files, separated by origin.
+      getKey: (event) => getRequestURL(event, { xForwardedHost: true, xForwardedProto: true }).origin
     });
     cachedBuilders.set(maxAge, cached);
   }
   return cached;
 }
 export default eventHandler(async (event) => {
-  const runtimeConfig = useRuntimeConfig(event)["nuxt-ai-ready"];
-  const cacheSeconds = runtimeConfig.llmsTxtCacheSeconds;
-  const cacheEnabled = !import.meta.dev && cacheSeconds > 0;
-  const content = cacheEnabled ? await getBuildLlmsTxtCached(cacheSeconds)(event) : await buildLlmsTxt(event);
+  assertMethod(event, "GET", true);
+  const runtimeConfig = useRuntimeConfig(event);
+  const config = runtimeConfig["nuxt-ai-ready"];
+  const cacheSeconds = config.llmsTxtCacheSeconds;
+  const cacheEnabled = !import.meta.dev && !import.meta.prerender && cacheSeconds > 0;
+  const files = cacheEnabled ? await getBuildLlmsTxtCached(cacheSeconds)(event) : await buildLlmsTxtFiles(event);
+  const path = toLogicalRoute(event.path.split("?")[0], runtimeConfig.app.baseURL);
+  const content = files[path];
+  if (content === void 0)
+    throw createError({ statusCode: 404, statusMessage: "llms.txt file not found" });
   setHeader(event, "Content-Type", "text/plain; charset=utf-8");
-  if (cacheEnabled) {
+  if (cacheEnabled)
     setHeader(event, "Cache-Control", publicCacheControl(cacheSeconds, 3600));
-  }
   return content;
 });

package/dist/runtime/server/routes/sitemap.md.get.js

@@ -1,4 +1,4 @@
-import { eventHandler, setHeader } from "#nuxtseo/h3";
+import { assertMethod, eventHandler, setHeader } from "#nuxtseo/h3";
 import { defineCachedFunction, useRuntimeConfig } from "#nuxtseo/nitro";
 import { getSiteConfig } from "#site-config/server/composables";
 import { publicCacheControl } from "../../cache-control.js";
@@ -42,6 +42,7 @@ function getBuildSitemapMdCached(maxAge) {
   return cached;
 }
 export default eventHandler(async (event) => {
+  assertMethod(event, "GET", true);
   const fullRuntimeConfig = useRuntimeConfig(event);
   const runtimeConfig = fullRuntimeConfig["nuxt-ai-ready"];
   const cacheSeconds = runtimeConfig.llmsTxtCacheSeconds ?? 600;

package/dist/runtime/server/utils/markdown-request.js

@@ -1,7 +1,7 @@
 import { negotiateContent, parseAcceptHeader } from "@mdream/js/negotiate";
 import { getBotInfo } from "@nuxtjs/robots/util";
 import { getHeaders } from "#nuxtseo/h3";
-import { isReservedPath } from "../../markdown-path.js";
+import { hasMarkdownTwin, isReservedPath } from "../../markdown-path.js";
 const RE_MD_EXT = /\.md$/;
 export function toMarkdownRequest(event, isPrerender = !!import.meta.prerender) {
   return { path: event.path, headers: getHeaders(event), isPrerender };
@@ -38,10 +38,7 @@ export function getRequestRenderInfo(request, mode = { _tag: "runtime", contentN
   const isExplicit = originalPath.endsWith(".md");
   if (isPrerender && !isExplicit)
     return null;
-  const lastSegment = originalPath.split("/").pop() || "";
-  const hasExtension = lastSegment.includes(".");
-  const extension = hasExtension ? lastSegment.substring(lastSegment.lastIndexOf(".")) : "";
-  if (hasExtension && extension !== ".md")
+  if (!isExplicit && !hasMarkdownTwin(originalPath))
     return null;
   const negotiation = isPrerender ? "markdown" : mode.contentNegotiation ? negotiateRequestRepresentation(request, mode.botNegotiation) : "html";
   if (isExplicit) {

package/dist/runtime/server/utils/sitemap.d.ts

@@ -53,5 +53,6 @@ export declare function fetchSitemapByRoute(event: H3Event | undefined, route: s
  * Fetch all URLs from all sitemaps (or single sitemap if not multi-sitemap)
  * Used for backwards compatibility and llms.txt generation
  */
+export declare function fetchSitemapUrlGroups(event: H3Event, names?: readonly string[]): Promise<Map<string, SitemapUrl[]>>;
 export declare function fetchSitemapUrls(event: H3Event): Promise<SitemapUrl[]>;
 export {};

package/dist/runtime/server/utils/sitemap.js

@@ -1,5 +1,7 @@
 import { createSitemapReader } from "sitemapd";
 import { parseSitemap } from "sitemapd/parse";
+import staticSitemapConfig from "#ai-ready-virtual/sitemap-config.mjs";
+import { getRequestURL } from "#nuxtseo/h3";
 import { useRuntimeConfig } from "#nuxtseo/nitro";
 import { logger } from "../logger.js";
 import { getCfEnv, hasAssets } from "./cloudflare.js";
@@ -19,17 +21,18 @@ const SITEMAP_READER_LIMITS = {
 const MAX_SITEMAP_CRAWL_ROUNDS = 10;
 export function getSitemapsFromConfig(event) {
   const runtimeConfig = useRuntimeConfig(event);
-  const sitemapConfig = runtimeConfig.sitemap;
+  const sitemapConfig = { ...staticSitemapConfig, ...runtimeConfig.sitemap };
   if (!sitemapConfig?.sitemaps)
     return [];
   const sitemaps = [];
   for (const [key, sitemap] of Object.entries(sitemapConfig.sitemaps)) {
     if (key === "index")
       continue;
-    if (sitemap._route) {
+    const route = sitemap._route || sitemap.route || (key === "chunks" ? sitemapConfig.sitemaps.index?._route || sitemapConfig.sitemaps.index?.route : void 0);
+    if (route) {
       sitemaps.push({
-        name: sitemap.sitemapName || key,
-        route: sitemap._route
+        name: (sitemap.sitemapName || key).replace(/\.xml$/, ""),
+        route: route.startsWith("/") ? route : `/${route}`
       });
     }
   }
@@ -305,6 +308,7 @@ export async function crawlSitemapByRoute(event, input) {
   const usePublicAsset = config.sitemapPrerendered && hasAssets(event);
   const configuredSiteUrl = createUniversalContext(event).siteUrl ?? SITEMAP_FALLBACK_ORIGIN;
   const siteUrl = parseAbsoluteUrl(configuredSiteUrl);
+  const requestOrigin = import.meta.dev && event ? getRequestURL(event, { xForwardedHost: true, xForwardedProto: true }).origin : void 0;
   if (siteUrl._tag === "error") {
     return {
       _tag: "failed",
@@ -346,7 +350,7 @@ export async function crawlSitemapByRoute(event, input) {
       const target = parseAbsoluteUrl(request.url);
       if (target._tag === "error")
         return { _tag: "deny", reason: `Invalid sitemap target: ${request.url}` };
-      if (target.value.origin !== siteUrl.value.origin) {
+      if (target.value.origin !== siteUrl.value.origin && target.value.origin !== requestOrigin) {
         return {
           _tag: "deny",
           reason: `Sitemap target origin ${target.value.origin} differs from ${siteUrl.value.origin}`
@@ -433,17 +437,21 @@ export async function fetchSitemapByRoute(event, route) {
   const result = await crawlSitemapByRoute(event, { route });
   return result._tag === "complete" ? { urls: result.urls } : { urls: result.urls, error: result.error };
 }
-export async function fetchSitemapUrls(event) {
+export async function fetchSitemapUrlGroups(event, names) {
   const sitemaps = getSitemapsFromConfig(event);
+  const groups = /* @__PURE__ */ new Map();
   if (sitemaps.length > 0) {
     logger.debug(`[sitemap] Multi-sitemap mode: ${sitemaps.length} sitemaps`);
-    const allUrls = [];
     for (const sitemap of sitemaps) {
+      if (names !== void 0 && !names.includes(sitemap.name))
+        continue;
       const { urls: urls2 } = await fetchSitemapByRoute(event, sitemap.route);
-      allUrls.push(...urls2);
+      groups.set(sitemap.name, [...groups.get(sitemap.name) || [], ...urls2]);
     }
-    return allUrls;
+    return groups;
   }
+  if (names !== void 0 && !names.includes("sitemap"))
+    return groups;
   logger.debug(`[sitemap] Single sitemap mode: /sitemap.xml`);
   const { urls, error } = await fetchSitemapByRoute(event, "/sitemap.xml");
   if (error) {
@@ -451,5 +459,9 @@ export async function fetchSitemapUrls(event) {
   } else if (urls.length === 0) {
     logger.warn("Sitemap is empty - add routes to sitemap or configure sitemap.sources in nuxt.config");
   }
-  return urls;
+  groups.set("sitemap", urls);
+  return groups;
+}
+export async function fetchSitemapUrls(event) {
+  return [...(await fetchSitemapUrlGroups(event)).values()].flat();
 }

package/dist/runtime/types.d.ts

@@ -185,7 +185,9 @@ export interface ModuleOptions {
     /**
      * Structured llms.txt configuration
      */
-    llmsTxt?: LlmsTxtConfig;
+    llmsTxt?: LlmsTxtConfig | false;
+    /** Generate and serve llms-full.txt. Set false to omit the export and its discovery link. */
+    llmsFullTxt?: boolean;
     /**
      * Generate a Markdown sitemap at `/sitemap.md` from indexed pages and end
      * every Markdown page with a `## Sitemap` section linking to it.
@@ -512,6 +514,10 @@ export interface LlmsTxtLink {
  * Section in llms.txt
  */
 export interface LlmsTxtSection {
+    /** Generate a separate file from these sitemap names. Chunked sitemaps retain their configured name. */
+    sitemap?: string | string[];
+    /** Generated file path. Defaults to /llms/<sitemap>.txt. */
+    path?: string;
     /** The title of the section */
     title: string;
     /** Description rendered in the heading-free preamble, or in each optional link note (can be array for multiple paragraphs) */
@@ -525,6 +531,12 @@ export interface LlmsTxtSection {
  * Structured llms.txt configuration
  */
 export interface LlmsTxtConfig {
+    /** Page admission. Sitemap URLs are authoritative by default. Use all to include stored pages outside the sitemap. */
+    pages?: 'sitemap' | 'all' | false;
+    /** Only read these sitemap names. An empty array disables sitemap page discovery. */
+    sitemaps?: string[];
+    /** Maximum characters per generated file. Large lists become linked files. Minimum 1024. Default 80000. */
+    maxLength?: number;
     /**
      * Rewrite automatically generated page links to their Markdown representation
      * when the final static output contains the file or the deployment retains