nuxt-ai-ready 1.5.6 → 1.5.8

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "1.5.6",
+  "version": "1.5.8",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -1,7 +1,7 @@
 import { createHash, randomBytes } from 'node:crypto';
-import { mkdir, writeFile, appendFile, stat, access, readFile } from 'node:fs/promises';
-import { join, dirname } from 'node:path';
-import { useLogger, useNuxt, addTypeTemplate, addTemplate, defineNuxtModule, createResolver, hasNuxtModule, addServerPlugin, addServerHandler, addPlugin } from '@nuxt/kit';
+import { mkdir, writeFile, appendFile, stat, readdir, access, readFile } from 'node:fs/promises';
+import { join, dirname, relative, resolve } from 'node:path';
+import { useLogger, useNuxt, resolveFiles, addTypeTemplate, addTemplate, defineNuxtModule, createResolver, hasNuxtModule, addServerPlugin, addServerHandler, addPlugin } from '@nuxt/kit';
 import defu from 'defu';
 import { installNuxtSiteConfig, useSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
 import { setupDevToolsUI } from 'nuxtseo-shared/devtools';
@@ -9,7 +9,7 @@ import { resolveNuxtContentVersion } from 'nuxtseo-shared/kit';
 import { readPackageJSON, resolvePackageJSON } from 'pkg-types';
 import { parseSitemapXml } from '@nuxtjs/sitemap/utils';
 import { colorize } from 'consola/utils';
-import { withLeadingSlash, withBase } from 'ufo';
+import { withLeadingSlash, withBase, joinURL } from 'ufo';
 import { normalizePagePath, toMarkdownPath } from '../dist/runtime/markdown-path.js';
 import { toLogicalRoute, toDeployedRoute } from '../dist/runtime/route-path.js';
 import { initSchema, computeContentHash, insertPage, queryAllPages, exportDbDump } from '../dist/runtime/server/db/shared.js';
@@ -25,6 +25,41 @@ const PRERENDER_PAGE_TIMEOUT = 3e4;
 const RE_HTML_MD_EXT = /\.(html|md)$/;
 const RE_INDEX_SUFFIX = /\/index$/;
 const RE_MD_EXT = /\.md$/;
+const RE_NEGATED_GLOB = /^(!?)(.*)$/;
+const RE_PARENT_DIR_GLOB = /!?\.\.\//;
+const MARKDOWN_LINK_AVAILABILITY_FILE = "markdown-link-availability.json";
+async function findOutputMarkdownPaths(publicDir, baseURL, srcDir, ignoredPaths, publicAssets) {
+  const paths = /* @__PURE__ */ new Set();
+  async function visit(directory, urlBase, relativeDirectory = "") {
+    const entries = await readdir(directory, { withFileTypes: true });
+    await Promise.all(entries.map(async (entry) => {
+      const relativePath = join(relativeDirectory, entry.name);
+      if (entry.isDirectory()) {
+        await visit(join(directory, entry.name), urlBase, relativePath);
+      } else if (entry.isFile() && entry.name.endsWith(".md")) {
+        const urlPath = joinURL(urlBase, relativePath.replaceAll("\\", "/"));
+        paths.add(joinURL(baseURL, urlPath));
+      }
+    }));
+  }
+  await visit(publicDir, "/");
+  for (const asset of publicAssets) {
+    const patterns = [
+      "**/*.md",
+      ...ignoredPaths.map((ignoredPath) => {
+        const [, negated = "", pattern = ""] = ignoredPath.match(RE_NEGATED_GLOB) || [];
+        const assetPattern = pattern.startsWith("*") ? pattern : relative(asset.dir, resolve(srcDir, pattern)).replaceAll("\\", "/");
+        return `${negated ? "" : "!"}${assetPattern}`;
+      }).filter((pattern) => !RE_PARENT_DIR_GLOB.test(pattern))
+    ];
+    const files = await resolveFiles(asset.dir, patterns);
+    for (const file of files) {
+      const urlPath = joinURL(asset.baseURL || "/", relative(asset.dir, file).replaceAll("\\", "/"));
+      paths.add(joinURL(baseURL, urlPath));
+    }
+  }
+  return [...paths].sort();
+}
 async function fetchPreviousMeta(siteUrl, indexNow) {
   const metaUrl = withBase("/__ai-ready/pages.meta.json", siteUrl);
   logger.info(`Fetching previous build meta from ${metaUrl}`);
@@ -399,6 +434,20 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, indexNo
           await submitIndexNow(changed, added, state.siteInfo.url, state.indexNow);
         }
       }
+      if (state.dbPath && state.llmsTxtConfig?.markdownLinks) {
+        const availabilityFile = join(dirname(state.dbPath), MARKDOWN_LINK_AVAILABILITY_FILE);
+        const paths = await findOutputMarkdownPaths(
+          nitro.options.output.publicDir,
+          nitro.options.baseURL,
+          nitro.options.srcDir,
+          nitro.options.ignore,
+          nitro.options.publicAssets
+        );
+        await writeFile(availabilityFile, JSON.stringify({
+          runtimeMarkdownAvailable: !nitro.options.static,
+          paths
+        }), "utf-8");
+      }
       const llmsStats = await prerenderRoute(nitro, "/llms.txt");
       const llmsFullStats = await stat(state.llmsFullTxtPath);
       nitro._prerenderedRoutes.push({ route: "/llms-full.txt", fileName: "/llms-full.txt" });
@@ -464,6 +513,10 @@ export {}
   addTemplate({
     filename: "types/ai-ready-virtual.d.ts",
     getContents: () => `declare module '#ai-ready-virtual/read-page-data.mjs' {
+  export function readMarkdownLinkAvailabilityFromFilesystem(): Promise<{
+    runtimeMarkdownAvailable: boolean
+    paths: string[]
+  }>
   export function readPageDataFromFilesystem(): Promise<{
     pages: Array<{
       route: string
@@ -616,7 +669,7 @@ const module$1 = defineNuxtModule({
       version: ">=6.0.0"
     },
     "@nuxtjs/sitemap": {
-      version: ">=8.0.0"
+      version: ">=8.3.0"
     },
     "nuxt-site-config": {
       version: ">=3.2"
@@ -765,12 +818,13 @@ const module$1 = defineNuxtModule({
       }
     }
     const mergedLlmsTxt = config.llmsTxt ? {
+      markdownLinks: config.llmsTxt.markdownLinks ?? false,
       sections: [
         ...defaultLlmsTxtSections,
         ...config.llmsTxt.sections || []
       ],
       notes: config.llmsTxt.notes
-    } : { sections: defaultLlmsTxtSections };
+    } : { markdownLinks: false, sections: defaultLlmsTxtSections };
     const llmsTxtPayload = {
       sections: mergedLlmsTxt.sections || [],
       notes: typeof mergedLlmsTxt.notes === "string" ? [mergedLlmsTxt.notes] : mergedLlmsTxt.notes || []
@@ -859,7 +913,30 @@ const module$1 = defineNuxtModule({
         }
       }
       nitroConfig.virtual = nitroConfig.virtual || {};
-      nitroConfig.virtual["#ai-ready-virtual/read-page-data.mjs"] = nuxt.options.dev ? `export async function readPageDataFromFilesystem() { return { pages: [], errorRoutes: [] } }` : `
+      const markdownLinkAvailabilityPath = join(dirname(buildDbPath), MARKDOWN_LINK_AVAILABILITY_FILE);
+      nitroConfig.virtual["#ai-ready-virtual/read-page-data.mjs"] = nuxt.options.dev ? `
+export async function readPageDataFromFilesystem() { return { pages: [], errorRoutes: [] } }
+export async function readMarkdownLinkAvailabilityFromFilesystem() { return { runtimeMarkdownAvailable: false, paths: [] } }
+` : `
+export async function readMarkdownLinkAvailabilityFromFilesystem() {
+  if (!import.meta.prerender) {
+    return { runtimeMarkdownAvailable: false, paths: [] }
+  }
+
+  const { readFile } = await import('node:fs/promises')
+  try {
+    const availability = JSON.parse(await readFile(${JSON.stringify(markdownLinkAvailabilityPath)}, 'utf8'))
+    return {
+      runtimeMarkdownAvailable: availability?.runtimeMarkdownAvailable === true,
+      paths: Array.isArray(availability?.paths) ? availability.paths.filter(path => typeof path === 'string') : [],
+    }
+  }
+  catch (error) {
+    console.warn('[nuxt-ai-ready] Failed to read Markdown link availability; keeping canonical links.', error)
+    return { runtimeMarkdownAvailable: false, paths: [] }
+  }
+}
+
 export async function readPageDataFromFilesystem() {
   if (!import.meta.prerender) {
     return { pages: [], errorRoutes: [] }

package/dist/runtime/llms-txt-format.d.ts

@@ -1,8 +1,19 @@
 /**
  * Pure formatting functions for llms.txt - no runtime dependencies
  */
+import type { RuntimeI18nConfig } from './server/utils/i18n.js';
 import type { LlmsTxtConfig } from './types.js';
+export declare function formatAvailableLanguagesSection(i18n: RuntimeI18nConfig, pageCounts: Map<string, number>, resolveHref?: (pathname: string) => string): string[];
+interface LlmsTxtPageLink {
+    pathname: string;
+    href?: string;
+    title?: string;
+    description?: string;
+}
+export declare function formatLlmsTxtPageLink(page: LlmsTxtPageLink): string;
 /**
- * Normalize llms.txt structured configuration to markdown string
+ * Normalize llms.txt structured configuration to the llmstxt.org Markdown format.
+ * Preamble prose must appear before the first H2; every H2 body is a file list.
  */
 export declare function normalizeLlmsTxtConfig(config: LlmsTxtConfig): string;
+export {};

package/dist/runtime/llms-txt-format.js

@@ -1,39 +1,100 @@
-function normalizeLink(link) {
-  const parts = [];
-  parts.push(`- [${link.title}](${link.href})`);
-  if (link.description)
-    parts.push(`  ${link.description}`);
-  return parts.join("\n");
+import { localePath } from "./server/utils/i18n.js";
+const RE_INLINE_WHITESPACE = /\s+/g;
+const RE_LINK_TITLE_BRACKET = /[[\]]/g;
+const RE_LINK_HREF_UNSAFE = /[\s()]/g;
+const RE_PREAMBLE_ATX_HEADING = /^( {0,3})(#{1,6})(?=\s)/gm;
+function normalizeInlineText(value) {
+  return value.trim().replace(RE_INLINE_WHITESPACE, " ");
 }
-function normalizeSection(section, headingLevel = 2) {
-  const prefix = "#".repeat(headingLevel);
-  const parts = [];
-  parts.push(`${prefix} ${section.title}`);
-  parts.push("");
-  if (section.description) {
-    const descriptions = Array.isArray(section.description) ? section.description : [section.description];
-    parts.push(...descriptions);
-    parts.push("");
-  }
+function normalizeLinkTitle(value) {
+  return normalizeInlineText(value).replace(RE_LINK_TITLE_BRACKET, (bracket) => bracket === "[" ? "[" : "]");
+}
+function normalizeLinkHref(value) {
+  return value.trim().replace(RE_LINK_HREF_UNSAFE, (character) => {
+    if (character === "(")
+      return "%28";
+    if (character === ")")
+      return "%29";
+    return encodeURIComponent(character);
+  });
+}
+function normalizeDescriptions(value) {
+  return (Array.isArray(value) ? value : value ? [value] : []).filter((description) => description.trim()).map(normalizeInlineText);
+}
+function normalizePreambleBlocks(value) {
+  return (Array.isArray(value) ? value : value ? [value] : []).filter((block) => block.trim()).map((block) => block.trim().replace(RE_PREAMBLE_ATX_HEADING, "$1\\$2"));
+}
+function normalizeLink(link, descriptionPrefixes = []) {
+  const descriptions = [...descriptionPrefixes, link.description].filter((value) => Boolean(value?.trim())).map(normalizeInlineText);
+  const description = descriptions.join("; ");
+  return `- [${normalizeLinkTitle(link.title)}](${normalizeLinkHref(link.href)})${description ? `: ${description}` : ""}`;
+}
+function normalizeSection(section) {
+  const parts = [`## ${normalizeInlineText(section.title)}`];
   if (section.links?.length)
-    parts.push(...section.links.map(normalizeLink));
+    parts.push("", ...section.links.map((link) => normalizeLink(link)));
   return parts.join("\n");
 }
-export function normalizeLlmsTxtConfig(config) {
+function normalizePreamble(config) {
   const parts = [];
-  const required = config.sections?.filter((s) => !s.optional) ?? [];
-  const optional = config.sections?.filter((s) => s.optional) ?? [];
-  if (required.length)
-    parts.push(...required.map((s) => normalizeSection(s)));
-  if (optional.length) {
-    parts.push("## Optional");
-    parts.push(...optional.map((s) => normalizeSection(s, 3)));
-  }
   if (config.notes) {
-    parts.push("## Notes");
-    parts.push("");
-    const notes = Array.isArray(config.notes) ? config.notes : [config.notes];
-    parts.push(...notes);
+    const notes = normalizePreambleBlocks(config.notes);
+    if (notes.length)
+      parts.push(["**Notes:**", ...notes].join("\n\n"));
+  }
+  for (const section of config.sections?.filter((section2) => !section2.optional) ?? []) {
+    if (!section.description)
+      continue;
+    const descriptions = normalizePreambleBlocks(section.description);
+    if (descriptions.length)
+      parts.push([`**${normalizeInlineText(section.title)}:**`, ...descriptions].join("\n\n"));
   }
+  return parts;
+}
+function normalizeOptionalSections(sections) {
+  const links = sections.flatMap(
+    (section) => section.links?.map((link) => normalizeLink(link, [section.title, ...normalizeDescriptions(section.description)])) ?? []
+  );
+  if (!links.length)
+    return void 0;
+  return ["## Optional", "", ...links].join("\n");
+}
+function normalizeRequiredSections(sections) {
+  return sections.filter((section) => section.links?.length).map(normalizeSection);
+}
+export function formatAvailableLanguagesSection(i18n, pageCounts, resolveHref = (pathname) => pathname) {
+  const lines = ["## Available Languages on Website", ""];
+  for (const locale of i18n.locales) {
+    const isDefault = locale.code === i18n.defaultLocale;
+    const prefix = localePath("/", locale.code, i18n);
+    const count = pageCounts.get(locale.code) ?? 0;
+    const display = locale.nativeName ? `${locale.nativeName} (${locale.code})` : locale.name ? `${locale.name} (${locale.code})` : locale.code;
+    const suffix = isDefault ? "content included below" : "visit this language for content";
+    lines.push(normalizeLink({
+      title: display,
+      href: resolveHref(prefix),
+      description: `${count} pages; ${suffix}.`
+    }));
+  }
+  return lines;
+}
+export function formatLlmsTxtPageLink(page) {
+  const description = page.description?.trim().replace(RE_INLINE_WHITESPACE, " ");
+  const normalizedTitle = page.title ? normalizeInlineText(page.title) : "";
+  const title = normalizedTitle && normalizedTitle !== page.pathname ? normalizedTitle : page.pathname;
+  const href = (page.href || page.pathname).trim();
+  const truncatedDescription = description ? `${description.substring(0, 160)}${description.length > 160 ? "..." : ""}` : void 0;
+  return normalizeLink({ title, href, description: truncatedDescription });
+}
+export function normalizeLlmsTxtConfig(config) {
+  const required = config.sections?.filter((section) => !section.optional) ?? [];
+  const optional = config.sections?.filter((section) => section.optional) ?? [];
+  const parts = [
+    ...normalizePreamble(config),
+    ...normalizeRequiredSections(required)
+  ];
+  const optionalSection = normalizeOptionalSections(optional);
+  if (optionalSection)
+    parts.push(optionalSection);
   return parts.join("\n\n");
 }

package/dist/runtime/llms-txt-utils.js

@@ -1,24 +1,32 @@
 import { useRuntimeConfig } from "nitropack/runtime";
+import { decodePath } from "ufo";
 import { getSiteConfig } from "#site-config/server/composables/getSiteConfig";
 import { withSiteTrailingSlash, withSiteUrl } from "#site-config/server/composables/utils";
-import { normalizeLlmsTxtConfig } from "./llms-txt-format.js";
+import { formatAvailableLanguagesSection, formatLlmsTxtPageLink, normalizeLlmsTxtConfig } from "./llms-txt-format.js";
+import { toMarkdownPath } from "./markdown-path.js";
 import { normalizePersistedRoute, toDeployedRoute, toLogicalRoute } from "./route-path.js";
 import { queryPages } from "./server/db/queries.js";
 import { logger } from "./server/logger.js";
-import { localePath, resolveLocaleFromRoute } from "./server/utils/i18n.js";
+import { resolveLocaleFromRoute } from "./server/utils/i18n.js";
 import { fetchSitemapUrls } from "./server/utils/sitemap.js";
 export { normalizeLlmsTxtConfig };
-function formatAvailableLanguagesSection(i18n, pageCounts, resolvePath) {
-  const lines = ["## Available Languages on Website", ""];
-  for (const locale of i18n.locales) {
-    const isDefault = locale.code === i18n.defaultLocale;
-    const prefix = resolvePath(localePath("/", locale.code, i18n));
-    const count = pageCounts.get(locale.code) ?? 0;
-    const display = locale.nativeName ? `${locale.nativeName} (${locale.code})` : locale.name ? `${locale.name} (${locale.code})` : locale.code;
-    const suffix = isDefault ? "Content included below" : "Visit website for content";
-    lines.push(`- ${display} - ${count} pages - ${prefix} - ${suffix}`);
+function hasRuntimeMarkdownHandler(pathname) {
+  const lastSegment = pathname.split("/").pop() || "";
+  return !pathname.startsWith("/api") && !pathname.startsWith("/_") && !pathname.startsWith("/@") && !lastSegment.includes(".");
+}
+async function findMarkdownLinkAvailability() {
+  if (!import.meta.prerender) {
+    return {
+      runtimeMarkdownAvailable: true,
+      paths: /* @__PURE__ */ new Set()
+    };
   }
-  return lines;
+  const { readMarkdownLinkAvailabilityFromFilesystem } = await import("#ai-ready-virtual/read-page-data.mjs");
+  const availability = await readMarkdownLinkAvailabilityFromFilesystem();
+  return {
+    runtimeMarkdownAvailable: availability.runtimeMarkdownAvailable,
+    paths: new Set(availability.paths.map(decodePath))
+  };
 }
 function getGroupPrefix(url, depth) {
   const segments = url.split("/").filter(Boolean);
@@ -103,12 +111,7 @@ function formatPagesWithGroups(pages) {
       urlsInCurrentGroup = 0;
     }
     urlsInCurrentGroup++;
-    const descText = page.description ? `: ${page.description.substring(0, 160)}${page.description.length > 160 ? "..." : ""}` : "";
-    const href = page.href ?? page.pathname;
-    if (page.title && page.title !== page.pathname)
-      lines.push(`- [${page.title}](${href})${descText}`);
-    else
-      lines.push(`- ${href}${descText}`);
+    lines.push(formatLlmsTxtPageLink(page));
   }
   return lines;
 }
@@ -194,8 +197,20 @@ Canonical Origin: ${canonicalSiteUrl}`);
       seenPaths.add(pathname);
     }
   }
+  let resolvePageHref = resolvePath;
+  if (llmsTxtConfig.markdownLinks) {
+    const markdownLinkAvailability = await findMarkdownLinkAvailability();
+    resolvePageHref = (pathname) => {
+      const deployedPathname = resolvePath(pathname);
+      const markdownPath = resolvePath(toMarkdownPath(pathname));
+      if (markdownLinkAvailability.paths.has(decodePath(markdownPath)) || markdownLinkAvailability.runtimeMarkdownAvailable && hasRuntimeMarkdownHandler(pathname)) {
+        return markdownPath;
+      }
+      return deployedPathname;
+    };
+  }
   for (const page of [...prerendered, ...other])
-    page.href = resolvePath(page.pathname);
+    page.href = resolvePageHref(page.pathname);
   if (i18n) {
     const pageCounts = /* @__PURE__ */ new Map();
     for (const locale of i18n.locales) pageCounts.set(locale.code, 0);
@@ -203,7 +218,7 @@ Canonical Origin: ${canonicalSiteUrl}`);
       const code = p.locale || resolveLocaleFromRoute(p.pathname, i18n).locale;
       pageCounts.set(code, (pageCounts.get(code) ?? 0) + 1);
     }
-    parts.push(...formatAvailableLanguagesSection(i18n, pageCounts, resolvePath));
+    parts.push(...formatAvailableLanguagesSection(i18n, pageCounts, resolvePageHref));
     parts.push("");
   }
   const isDefaultLocale = (item) => {

package/dist/runtime/server/utils/cloudflare.d.ts

@@ -17,5 +17,5 @@ export declare function hasAssets(event?: H3Event): boolean;
  * Falls back to $fetch with timeout to avoid self-fetch hangs on CF Workers.
  */
 export declare function fetchPublicAsset<T = unknown>(event: H3Event | undefined, path: string, options?: {
-    responseType?: 'json' | 'text' | 'arrayBuffer';
+    responseType?: 'json' | 'text' | 'arrayBuffer' | 'stream';
 }): Promise<T | null>;

package/dist/runtime/server/utils/cloudflare.js

@@ -19,6 +19,8 @@ export async function fetchPublicAsset(event, path, options) {
         return response.text().catch(() => null);
       if (responseType === "arrayBuffer")
         return response.arrayBuffer().catch(() => null);
+      if (responseType === "stream")
+        return response.body;
     }
     return null;
   }
@@ -27,6 +29,6 @@ export async function fetchPublicAsset(event, path, options) {
   return globalThis.$fetch(path, {
     baseURL: "/",
     signal: controller.signal,
-    responseType: responseType === "arrayBuffer" ? "arrayBuffer" : void 0
+    responseType: responseType === "arrayBuffer" || responseType === "stream" ? responseType : void 0
   }).catch(() => null).finally(() => clearTimeout(timeout));
 }

package/dist/runtime/server/utils/sitemap.js

@@ -1,4 +1,4 @@
-import { isSitemapIndex, parseSitemapIndex, parseSitemapXml } from "@nuxtjs/sitemap/utils";
+import { parseSitemapStream } from "@nuxtjs/sitemap/utils";
 import { useRuntimeConfig } from "nitropack/runtime";
 import { withLeadingSlash } from "ufo";
 import { logger } from "../logger.js";
@@ -26,26 +26,26 @@ export function hasMultipleSitemaps(event) {
   const sitemaps = getSitemapsFromConfig(event);
   return sitemaps.length > 1;
 }
-function normalizeUrls(urls) {
-  return urls.map((entry) => {
-    if (typeof entry === "string")
-      return { loc: entry };
-    const e = entry;
-    return {
-      loc: e.loc,
-      lastmod: e.lastmod instanceof Date ? e.lastmod.toISOString() : e.lastmod
-    };
-  });
+function normalizeUrl(entry) {
+  if (typeof entry === "string")
+    return { loc: entry };
+  return {
+    loc: entry.loc,
+    lastmod: entry.lastmod instanceof Date ? entry.lastmod.toISOString() : entry.lastmod
+  };
+}
+function isSitemapXmlInput(value) {
+  return typeof value === "string" || value instanceof Uint8Array || typeof value === "object" && value !== null && "getReader" in value;
 }
 export async function fetchSitemapByRoute(event, route, depth = 0) {
   const config = useRuntimeConfig(event)["nuxt-ai-ready"];
   const fetchRoute = withLeadingSlash(route);
   const usePublicAsset = config.sitemapPrerendered && hasAssets(event);
   logger.debug(`[sitemap] Fetching ${fetchRoute} via ${usePublicAsset ? "ASSETS.fetch" : event ? "event.$fetch" : "globalThis.$fetch"}`);
-  let sitemapXml = null;
+  let sitemapInput = null;
   if (usePublicAsset) {
-    sitemapXml = await fetchPublicAsset(event, fetchRoute, { responseType: "text" });
-    if (!sitemapXml) {
+    sitemapInput = await fetchPublicAsset(event, fetchRoute, { responseType: "stream" });
+    if (!sitemapInput) {
       logger.warn(`[sitemap] Not found in ASSETS: ${fetchRoute}`);
       return { urls: [], error: "Not found in ASSETS" };
     }
@@ -53,38 +53,53 @@ export async function fetchSitemapByRoute(event, route, depth = 0) {
     try {
       const $fetch = event?.$fetch ?? globalThis.$fetch;
       const res = await $fetch(fetchRoute, {
-        responseType: "text",
+        responseType: "stream",
         timeout: FETCH_TIMEOUT
       });
-      if (!res || typeof res !== "string") {
+      if (!res) {
         logger.warn(`[sitemap] Empty response from ${fetchRoute}`);
         return { urls: [], error: "Empty response" };
       }
-      sitemapXml = res;
+      if (!isSitemapXmlInput(res)) {
+        logger.warn(`[sitemap] Invalid response body from ${fetchRoute}`);
+        return { urls: [], error: "Invalid response body" };
+      }
+      sitemapInput = res;
     } catch (e) {
       const msg = e instanceof Error ? e.message : String(e);
       logger.warn(`[sitemap] Failed to fetch ${fetchRoute}: ${msg}`);
       return { urls: [], error: msg };
     }
   }
-  logger.debug(`[sitemap] Parsing sitemap XML (${sitemapXml.length} bytes)`);
-  if (isSitemapIndex(sitemapXml)) {
+  logger.debug(`[sitemap] Parsing sitemap XML stream`);
+  const urls = [];
+  const indexEntries = [];
+  let kind;
+  try {
+    for await (const parsed of parseSitemapStream(sitemapInput)) {
+      if (parsed._tag === "kind")
+        kind = parsed.kind;
+      else if (parsed._tag === "url")
+        urls.push(normalizeUrl(parsed.url));
+      else if (parsed._tag === "sitemap")
+        indexEntries.push(parsed.sitemap);
+      else
+        logger.warn(`[sitemap] ${fetchRoute}: ${parsed.warning.message}`);
+    }
+  } catch (e) {
+    const msg = e instanceof Error ? e.message : String(e);
+    logger.warn(`[sitemap] Failed to parse ${fetchRoute}: ${msg}`);
+    return { urls: [], error: msg };
+  }
+  if (kind === "index") {
     if (depth >= 3) {
       logger.warn(`[sitemap] Sitemap index nesting too deep at ${fetchRoute}, stopping`);
       return { urls: [] };
     }
-    let index;
-    try {
-      index = await parseSitemapIndex(sitemapXml);
-    } catch (e) {
-      const msg = e instanceof Error ? e.message : String(e);
-      logger.warn(`[sitemap] Failed to parse sitemap index ${fetchRoute}: ${msg}`);
-      return { urls: [], error: msg };
-    }
-    logger.debug(`[sitemap] ${fetchRoute} is a sitemap index with ${index.entries.length} children`);
+    logger.debug(`[sitemap] ${fetchRoute} is a sitemap index with ${indexEntries.length} children`);
     const allUrls = [];
     const childErrors = [];
-    for (const entry of index.entries) {
+    for (const entry of indexEntries) {
       const childRoute = entry.loc.startsWith("http") ? new URL(entry.loc).pathname : entry.loc;
       if (withLeadingSlash(childRoute) === fetchRoute)
         continue;
@@ -94,21 +109,12 @@ export async function fetchSitemapByRoute(event, route, depth = 0) {
         childErrors.push(`${withLeadingSlash(childRoute)}: ${error}`);
     }
     if (childErrors.length > 0) {
-      const msg = `${childErrors.length}/${index.entries.length} child sitemaps failed (${childErrors.join("; ")})`;
+      const msg = `${childErrors.length}/${indexEntries.length} child sitemaps failed (${childErrors.join("; ")})`;
       logger.warn(`[sitemap] Sitemap index ${fetchRoute}: ${msg}`);
       return { urls: allUrls, error: msg };
     }
     return { urls: allUrls };
   }
-  let result;
-  try {
-    result = await parseSitemapXml(sitemapXml);
-  } catch (e) {
-    const msg = e instanceof Error ? e.message : String(e);
-    logger.warn(`[sitemap] Failed to parse ${fetchRoute}: ${msg}`);
-    return { urls: [], error: msg };
-  }
-  const urls = normalizeUrls(result?.urls || []);
   logger.debug(`[sitemap] Found ${urls.length} URLs in ${fetchRoute}`);
   return { urls };
 }

package/dist/runtime/types.d.ts

@@ -260,20 +260,27 @@ export interface LlmsTxtLink {
 export interface LlmsTxtSection {
     /** The title of the section */
     title: string;
-    /** The description of the section (can be array for multiple paragraphs) */
+    /** Description rendered in the heading-free preamble, or in each optional link note (can be array for multiple paragraphs) */
     description?: string | string[];
     /** The links of the section */
     links?: LlmsTxtLink[];
-    /** Mark section as optional per llms.txt spec. Optional sections appear under `## Optional` and LLMs may skip them with shorter context windows. */
+    /** Mark links as optional per llms.txt spec. Links from all optional sections are flattened under `## Optional`. */
     optional?: boolean;
 }
 /**
  * Structured llms.txt configuration
  */
 export interface LlmsTxtConfig {
+    /**
+     * Rewrite automatically generated page links to their Markdown representation
+     * when the final static output contains the file or the deployment retains
+     * the runtime Markdown handler.
+     * @default false
+     */
+    markdownLinks?: boolean;
     /** The sections of the documentation */
     sections?: LlmsTxtSection[];
-    /** Notes section (always appears at the end) */
+    /** Additional heading-free preamble notes rendered before file-list sections */
     notes?: string | string[];
 }
 /**

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "1.5.6",
+  "version": "1.5.8",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -42,7 +42,7 @@
   "peerDependencies": {
     "@libsql/client": "^0.14.0",
     "@neondatabase/serverless": "^1.0.0",
-    "@nuxtjs/sitemap": ">=7.0.0",
+    "@nuxtjs/sitemap": ">=8.3.0",
     "better-sqlite3": "^11.0.0 || ^12.0.0"
   },
   "peerDependenciesMeta": {
@@ -64,11 +64,11 @@
     "defu": "^6.1.7",
     "drizzle-orm": "^0.45.2",
     "mdream": "^1.5.3",
-    "nuxt-site-config": "^4.1.1",
+    "nuxt-site-config": "^4.1.2",
     "nuxtseo-shared": "^5.3.3",
     "pathe": "^2.0.3",
     "pkg-types": "^2.3.1",
-    "site-config-stack": "^4.1.1",
+    "site-config-stack": "^4.1.2",
     "ufo": "^1.6.4",
     "uncrypto": "^0.1.3"
   },
@@ -82,7 +82,7 @@
     "@nuxtjs/eslint-config-typescript": "^12.1.0",
     "@nuxtjs/mcp-toolkit": "^0.18.0",
     "@nuxtjs/robots": "^6.1.3",
-    "@nuxtjs/sitemap": "^8.2.2",
+    "@nuxtjs/sitemap": "^8.3.0",
     "@types/better-sqlite3": "^7.6.13",
     "@vitest/coverage-v8": "^4.1.10",
     "@vue/test-utils": "^2.4.11",
@@ -95,7 +95,7 @@
     "happy-dom": "^20.11.0",
     "nitropack": "^2.13.4",
     "nuxt": "^4.5.0",
-    "nuxt-site-config": "^4.1.1",
+    "nuxt-site-config": "^4.1.2",
     "nuxtseo-layer-devtools": "^5.3.3",
     "playwright": "^1.61.1",
     "playwright-core": "^1.61.1",