nuxt-ai-ready 1.5.6 → 1.5.7

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "1.5.6",
+  "version": "1.5.7",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -616,7 +616,7 @@ const module$1 = defineNuxtModule({
       version: ">=6.0.0"
     },
     "@nuxtjs/sitemap": {
-      version: ">=8.0.0"
+      version: ">=8.3.0"
     },
     "nuxt-site-config": {
       version: ">=3.2"

package/dist/runtime/llms-txt-format.d.ts

@@ -1,8 +1,19 @@
 /**
  * Pure formatting functions for llms.txt - no runtime dependencies
  */
+import type { RuntimeI18nConfig } from './server/utils/i18n.js';
 import type { LlmsTxtConfig } from './types.js';
+export declare function formatAvailableLanguagesSection(i18n: RuntimeI18nConfig, pageCounts: Map<string, number>, resolveHref?: (pathname: string) => string): string[];
+interface LlmsTxtPageLink {
+    pathname: string;
+    href?: string;
+    title?: string;
+    description?: string;
+}
+export declare function formatLlmsTxtPageLink(page: LlmsTxtPageLink): string;
 /**
- * Normalize llms.txt structured configuration to markdown string
+ * Normalize llms.txt structured configuration to the llmstxt.org Markdown format.
+ * Preamble prose must appear before the first H2; every H2 body is a file list.
  */
 export declare function normalizeLlmsTxtConfig(config: LlmsTxtConfig): string;
+export {};

package/dist/runtime/llms-txt-format.js

@@ -1,39 +1,100 @@
-function normalizeLink(link) {
-  const parts = [];
-  parts.push(`- [${link.title}](${link.href})`);
-  if (link.description)
-    parts.push(`  ${link.description}`);
-  return parts.join("\n");
+import { localePath } from "./server/utils/i18n.js";
+const RE_INLINE_WHITESPACE = /\s+/g;
+const RE_LINK_TITLE_BRACKET = /[[\]]/g;
+const RE_LINK_HREF_UNSAFE = /[\s()]/g;
+const RE_PREAMBLE_ATX_HEADING = /^( {0,3})(#{1,6})(?=\s)/gm;
+function normalizeInlineText(value) {
+  return value.trim().replace(RE_INLINE_WHITESPACE, " ");
 }
-function normalizeSection(section, headingLevel = 2) {
-  const prefix = "#".repeat(headingLevel);
-  const parts = [];
-  parts.push(`${prefix} ${section.title}`);
-  parts.push("");
-  if (section.description) {
-    const descriptions = Array.isArray(section.description) ? section.description : [section.description];
-    parts.push(...descriptions);
-    parts.push("");
-  }
+function normalizeLinkTitle(value) {
+  return normalizeInlineText(value).replace(RE_LINK_TITLE_BRACKET, (bracket) => bracket === "[" ? "[" : "]");
+}
+function normalizeLinkHref(value) {
+  return value.trim().replace(RE_LINK_HREF_UNSAFE, (character) => {
+    if (character === "(")
+      return "%28";
+    if (character === ")")
+      return "%29";
+    return encodeURIComponent(character);
+  });
+}
+function normalizeDescriptions(value) {
+  return (Array.isArray(value) ? value : value ? [value] : []).filter((description) => description.trim()).map(normalizeInlineText);
+}
+function normalizePreambleBlocks(value) {
+  return (Array.isArray(value) ? value : value ? [value] : []).filter((block) => block.trim()).map((block) => block.trim().replace(RE_PREAMBLE_ATX_HEADING, "$1\\$2"));
+}
+function normalizeLink(link, descriptionPrefixes = []) {
+  const descriptions = [...descriptionPrefixes, link.description].filter((value) => Boolean(value?.trim())).map(normalizeInlineText);
+  const description = descriptions.join("; ");
+  return `- [${normalizeLinkTitle(link.title)}](${normalizeLinkHref(link.href)})${description ? `: ${description}` : ""}`;
+}
+function normalizeSection(section) {
+  const parts = [`## ${normalizeInlineText(section.title)}`];
   if (section.links?.length)
-    parts.push(...section.links.map(normalizeLink));
+    parts.push("", ...section.links.map((link) => normalizeLink(link)));
   return parts.join("\n");
 }
-export function normalizeLlmsTxtConfig(config) {
+function normalizePreamble(config) {
   const parts = [];
-  const required = config.sections?.filter((s) => !s.optional) ?? [];
-  const optional = config.sections?.filter((s) => s.optional) ?? [];
-  if (required.length)
-    parts.push(...required.map((s) => normalizeSection(s)));
-  if (optional.length) {
-    parts.push("## Optional");
-    parts.push(...optional.map((s) => normalizeSection(s, 3)));
-  }
   if (config.notes) {
-    parts.push("## Notes");
-    parts.push("");
-    const notes = Array.isArray(config.notes) ? config.notes : [config.notes];
-    parts.push(...notes);
+    const notes = normalizePreambleBlocks(config.notes);
+    if (notes.length)
+      parts.push(["**Notes:**", ...notes].join("\n\n"));
+  }
+  for (const section of config.sections?.filter((section2) => !section2.optional) ?? []) {
+    if (!section.description)
+      continue;
+    const descriptions = normalizePreambleBlocks(section.description);
+    if (descriptions.length)
+      parts.push([`**${normalizeInlineText(section.title)}:**`, ...descriptions].join("\n\n"));
   }
+  return parts;
+}
+function normalizeOptionalSections(sections) {
+  const links = sections.flatMap(
+    (section) => section.links?.map((link) => normalizeLink(link, [section.title, ...normalizeDescriptions(section.description)])) ?? []
+  );
+  if (!links.length)
+    return void 0;
+  return ["## Optional", "", ...links].join("\n");
+}
+function normalizeRequiredSections(sections) {
+  return sections.filter((section) => section.links?.length).map(normalizeSection);
+}
+export function formatAvailableLanguagesSection(i18n, pageCounts, resolveHref = (pathname) => pathname) {
+  const lines = ["## Available Languages on Website", ""];
+  for (const locale of i18n.locales) {
+    const isDefault = locale.code === i18n.defaultLocale;
+    const prefix = localePath("/", locale.code, i18n);
+    const count = pageCounts.get(locale.code) ?? 0;
+    const display = locale.nativeName ? `${locale.nativeName} (${locale.code})` : locale.name ? `${locale.name} (${locale.code})` : locale.code;
+    const suffix = isDefault ? "content included below" : "visit this language for content";
+    lines.push(normalizeLink({
+      title: display,
+      href: resolveHref(prefix),
+      description: `${count} pages; ${suffix}.`
+    }));
+  }
+  return lines;
+}
+export function formatLlmsTxtPageLink(page) {
+  const description = page.description?.trim().replace(RE_INLINE_WHITESPACE, " ");
+  const normalizedTitle = page.title ? normalizeInlineText(page.title) : "";
+  const title = normalizedTitle && normalizedTitle !== page.pathname ? normalizedTitle : page.pathname;
+  const href = (page.href || page.pathname).trim();
+  const truncatedDescription = description ? `${description.substring(0, 160)}${description.length > 160 ? "..." : ""}` : void 0;
+  return normalizeLink({ title, href, description: truncatedDescription });
+}
+export function normalizeLlmsTxtConfig(config) {
+  const required = config.sections?.filter((section) => !section.optional) ?? [];
+  const optional = config.sections?.filter((section) => section.optional) ?? [];
+  const parts = [
+    ...normalizePreamble(config),
+    ...normalizeRequiredSections(required)
+  ];
+  const optionalSection = normalizeOptionalSections(optional);
+  if (optionalSection)
+    parts.push(optionalSection);
   return parts.join("\n\n");
 }

package/dist/runtime/llms-txt-utils.js

@@ -1,25 +1,13 @@
 import { useRuntimeConfig } from "nitropack/runtime";
 import { getSiteConfig } from "#site-config/server/composables/getSiteConfig";
 import { withSiteTrailingSlash, withSiteUrl } from "#site-config/server/composables/utils";
-import { normalizeLlmsTxtConfig } from "./llms-txt-format.js";
+import { formatAvailableLanguagesSection, formatLlmsTxtPageLink, normalizeLlmsTxtConfig } from "./llms-txt-format.js";
 import { normalizePersistedRoute, toDeployedRoute, toLogicalRoute } from "./route-path.js";
 import { queryPages } from "./server/db/queries.js";
 import { logger } from "./server/logger.js";
-import { localePath, resolveLocaleFromRoute } from "./server/utils/i18n.js";
+import { resolveLocaleFromRoute } from "./server/utils/i18n.js";
 import { fetchSitemapUrls } from "./server/utils/sitemap.js";
 export { normalizeLlmsTxtConfig };
-function formatAvailableLanguagesSection(i18n, pageCounts, resolvePath) {
-  const lines = ["## Available Languages on Website", ""];
-  for (const locale of i18n.locales) {
-    const isDefault = locale.code === i18n.defaultLocale;
-    const prefix = resolvePath(localePath("/", locale.code, i18n));
-    const count = pageCounts.get(locale.code) ?? 0;
-    const display = locale.nativeName ? `${locale.nativeName} (${locale.code})` : locale.name ? `${locale.name} (${locale.code})` : locale.code;
-    const suffix = isDefault ? "Content included below" : "Visit website for content";
-    lines.push(`- ${display} - ${count} pages - ${prefix} - ${suffix}`);
-  }
-  return lines;
-}
 function getGroupPrefix(url, depth) {
   const segments = url.split("/").filter(Boolean);
   if (segments.length === 0)
@@ -103,12 +91,7 @@ function formatPagesWithGroups(pages) {
       urlsInCurrentGroup = 0;
     }
     urlsInCurrentGroup++;
-    const descText = page.description ? `: ${page.description.substring(0, 160)}${page.description.length > 160 ? "..." : ""}` : "";
-    const href = page.href ?? page.pathname;
-    if (page.title && page.title !== page.pathname)
-      lines.push(`- [${page.title}](${href})${descText}`);
-    else
-      lines.push(`- ${href}${descText}`);
+    lines.push(formatLlmsTxtPageLink(page));
   }
   return lines;
 }

package/dist/runtime/server/utils/cloudflare.d.ts

@@ -17,5 +17,5 @@ export declare function hasAssets(event?: H3Event): boolean;
  * Falls back to $fetch with timeout to avoid self-fetch hangs on CF Workers.
  */
 export declare function fetchPublicAsset<T = unknown>(event: H3Event | undefined, path: string, options?: {
-    responseType?: 'json' | 'text' | 'arrayBuffer';
+    responseType?: 'json' | 'text' | 'arrayBuffer' | 'stream';
 }): Promise<T | null>;

package/dist/runtime/server/utils/cloudflare.js

@@ -19,6 +19,8 @@ export async function fetchPublicAsset(event, path, options) {
         return response.text().catch(() => null);
       if (responseType === "arrayBuffer")
         return response.arrayBuffer().catch(() => null);
+      if (responseType === "stream")
+        return response.body;
     }
     return null;
   }
@@ -27,6 +29,6 @@ export async function fetchPublicAsset(event, path, options) {
   return globalThis.$fetch(path, {
     baseURL: "/",
     signal: controller.signal,
-    responseType: responseType === "arrayBuffer" ? "arrayBuffer" : void 0
+    responseType: responseType === "arrayBuffer" || responseType === "stream" ? responseType : void 0
   }).catch(() => null).finally(() => clearTimeout(timeout));
 }

package/dist/runtime/server/utils/sitemap.js

@@ -1,4 +1,4 @@
-import { isSitemapIndex, parseSitemapIndex, parseSitemapXml } from "@nuxtjs/sitemap/utils";
+import { parseSitemapStream } from "@nuxtjs/sitemap/utils";
 import { useRuntimeConfig } from "nitropack/runtime";
 import { withLeadingSlash } from "ufo";
 import { logger } from "../logger.js";
@@ -26,26 +26,26 @@ export function hasMultipleSitemaps(event) {
   const sitemaps = getSitemapsFromConfig(event);
   return sitemaps.length > 1;
 }
-function normalizeUrls(urls) {
-  return urls.map((entry) => {
-    if (typeof entry === "string")
-      return { loc: entry };
-    const e = entry;
-    return {
-      loc: e.loc,
-      lastmod: e.lastmod instanceof Date ? e.lastmod.toISOString() : e.lastmod
-    };
-  });
+function normalizeUrl(entry) {
+  if (typeof entry === "string")
+    return { loc: entry };
+  return {
+    loc: entry.loc,
+    lastmod: entry.lastmod instanceof Date ? entry.lastmod.toISOString() : entry.lastmod
+  };
+}
+function isSitemapXmlInput(value) {
+  return typeof value === "string" || value instanceof Uint8Array || typeof value === "object" && value !== null && "getReader" in value;
 }
 export async function fetchSitemapByRoute(event, route, depth = 0) {
   const config = useRuntimeConfig(event)["nuxt-ai-ready"];
   const fetchRoute = withLeadingSlash(route);
   const usePublicAsset = config.sitemapPrerendered && hasAssets(event);
   logger.debug(`[sitemap] Fetching ${fetchRoute} via ${usePublicAsset ? "ASSETS.fetch" : event ? "event.$fetch" : "globalThis.$fetch"}`);
-  let sitemapXml = null;
+  let sitemapInput = null;
   if (usePublicAsset) {
-    sitemapXml = await fetchPublicAsset(event, fetchRoute, { responseType: "text" });
-    if (!sitemapXml) {
+    sitemapInput = await fetchPublicAsset(event, fetchRoute, { responseType: "stream" });
+    if (!sitemapInput) {
       logger.warn(`[sitemap] Not found in ASSETS: ${fetchRoute}`);
       return { urls: [], error: "Not found in ASSETS" };
     }
@@ -53,38 +53,53 @@ export async function fetchSitemapByRoute(event, route, depth = 0) {
     try {
       const $fetch = event?.$fetch ?? globalThis.$fetch;
       const res = await $fetch(fetchRoute, {
-        responseType: "text",
+        responseType: "stream",
         timeout: FETCH_TIMEOUT
       });
-      if (!res || typeof res !== "string") {
+      if (!res) {
         logger.warn(`[sitemap] Empty response from ${fetchRoute}`);
         return { urls: [], error: "Empty response" };
       }
-      sitemapXml = res;
+      if (!isSitemapXmlInput(res)) {
+        logger.warn(`[sitemap] Invalid response body from ${fetchRoute}`);
+        return { urls: [], error: "Invalid response body" };
+      }
+      sitemapInput = res;
     } catch (e) {
       const msg = e instanceof Error ? e.message : String(e);
       logger.warn(`[sitemap] Failed to fetch ${fetchRoute}: ${msg}`);
       return { urls: [], error: msg };
     }
   }
-  logger.debug(`[sitemap] Parsing sitemap XML (${sitemapXml.length} bytes)`);
-  if (isSitemapIndex(sitemapXml)) {
+  logger.debug(`[sitemap] Parsing sitemap XML stream`);
+  const urls = [];
+  const indexEntries = [];
+  let kind;
+  try {
+    for await (const parsed of parseSitemapStream(sitemapInput)) {
+      if (parsed._tag === "kind")
+        kind = parsed.kind;
+      else if (parsed._tag === "url")
+        urls.push(normalizeUrl(parsed.url));
+      else if (parsed._tag === "sitemap")
+        indexEntries.push(parsed.sitemap);
+      else
+        logger.warn(`[sitemap] ${fetchRoute}: ${parsed.warning.message}`);
+    }
+  } catch (e) {
+    const msg = e instanceof Error ? e.message : String(e);
+    logger.warn(`[sitemap] Failed to parse ${fetchRoute}: ${msg}`);
+    return { urls: [], error: msg };
+  }
+  if (kind === "index") {
     if (depth >= 3) {
       logger.warn(`[sitemap] Sitemap index nesting too deep at ${fetchRoute}, stopping`);
       return { urls: [] };
     }
-    let index;
-    try {
-      index = await parseSitemapIndex(sitemapXml);
-    } catch (e) {
-      const msg = e instanceof Error ? e.message : String(e);
-      logger.warn(`[sitemap] Failed to parse sitemap index ${fetchRoute}: ${msg}`);
-      return { urls: [], error: msg };
-    }
-    logger.debug(`[sitemap] ${fetchRoute} is a sitemap index with ${index.entries.length} children`);
+    logger.debug(`[sitemap] ${fetchRoute} is a sitemap index with ${indexEntries.length} children`);
     const allUrls = [];
     const childErrors = [];
-    for (const entry of index.entries) {
+    for (const entry of indexEntries) {
       const childRoute = entry.loc.startsWith("http") ? new URL(entry.loc).pathname : entry.loc;
       if (withLeadingSlash(childRoute) === fetchRoute)
         continue;
@@ -94,21 +109,12 @@ export async function fetchSitemapByRoute(event, route, depth = 0) {
         childErrors.push(`${withLeadingSlash(childRoute)}: ${error}`);
     }
     if (childErrors.length > 0) {
-      const msg = `${childErrors.length}/${index.entries.length} child sitemaps failed (${childErrors.join("; ")})`;
+      const msg = `${childErrors.length}/${indexEntries.length} child sitemaps failed (${childErrors.join("; ")})`;
       logger.warn(`[sitemap] Sitemap index ${fetchRoute}: ${msg}`);
       return { urls: allUrls, error: msg };
     }
     return { urls: allUrls };
   }
-  let result;
-  try {
-    result = await parseSitemapXml(sitemapXml);
-  } catch (e) {
-    const msg = e instanceof Error ? e.message : String(e);
-    logger.warn(`[sitemap] Failed to parse ${fetchRoute}: ${msg}`);
-    return { urls: [], error: msg };
-  }
-  const urls = normalizeUrls(result?.urls || []);
   logger.debug(`[sitemap] Found ${urls.length} URLs in ${fetchRoute}`);
   return { urls };
 }

package/dist/runtime/types.d.ts

@@ -260,11 +260,11 @@ export interface LlmsTxtLink {
 export interface LlmsTxtSection {
     /** The title of the section */
     title: string;
-    /** The description of the section (can be array for multiple paragraphs) */
+    /** Description rendered in the heading-free preamble, or in each optional link note (can be array for multiple paragraphs) */
     description?: string | string[];
     /** The links of the section */
     links?: LlmsTxtLink[];
-    /** Mark section as optional per llms.txt spec. Optional sections appear under `## Optional` and LLMs may skip them with shorter context windows. */
+    /** Mark links as optional per llms.txt spec. Links from all optional sections are flattened under `## Optional`. */
     optional?: boolean;
 }
 /**
@@ -273,7 +273,7 @@ export interface LlmsTxtSection {
 export interface LlmsTxtConfig {
     /** The sections of the documentation */
     sections?: LlmsTxtSection[];
-    /** Notes section (always appears at the end) */
+    /** Additional heading-free preamble notes rendered before file-list sections */
     notes?: string | string[];
 }
 /**

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "1.5.6",
+  "version": "1.5.7",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -42,7 +42,7 @@
   "peerDependencies": {
     "@libsql/client": "^0.14.0",
     "@neondatabase/serverless": "^1.0.0",
-    "@nuxtjs/sitemap": ">=7.0.0",
+    "@nuxtjs/sitemap": ">=8.3.0",
     "better-sqlite3": "^11.0.0 || ^12.0.0"
   },
   "peerDependenciesMeta": {
@@ -64,11 +64,11 @@
     "defu": "^6.1.7",
     "drizzle-orm": "^0.45.2",
     "mdream": "^1.5.3",
-    "nuxt-site-config": "^4.1.1",
+    "nuxt-site-config": "^4.1.2",
     "nuxtseo-shared": "^5.3.3",
     "pathe": "^2.0.3",
     "pkg-types": "^2.3.1",
-    "site-config-stack": "^4.1.1",
+    "site-config-stack": "^4.1.2",
     "ufo": "^1.6.4",
     "uncrypto": "^0.1.3"
   },
@@ -82,7 +82,7 @@
     "@nuxtjs/eslint-config-typescript": "^12.1.0",
     "@nuxtjs/mcp-toolkit": "^0.18.0",
     "@nuxtjs/robots": "^6.1.3",
-    "@nuxtjs/sitemap": "^8.2.2",
+    "@nuxtjs/sitemap": "^8.3.0",
     "@types/better-sqlite3": "^7.6.13",
     "@vitest/coverage-v8": "^4.1.10",
     "@vue/test-utils": "^2.4.11",
@@ -95,7 +95,7 @@
     "happy-dom": "^20.11.0",
     "nitropack": "^2.13.4",
     "nuxt": "^4.5.0",
-    "nuxt-site-config": "^4.1.1",
+    "nuxt-site-config": "^4.1.2",
     "nuxtseo-layer-devtools": "^5.3.3",
     "playwright": "^1.61.1",
     "playwright-core": "^1.61.1",