nuxt-ai-ready 0.4.1 → 0.4.2

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -99,6 +99,7 @@ function normalizeLlmsTxtConfig(config) {
 function createCrawlerState(pageDataPath, llmsFullTxtPath, siteInfo, llmsTxtConfig) {
   return {
     prerenderedRoutes: /* @__PURE__ */ new Set(),
+    errorRoutes: /* @__PURE__ */ new Set(),
     totalProcessingTime: 0,
     initialized: false,
     jsonlInitialized: false,
@@ -221,7 +222,7 @@ async function crawlSitemapEntries(state, nuxt, nitro, entries) {
     const res = await globalThis.$fetch(mdUrl, {
       headers: { "x-nitro-prerender": mdRoute }
     }).catch((err) => {
-      logger.debug(`Failed to fetch ${mdUrl}: ${err.message}`);
+      logger.debug(`Skipping ${route}: ${err.message}`);
       return null;
     });
     if (!res)
@@ -230,7 +231,7 @@ async function crawlSitemapEntries(state, nuxt, nitro, entries) {
     await processMarkdownRoute(state, nuxt, route, parsed, lastmod);
     crawled++;
   }
-  logger.debug(`Sitemap crawl complete: ${crawled} crawled, ${skipped} skipped (already indexed)`);
+  logger.debug(`Sitemap crawl complete: ${crawled} crawled, ${skipped} skipped`);
   return crawled;
 }
 async function crawlSitemapContent(state, nuxt, nitro, sitemapContent) {
@@ -296,6 +297,12 @@ function setupPrerenderHandler(pageDataPath, siteInfo, llmsTxtConfig) {
     const state = createCrawlerState(pageDataPath, llmsFullTxtPath, siteInfo, llmsTxtConfig);
     let initPromise = null;
     nitro.hooks.hook("prerender:generate", async (route) => {
+      if (route.error) {
+        const pageRoute2 = route.route.replace(/\.(html|md)$/, "").replace(/\/index$/, "") || "/";
+        state.errorRoutes.add(pageRoute2);
+        logger.debug(`Detected error page: ${pageRoute2}`);
+        return;
+      }
       if (!route.fileName?.endsWith(".md"))
         return;
       let pageRoute = route.route.replace(/\.md$/, "");
@@ -341,6 +348,13 @@ function setupPrerenderHandler(pageDataPath, siteInfo, llmsTxtConfig) {
       state.totalProcessingTime += Date.now() - pageStartTime;
     });
     async function writeLlmsFiles() {
+      if (state.pageDataPath && state.errorRoutes.size > 0) {
+        for (const route of state.errorRoutes) {
+          await appendFile(state.pageDataPath, `${JSON.stringify({ route, _error: true })}
+`, "utf-8");
+        }
+        logger.debug(`Wrote ${state.errorRoutes.size} error routes to page data`);
+      }
       const llmsStats = await prerenderRoute(nitro, "/llms.txt");
       const llmsFullStats = await stat(state.llmsFullTxtPath);
       const kb = (b) => (b / 1024).toFixed(1);
@@ -536,14 +550,18 @@ import { readFile } from 'node:fs/promises'
 
 export async function readPageDataFromFilesystem() {
   if (!import.meta.prerender) {
-    return null
+    return { pages: [], errorRoutes: [] }
   }
   const data = await readFile(${JSON.stringify(pageDataPath)}, 'utf-8').catch(() => null)
-  if (!data) return []
-  return data.trim().split('\\n').filter(Boolean).map(line => JSON.parse(line))
+  if (!data) return { pages: [], errorRoutes: [] }
+  const entries = data.trim().split('\\n').filter(Boolean).map(line => JSON.parse(line))
+  const pages = entries.filter(e => !e._error)
+  const errorRoutes = entries.filter(e => e._error).map(e => e.route)
+  return { pages, errorRoutes }
 }
 `;
-      nitroConfig.virtual["#ai-ready-virtual/page-data.mjs"] = `export const pages = []`;
+      nitroConfig.virtual["#ai-ready-virtual/page-data.mjs"] = `export const pages = []
+export const errorRoutes = []`;
     });
     nuxt.options.runtimeConfig["nuxt-ai-ready"] = {
       version: version || "0.0.0",

package/dist/runtime/llms-txt-utils.js

@@ -1,7 +1,123 @@
 import { getSiteConfig } from "#site-config/server/composables/getSiteConfig";
 import { useRuntimeConfig } from "nitropack/runtime";
-import { getPages } from "./server/utils/pageData.js";
+import { getErrorRoutes, getPages } from "./server/utils/pageData.js";
 import { fetchSitemapUrls } from "./server/utils/sitemap.js";
+function getGroupPrefix(url, depth) {
+  const segments = url.split("/").filter(Boolean);
+  if (segments.length === 0)
+    return "/";
+  if (depth === 1 || segments.length === 1)
+    return `/${segments[0]}`;
+  return `/${segments[0]}/${segments[1]}`;
+}
+function getPathSegments(pathname) {
+  return pathname.split("/").filter((s) => Boolean(s));
+}
+function sortPagesByPath(pages) {
+  const twoSegmentCount = /* @__PURE__ */ new Map();
+  for (const page of pages) {
+    const prefix = getGroupPrefix(page.pathname, 2);
+    twoSegmentCount.set(prefix, (twoSegmentCount.get(prefix) || 0) + 1);
+  }
+  const segmentHasNested = /* @__PURE__ */ new Map();
+  for (const page of pages) {
+    const segments = getPathSegments(page.pathname);
+    const firstSegment = segments[0] || "";
+    if (!segmentHasNested.has(firstSegment))
+      segmentHasNested.set(firstSegment, false);
+    if (segments.length > 1)
+      segmentHasNested.set(firstSegment, true);
+  }
+  return pages.sort((a, b) => {
+    const segmentsA = getPathSegments(a.pathname);
+    const segmentsB = getPathSegments(b.pathname);
+    const firstSegmentA = segmentsA[0] || "";
+    const firstSegmentB = segmentsB[0] || "";
+    const twoSegPrefixA = getGroupPrefix(a.pathname, 2);
+    const twoSegPrefixB = getGroupPrefix(b.pathname, 2);
+    const twoSegCountA = twoSegmentCount.get(twoSegPrefixA) || 0;
+    const twoSegCountB = twoSegmentCount.get(twoSegPrefixB) || 0;
+    let groupKeyA = twoSegCountA > 1 ? twoSegPrefixA : `/${firstSegmentA}`;
+    let groupKeyB = twoSegCountB > 1 ? twoSegPrefixB : `/${firstSegmentB}`;
+    const isRootLevelA = segmentsA.length <= 1;
+    const isRootLevelB = segmentsB.length <= 1;
+    const hasNestedA = segmentHasNested.get(firstSegmentA);
+    const hasNestedB = segmentHasNested.get(firstSegmentB);
+    if (isRootLevelA && !hasNestedA)
+      groupKeyA = "";
+    if (isRootLevelB && !hasNestedB)
+      groupKeyB = "";
+    if (groupKeyA === "" && groupKeyB !== "")
+      return -1;
+    if (groupKeyA !== "" && groupKeyB === "")
+      return 1;
+    if (groupKeyA !== groupKeyB)
+      return groupKeyA.localeCompare(groupKeyB);
+    if (segmentsA.length === 0)
+      return -1;
+    if (segmentsB.length === 0)
+      return 1;
+    const minLen = Math.min(segmentsA.length, segmentsB.length);
+    for (let i = 0; i < minLen; i++) {
+      const cmp = segmentsA[i].localeCompare(segmentsB[i]);
+      if (cmp !== 0)
+        return cmp;
+    }
+    return segmentsA.length - segmentsB.length;
+  });
+}
+function getPageGroupKey(pathname, twoSegmentCount, segmentHasNested) {
+  const segments = getPathSegments(pathname);
+  const firstSegment = segments[0] || "";
+  const twoSegPrefix = getGroupPrefix(pathname, 2);
+  const twoSegCount = twoSegmentCount.get(twoSegPrefix) || 0;
+  let groupKey = twoSegCount > 1 ? twoSegPrefix : `/${firstSegment}`;
+  const isRootLevel = segments.length <= 1;
+  const hasNested = segmentHasNested.get(firstSegment);
+  if (isRootLevel && !hasNested)
+    groupKey = "";
+  return groupKey;
+}
+function formatPagesWithGroups(pages) {
+  if (pages.length === 0)
+    return [];
+  const twoSegmentCount = /* @__PURE__ */ new Map();
+  const segmentHasNested = /* @__PURE__ */ new Map();
+  for (const page of pages) {
+    const prefix = getGroupPrefix(page.pathname, 2);
+    twoSegmentCount.set(prefix, (twoSegmentCount.get(prefix) || 0) + 1);
+    const segments = getPathSegments(page.pathname);
+    const firstSegment = segments[0] || "";
+    if (!segmentHasNested.has(firstSegment))
+      segmentHasNested.set(firstSegment, false);
+    if (segments.length > 1)
+      segmentHasNested.set(firstSegment, true);
+  }
+  const lines = [];
+  let currentGroup = "";
+  let segmentGroupIndex = 0;
+  let urlsInCurrentGroup = 0;
+  for (const page of pages) {
+    const groupKey = getPageGroupKey(page.pathname, twoSegmentCount, segmentHasNested);
+    if (groupKey !== currentGroup) {
+      if (urlsInCurrentGroup > 0) {
+        const shouldAddBlankLine = segmentGroupIndex === 0 || segmentGroupIndex >= 1 && segmentGroupIndex <= 2 && urlsInCurrentGroup > 1;
+        if (shouldAddBlankLine)
+          lines.push("");
+      }
+      currentGroup = groupKey;
+      segmentGroupIndex++;
+      urlsInCurrentGroup = 0;
+    }
+    urlsInCurrentGroup++;
+    const descText = page.description ? `: ${page.description.substring(0, 160)}${page.description.length > 160 ? "..." : ""}` : "";
+    if (page.title && page.title !== page.pathname)
+      lines.push(`- [${page.title}](${page.pathname})${descText}`);
+    else
+      lines.push(`- ${page.pathname}${descText}`);
+  }
+  return lines;
+}
 function normalizeLink(link) {
   const parts = [];
   parts.push(`- [${link.title}](${link.href})`);
@@ -71,43 +187,41 @@ Canonical Origin: ${siteConfig.url}`);
   }
   const pages = await getPages();
   const urls = await fetchSitemapUrls(event);
+  const errorRoutes = await getErrorRoutes();
   const devModeHint = import.meta.dev && pages.size === 0 ? " (dev mode - run `nuxi generate` for page titles)" : "";
   const prerendered = [];
+  const seenPaths = /* @__PURE__ */ new Set();
   for (const [pathname, page] of pages) {
-    prerendered.push({ pathname, title: page.title });
+    prerendered.push({ pathname, title: page.title, description: page.description });
+    seenPaths.add(pathname);
   }
   const other = [];
   for (const url of urls) {
     const pathname = url.loc.startsWith("http") ? new URL(url.loc).pathname : url.loc;
-    if (!pages.has(pathname)) {
-      other.push(pathname);
+    if (!seenPaths.has(pathname) && !errorRoutes.has(pathname)) {
+      other.push({ pathname });
+      seenPaths.add(pathname);
     }
   }
-  if (prerendered.length > 0 && other.length > 0) {
+  const sortedPrerendered = sortPagesByPath(prerendered);
+  const sortedOther = sortPagesByPath(other);
+  if (sortedPrerendered.length > 0 && sortedOther.length > 0) {
     parts.push(`## Prerendered Pages${devModeHint}
 `);
-    for (const { pathname, title } of prerendered) {
-      parts.push(title && title !== pathname ? `- [${title}](${pathname})` : `- ${pathname}`);
-    }
+    parts.push(...formatPagesWithGroups(sortedPrerendered));
     parts.push("");
     parts.push("## Other Pages\n");
-    for (const pathname of other) {
-      parts.push(`- ${pathname}`);
-    }
+    parts.push(...formatPagesWithGroups(sortedOther));
     parts.push("");
-  } else if (prerendered.length > 0) {
+  } else if (sortedPrerendered.length > 0) {
     parts.push(`## Pages${devModeHint}
 `);
-    for (const { pathname, title } of prerendered) {
-      parts.push(title && title !== pathname ? `- [${title}](${pathname})` : `- ${pathname}`);
-    }
+    parts.push(...formatPagesWithGroups(sortedPrerendered));
     parts.push("");
-  } else if (other.length > 0) {
+  } else if (sortedOther.length > 0) {
     parts.push(`## Pages${devModeHint}
 `);
-    for (const pathname of other) {
-      parts.push(`- ${pathname}`);
-    }
+    parts.push(...formatPagesWithGroups(sortedOther));
     parts.push("");
   }
   return parts.join("\n");

package/dist/runtime/server/utils/pageData.d.ts

@@ -12,8 +12,6 @@ export interface PageData extends PageEntry {
 }
 /** Read page data - returns page data indexed by route */
 export declare function getPages(): Promise<Map<string, PageEntry>>;
-/** Get all page data including markdown (prerender only) */
-export declare function readPrerenderedPageData(): Promise<Map<string, PageData>>;
 /** Page list item for MCP tools/resources */
 export interface PageListItem {
     route: string;
@@ -23,3 +21,5 @@ export interface PageListItem {
 }
 /** Get pages as flat list for MCP consumption */
 export declare function getPagesList(): Promise<PageListItem[]>;
+/** Get error routes detected during prerender */
+export declare function getErrorRoutes(): Promise<Set<string>>;

package/dist/runtime/server/utils/pageData.js

@@ -2,17 +2,21 @@ export async function getPages() {
   if (import.meta.dev)
     return /* @__PURE__ */ new Map();
   if (import.meta.prerender) {
-    return readPrerenderedPageData();
+    const data = await readPrerenderedData();
+    return data.pages;
   }
   const m = await import("#ai-ready-virtual/page-data.mjs");
   return m.pages?.length ? new Map(m.pages.map((p) => [p.route, p])) : /* @__PURE__ */ new Map();
 }
-export async function readPrerenderedPageData() {
+async function readPrerenderedData() {
   if (!import.meta.prerender)
-    return /* @__PURE__ */ new Map();
-  const { readPageDataFromFilesystem } = await import("#ai-ready-virtual/read-page-data.mjs");
-  const pages = await readPageDataFromFilesystem();
-  return pages?.length ? new Map(pages.map((p) => [p.route, p])) : /* @__PURE__ */ new Map();
+    return { pages: /* @__PURE__ */ new Map(), errorRoutes: /* @__PURE__ */ new Set() };
+  const m = await import("#ai-ready-virtual/read-page-data.mjs");
+  const data = await m.readPageDataFromFilesystem();
+  return {
+    pages: data.pages?.length ? new Map(data.pages.map((p) => [p.route, p])) : /* @__PURE__ */ new Map(),
+    errorRoutes: new Set(data.errorRoutes || [])
+  };
 }
 export async function getPagesList() {
   const pages = await getPages();
@@ -23,3 +27,13 @@ export async function getPagesList() {
     headings: p.headings || void 0
   }));
 }
+export async function getErrorRoutes() {
+  if (import.meta.dev)
+    return /* @__PURE__ */ new Set();
+  if (import.meta.prerender) {
+    const data = await readPrerenderedData();
+    return data.errorRoutes;
+  }
+  const m = await import("#ai-ready-virtual/page-data.mjs");
+  return new Set(m.errorRoutes || []);
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",