nuxt-ai-ready 1.5.1 → 1.5.3

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "1.5.1",
+  "version": "1.5.3",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -672,7 +672,7 @@ const module$1 = defineNuxtModule({
     const contentVersion = await resolveNuxtContentVersion();
     const hasNuxtContentV3 = !!(contentVersion && contentVersion.version === 3);
     if (typeof config.contentSignal === "object") {
-      const robotsOpts = nuxt.options.robots !== false ? nuxt.options.robots : {};
+      const robotsOpts = nuxt.options.robots || {};
       nuxt.options.robots = robotsOpts;
       const groups = robotsOpts.groups || [];
       robotsOpts.groups = groups;

package/dist/runtime/server/db/queries.d.ts

@@ -315,6 +315,12 @@ export declare function syncSitemaps(event: H3Event | undefined, sitemaps: Array
  * Skips sitemaps crawled within minIntervalMinutes (default 5 min)
  */
 export declare function getNextSitemapToCrawl(event: H3Event | undefined, minIntervalMinutes?: number): Promise<SitemapEntry | null>;
+/**
+ * Get the last crawl timestamp for a single sitemap. Used to throttle runtime
+ * re-seeding (the sitemap:resolved hook fires on every sitemap request).
+ * Returns null when the sitemap row doesn't exist yet.
+ */
+export declare function getSitemapLastCrawledAt(event: H3Event | undefined, name: string): Promise<number | null>;
 /**
  * Mark sitemap as successfully crawled
  */

package/dist/runtime/server/db/queries.js

@@ -301,22 +301,31 @@ export async function getPageHash(event, route) {
 }
 export async function seedRoutes(event, routes) {
   const db = await getDb(event);
-  if (!db)
+  if (!db || routes.length === 0)
     return 0;
   const now = (/* @__PURE__ */ new Date()).toISOString();
   const nowMs = Date.now();
+  const byRoute = /* @__PURE__ */ new Map();
   for (const entry of routes) {
     const route = typeof entry === "string" ? entry : entry.route;
     const explicitLocale = typeof entry === "string" ? void 0 : entry.locale;
-    const locale = deriveLocale(event, route, explicitLocale);
-    const routeKey = normalizeRouteKey(route);
+    byRoute.set(route, {
+      route,
+      routeKey: normalizeRouteKey(route),
+      locale: deriveLocale(event, route, explicitLocale)
+    });
+  }
+  const ROWS_PER_INSERT = 20;
+  for (const batch of chunk([...byRoute.values()], ROWS_PER_INSERT)) {
+    const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', ?, 0, 0, 0, 'runtime', ?, ?)`).join(", ");
+    const params = batch.flatMap((r) => [r.route, r.routeKey, now, nowMs, r.locale]);
     await db.exec(`
       INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at, locale)
-      VALUES (?, ?, '', '', '', '[]', '[]', ?, 0, 0, 0, 'runtime', ?, ?)
+      VALUES ${valuesSql}
       ON CONFLICT(route) DO UPDATE SET last_seen_at = excluded.last_seen_at, locale = excluded.locale
-    `, [route, routeKey, now, nowMs, locale]);
+    `, params);
   }
-  return routes.length;
+  return byRoute.size;
 }
 export async function getSitemapSeededAt(event) {
   const db = await getDb(event);
@@ -732,6 +741,16 @@ export async function getNextSitemapToCrawl(event, minIntervalMinutes = 5) {
   `, [threshold]);
   return row ? rowToSitemapEntry(row) : null;
 }
+export async function getSitemapLastCrawledAt(event, name) {
+  const db = await getDb(event);
+  if (!db)
+    return null;
+  const row = await db.first(
+    "SELECT last_crawled_at FROM ai_ready_sitemaps WHERE name = ?",
+    [name]
+  );
+  return row?.last_crawled_at ?? null;
+}
 export async function markSitemapCrawled(event, name, urlCount) {
   const db = await getDb(event);
   if (!db)

package/dist/runtime/server/plugins/sitemap-seeder.js

@@ -1,5 +1,58 @@
-import { getPageLastmods, markSitemapCrawled, seedRoutes } from "../db/queries.js";
+import { useRuntimeConfig } from "nitropack/runtime";
+import { getPageLastmods, getSitemapLastCrawledAt, markSitemapCrawled, seedRoutes } from "../db/queries.js";
 import { logger } from "../logger.js";
+const DIAGNOSTICS_KEY = "_aiReadySitemapDiagnostics";
+function isDebug(event) {
+  return !!useRuntimeConfig(event)["nuxt-ai-ready"]?.debug;
+}
+function recordDiagnostic(event, message) {
+  const ctx = event.context;
+  const list = ctx[DIAGNOSTICS_KEY] ??= [];
+  list.push(message);
+}
+const SEED_INTERVAL_MS = 5 * 60 * 1e3;
+const READ_TIMEOUT_MS = 3e3;
+const SLOW_READ_WARN_MS = 1e3;
+const SLOW_SEED_WARN_MS = 1e4;
+function withTimeout(promise, ms, label, fallback, onIssue) {
+  return new Promise((resolve) => {
+    let settled = false;
+    const timer = setTimeout(() => {
+      if (settled)
+        return;
+      settled = true;
+      const message = `${label} timed out after ${ms}ms`;
+      logger.warn(`[sitemap-seeder] ${message}`);
+      onIssue?.(message);
+      resolve(fallback);
+    }, ms);
+    const done = (value) => {
+      if (settled)
+        return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(value);
+    };
+    promise.then(done, (e) => {
+      const message = `${label} failed: ${e?.message}`;
+      logger.warn(`[sitemap-seeder] ${message}`);
+      onIssue?.(message);
+      done(fallback);
+    });
+  });
+}
+function toXmlComment(messages) {
+  const body = messages.map((m) => `  - ${m.replace(/-{2,}/g, "- ")}`).join("\n");
+  return `<!-- nuxt-ai-ready sitemap-seeder diagnostics:
+${body}
+-->
+`;
+}
+function injectComment(xml, comment) {
+  const prolog = xml.match(/^\s*(?:<\?[^>]*\?>\s*)*/);
+  const idx = prolog ? prolog[0].length : 0;
+  return xml.slice(0, idx) + comment + xml.slice(idx);
+}
 export default function sitemapSeederPlugin(nitroApp) {
   nitroApp.hooks.hook("sitemap:resolved", async (ctx) => {
     if (import.meta.dev)
@@ -7,26 +60,18 @@ export default function sitemapSeederPlugin(nitroApp) {
     const { urls, sitemapName, event } = ctx;
     if (urls.length === 0)
       return;
-    logger.debug(`[sitemap-seeder] Processing ${urls.length} routes from ${sitemapName}`);
-    const routes = [];
     const routeToUrl = /* @__PURE__ */ new Map();
     for (const u of urls) {
-      let route;
-      if (u._path?.pathname) {
-        route = u._path.pathname;
-      } else {
-        const loc = u.loc;
-        route = loc.startsWith("/") ? loc.split("?")[0] ?? loc : new URL(loc).pathname;
-      }
-      if (!route.includes(".")) {
-        routes.push(route);
+      const route = u._path?.pathname ?? (u.loc.startsWith("/") ? u.loc.split("?")[0] ?? u.loc : new URL(u.loc).pathname);
+      if (!route.includes("."))
         routeToUrl.set(route, u);
-      }
     }
-    const lastmods = await getPageLastmods(event).catch((e) => {
-      logger.warn(`[sitemap-seeder] Failed to get lastmods: ${e.message}`);
-      return /* @__PURE__ */ new Map();
-    });
+    if (routeToUrl.size === 0)
+      return;
+    logger.debug(`[sitemap-seeder] Processing ${routeToUrl.size} routes from ${sitemapName}`);
+    const record = isDebug(event) ? (m) => recordDiagnostic(event, m) : void 0;
+    const readStart = Date.now();
+    const lastmods = await withTimeout(getPageLastmods(event), READ_TIMEOUT_MS, "getPageLastmods", /* @__PURE__ */ new Map(), record);
     let enriched = 0;
     for (const [route, url] of routeToUrl) {
       const lastmod = lastmods.get(route);
@@ -38,15 +83,44 @@ export default function sitemapSeederPlugin(nitroApp) {
     if (enriched > 0) {
       logger.debug(`[sitemap-seeder] Enriched ${enriched} URLs with lastmod`);
     }
-    const seeded = await seedRoutes(event, routes).catch((e) => {
-      logger.warn(`[sitemap-seeder] Failed to seed routes: ${e.message}`);
-      return 0;
-    });
-    await markSitemapCrawled(event, sitemapName, urls.length).catch((e) => {
-      logger.warn(`[sitemap-seeder] Failed to mark sitemap: ${e.message}`);
-    });
-    if (seeded > 0) {
-      logger.debug(`[sitemap-seeder] Seeded ${seeded} new routes from ${sitemapName}`);
+    const lastCrawled = await withTimeout(getSitemapLastCrawledAt(event, sitemapName), READ_TIMEOUT_MS, "getSitemapLastCrawledAt", null, record);
+    const readMs = Date.now() - readStart;
+    if (readMs >= SLOW_READ_WARN_MS) {
+      const message = `slow DB reads: ${readMs}ms (timeout ${READ_TIMEOUT_MS}ms)`;
+      logger.warn(`[sitemap-seeder] ${message} for ${sitemapName}`);
+      record?.(message);
     }
+    if (lastCrawled && Date.now() - lastCrawled < SEED_INTERVAL_MS)
+      return;
+    const routes = [...routeToUrl.keys()];
+    const urlCount = urls.length;
+    const seed = async () => {
+      const seedStart = Date.now();
+      const seeded = await seedRoutes(event, routes).catch((e) => {
+        logger.warn(`[sitemap-seeder] Failed to seed routes: ${e.message}`);
+        return 0;
+      });
+      await markSitemapCrawled(event, sitemapName, urlCount).catch((e) => {
+        logger.warn(`[sitemap-seeder] Failed to mark sitemap: ${e.message}`);
+      });
+      const seedMs = Date.now() - seedStart;
+      if (seedMs >= SLOW_SEED_WARN_MS)
+        logger.warn(`[sitemap-seeder] Slow seed: ${seedMs}ms for ${seeded} routes in ${sitemapName}`);
+      else if (seeded > 0)
+        logger.debug(`[sitemap-seeder] Seeded ${seeded} routes from ${sitemapName} in ${seedMs}ms`);
+    };
+    if (event.waitUntil) {
+      event.waitUntil(seed().catch(
+        (err) => logger.error(`[sitemap-seeder] Background seed failed: ${err.message}`)
+      ));
+    } else {
+      await seed();
+    }
+  });
+  nitroApp.hooks.hook("sitemap:output", (ctx) => {
+    const diagnostics = ctx.event.context[DIAGNOSTICS_KEY];
+    if (!diagnostics?.length)
+      return;
+    ctx.sitemap = injectComment(ctx.sitemap, toXmlComment(diagnostics));
   });
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "1.5.1",
+  "version": "1.5.3",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -81,15 +81,15 @@
     "@nuxt/test-utils": "^4.0.3",
     "@nuxtjs/eslint-config-typescript": "^12.1.0",
     "@nuxtjs/mcp-toolkit": "^0.17.2",
-    "@nuxtjs/robots": "^6.1.1",
-    "@nuxtjs/sitemap": "^8.2.1",
+    "@nuxtjs/robots": "^6.1.2",
+    "@nuxtjs/sitemap": "^8.2.2",
     "@types/better-sqlite3": "^7.6.13",
     "@vitest/coverage-v8": "^4.1.9",
     "@vue/test-utils": "^2.4.11",
     "@vueuse/nuxt": "^14.3.0",
     "better-sqlite3": "^12.11.1",
     "bumpp": "^11.1.0",
-    "eslint": "^10.5.0",
+    "eslint": "^10.6.0",
     "eslint-plugin-harlanzw": "^0.17.0",
     "execa": "^9.6.1",
     "happy-dom": "^20.10.6",
@@ -103,10 +103,10 @@
     "typescript": "^6.0.3",
     "unbuild": "^3.6.1",
     "vitest": "^4.1.9",
-    "vue": "^3.5.38",
+    "vue": "^3.5.39",
     "vue-router": "^5.1.0",
     "vue-tsc": "^3.3.5",
-    "wrangler": "^4.104.0",
+    "wrangler": "^4.105.0",
     "zod": "^4.4.3"
   },
   "scripts": {