nuxt-ai-ready 0.7.15 → 0.8.0

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "0.7.15",
+  "version": "0.8.0",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -67,7 +67,121 @@ function hookNuxtSeoProLicense() {
   }
 }
 
-function createCrawlerState(dbPath, llmsFullTxtPath, siteInfo, llmsTxtConfig) {
+const INDEXNOW_HOSTS = ["api.indexnow.org", "www.bing.com"];
+function getIndexNowEndpoints() {
+  const testEndpoint = process.env.INDEXNOW_TEST_ENDPOINT;
+  if (testEndpoint) {
+    return [testEndpoint];
+  }
+  return INDEXNOW_HOSTS.map((host) => `https://${host}/indexnow`);
+}
+function buildIndexNowBody(routes, key, siteUrl) {
+  const urlList = routes.map(
+    (route) => route.startsWith("http") ? route : `${siteUrl}${route}`
+  );
+  return {
+    host: new URL(siteUrl).host,
+    key,
+    keyLocation: `${siteUrl}/${key}.txt`,
+    urlList
+  };
+}
+async function submitToIndexNow(routes, key, siteUrl, options) {
+  if (!siteUrl) {
+    return { success: false, error: "Site URL not configured" };
+  }
+  const fetchFn = options?.fetchFn ?? globalThis.fetch;
+  const log = options?.logger;
+  const body = buildIndexNowBody(routes, key, siteUrl);
+  const endpoints = getIndexNowEndpoints();
+  let lastError;
+  for (const endpoint of endpoints) {
+    log?.debug(`[indexnow] Submitting ${body.urlList.length} URLs to ${endpoint}`);
+    const response = await fetchFn(endpoint, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(body)
+    }).then((r) => r.ok ? { ok: true } : { error: `HTTP ${r.status}` }).catch((err) => ({ error: err.message }));
+    if ("error" in response) {
+      lastError = response.error;
+      if (lastError?.includes("429")) {
+        log?.warn(`[indexnow] Rate limited on ${endpoint}, trying fallback...`);
+        continue;
+      }
+      log?.warn(`[indexnow] Submission failed on ${endpoint}: ${lastError}`);
+      return { success: false, error: lastError, host: endpoint };
+    }
+    log?.debug(`[indexnow] Successfully submitted ${body.urlList.length} URLs via ${endpoint}`);
+    return { success: true, host: endpoint };
+  }
+  return {
+    success: false,
+    error: lastError || "All endpoints rate limited",
+    host: endpoints[endpoints.length - 1]
+  };
+}
+function comparePageHashes(currentPages, prevMeta) {
+  if (!prevMeta?.pages) {
+    return { changed: [], added: [], removed: [] };
+  }
+  const prevHashes = new Map(prevMeta.pages.map((p) => [p.route, p.hash]));
+  const currentRoutes = new Set(currentPages.map((p) => p.route));
+  const changed = [];
+  const added = [];
+  for (const page of currentPages) {
+    const prevHash = prevHashes.get(page.route);
+    if (!prevHash) {
+      added.push(page.route);
+    } else if (prevHash !== page.hash) {
+      changed.push(page.route);
+    }
+  }
+  const removed = [];
+  for (const route of prevHashes.keys()) {
+    if (!currentRoutes.has(route)) {
+      removed.push(route);
+    }
+  }
+  return { changed, added, removed };
+}
+
+async function fetchPreviousMeta(siteUrl, indexNowKey) {
+  const metaUrl = `${siteUrl}/__ai-ready/pages.meta.json`;
+  logger.debug(`[indexnow] Fetching previous meta from ${metaUrl}`);
+  const prevMeta = await fetch(metaUrl).then((r) => r.ok ? r.json() : null).catch(() => null);
+  if (!prevMeta?.pages) {
+    logger.info("[indexnow] First deploy or no previous meta - skipping IndexNow");
+    return null;
+  }
+  const keyUrl = `${siteUrl}/${indexNowKey}.txt`;
+  const keyLive = await fetch(keyUrl).then((r) => r.ok).catch(() => false);
+  if (!keyLive) {
+    logger.info("[indexnow] Key file not live yet - skipping IndexNow");
+    return null;
+  }
+  return prevMeta;
+}
+async function handleStaticIndexNow(currentPages, siteUrl, indexNowKey, prevMeta) {
+  const { changed, added } = comparePageHashes(currentPages, prevMeta);
+  const totalChanged = changed.length + added.length;
+  if (totalChanged === 0) {
+    logger.debug("[indexnow] No content changes detected");
+    return;
+  }
+  logger.info(`[indexnow] Submitting ${totalChanged} changed pages (${changed.length} modified, ${added.length} new)`);
+  const result = await submitToIndexNow(
+    [...changed, ...added],
+    indexNowKey,
+    siteUrl,
+    { logger }
+  );
+  if (result.success) {
+    logger.info(`[indexnow] Successfully notified search engines of ${totalChanged} changes`);
+  } else {
+    logger.warn(`[indexnow] Failed to submit: ${result.error}`);
+  }
+}
+function createCrawlerState(dbPath, llmsFullTxtPath, siteInfo, llmsTxtConfig, indexNowKey) {
   return {
     prerenderedRoutes: /* @__PURE__ */ new Set(),
     errorRoutes: /* @__PURE__ */ new Set(),
@@ -76,7 +190,8 @@ function createCrawlerState(dbPath, llmsFullTxtPath, siteInfo, llmsTxtConfig) {
     dbPath,
     llmsFullTxtPath,
     siteInfo,
-    llmsTxtConfig
+    llmsTxtConfig,
+    indexNowKey
   };
 }
 async function initCrawler(state) {
@@ -227,11 +342,11 @@ async function prerenderRoute(nitro, route) {
   nitro._prerenderedRoutes.push(_route);
   return stat(filePath);
 }
-function setupPrerenderHandler(dbPath, siteInfo, llmsTxtConfig) {
+function setupPrerenderHandler(dbPath, siteInfo, llmsTxtConfig, indexNowKey) {
   const nuxt = useNuxt();
   nuxt.hooks.hook("nitro:init", async (nitro) => {
     const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
-    const state = createCrawlerState(dbPath, llmsFullTxtPath, siteInfo, llmsTxtConfig);
+    const state = createCrawlerState(dbPath, llmsFullTxtPath, siteInfo, llmsTxtConfig, indexNowKey);
     let initPromise = null;
     nitro.hooks.hook("prerender:generate", async (route) => {
       if (route.error) {
@@ -294,6 +409,23 @@ function setupPrerenderHandler(dbPath, siteInfo, llmsTxtConfig) {
         const dumpPath = join(publicDataDir, "pages.dump");
         await writeFile(dumpPath, dumpData, "utf-8");
         logger.debug(`Created database dump at __ai-ready/pages.dump (${(dumpData.length / 1024).toFixed(1)}kb compressed)`);
+        const pageHashes = pages.filter((p) => p.contentHash).map((p) => ({ route: p.route, hash: p.contentHash }));
+        let prevMeta = null;
+        if (state.indexNowKey && state.siteInfo?.url) {
+          prevMeta = await fetchPreviousMeta(state.siteInfo.url, state.indexNowKey);
+        }
+        const buildId = Date.now().toString(36);
+        const metaContent = JSON.stringify({
+          buildId,
+          pageCount: pages.length,
+          createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+          pages: pageHashes
+        });
+        await writeFile(join(publicDataDir, "pages.meta.json"), metaContent, "utf-8");
+        logger.debug(`Wrote build metadata: buildId=${buildId}, ${pageHashes.length} page hashes`);
+        if (state.indexNowKey && state.siteInfo?.url && prevMeta) {
+          await handleStaticIndexNow(pageHashes, state.siteInfo.url, state.indexNowKey, prevMeta);
+        }
       }
       const llmsStats = await prerenderRoute(nitro, "/llms.txt");
       const llmsFullStats = await stat(state.llmsFullTxtPath);
@@ -666,9 +798,6 @@ export const errorRoutes = []`;
       runtimeSyncSecret: config.runtimeSyncSecret,
       indexNowKey
     };
-    nuxt.options.nitro.plugins = nuxt.options.nitro.plugins || [];
-    if (runtimeSyncEnabled)
-      nuxt.options.nitro.plugins.push(resolve("./runtime/server/plugins/sitemap-seeder"));
     addServerHandler({
       middleware: true,
       handler: resolve("./runtime/server/middleware/markdown.prerender")
@@ -721,7 +850,7 @@ export const errorRoutes = []`;
         name: siteConfig.name,
         url: siteConfig.url,
         description: siteConfig.description
-      }, mergedLlmsTxt);
+      }, mergedLlmsTxt, indexNowKey);
     }
     nuxt.options.nitro.routeRules = nuxt.options.nitro.routeRules || {};
     nuxt.options.nitro.routeRules["/llms.txt"] = { headers: { "Content-Type": "text/plain; charset=utf-8" } };

package/dist/runtime/index.d.ts

@@ -1,6 +1,6 @@
 export { useDatabase } from './server/db/index.js';
 export type { DatabaseAdapter } from './server/db/index.js';
-export { countPages, getSitemapSeededAt, getStaleRoutes, isPageFresh, pruneStaleRoutes, queryPages, searchPages, seedRoutes, setSitemapSeededAt, streamPages, upsertPage, } from './server/db/queries.js';
+export { countPages, getStaleRoutes, isPageFresh, pruneStaleRoutes, queryPages, searchPages, seedRoutes, streamPages, upsertPage, } from './server/db/queries.js';
 export type { CountPagesOptions, PageData, PageEntry, PageRow, QueryPagesOptions, SearchPagesOptions, SearchResult, StreamPagesOptions, UpsertPageInput, } from './server/db/queries.js';
 export { indexPage, indexPageByRoute } from './server/utils/indexPage.js';
 export type { IndexPageOptions, IndexPageResult } from './server/utils/indexPage.js';

package/dist/runtime/index.js

@@ -1,14 +1,12 @@
 export { useDatabase } from "./server/db/index.js";
 export {
   countPages,
-  getSitemapSeededAt,
   getStaleRoutes,
   isPageFresh,
   pruneStaleRoutes,
   queryPages,
   searchPages,
   seedRoutes,
-  setSitemapSeededAt,
   streamPages,
   upsertPage
 } from "./server/db/queries.js";

package/dist/runtime/server/db/queries.d.ts

@@ -154,6 +154,11 @@ export declare function markIndexNowSynced(event: H3Event | undefined, routes: s
  * Uses atomic SQL to handle concurrent updates safely
  */
 export declare function updateIndexNowStats(event: H3Event | undefined, submitted: number, error?: string): Promise<void>;
+/**
+ * Batch update for IndexNow: mark pages synced + update stats
+ * Runs all queries in parallel for speed
+ */
+export declare function batchIndexNowUpdate(event: H3Event | undefined, routes: string[], submitted: number): Promise<void>;
 export interface IndexNowStats {
     totalSubmitted: number;
     lastSubmittedAt: number | null;

package/dist/runtime/server/db/queries.js

@@ -363,6 +363,32 @@ export async function updateIndexNowStats(event, submitted, error) {
     );
   }
 }
+export async function batchIndexNowUpdate(event, routes, submitted) {
+  const db = await getDb(event);
+  if (!db || routes.length === 0)
+    return;
+  const now = Date.now();
+  const placeholders = routes.map(() => "?").join(",");
+  await Promise.all([
+    // Mark pages as synced
+    db.exec(
+      `UPDATE ai_ready_pages SET indexnow_synced_at = ? WHERE route IN (${placeholders})`,
+      [now, ...routes]
+    ),
+    // Atomic increment total submitted
+    db.exec(`
+      INSERT INTO _ai_ready_info (id, value) VALUES ('indexnow_total_submitted', ?)
+      ON CONFLICT(id) DO UPDATE SET value = CAST((CAST(value AS INTEGER) + ?) AS TEXT)
+    `, [String(submitted), submitted]),
+    // Update last submitted timestamp
+    db.exec(
+      "INSERT OR REPLACE INTO _ai_ready_info (id, value) VALUES (?, ?)",
+      ["indexnow_last_submitted_at", String(now)]
+    ),
+    // Clear any previous error
+    db.exec("DELETE FROM _ai_ready_info WHERE id = ?", ["indexnow_last_error"])
+  ]);
+}
 export async function getIndexNowStats(event) {
   const db = await getDb(event);
   if (!db)

package/dist/runtime/server/db/shared.d.ts

@@ -13,7 +13,8 @@ export interface DatabaseAdapter {
  */
 export declare function createAdapter(connector: Connector): DatabaseAdapter;
 /**
- * Initialize database schema with version checking and lazy restore
+ * Initialize database schema with version checking
+ * Note: Restore logic moved to cron for reliability on Cloudflare
  */
 export declare function initSchema(db: DatabaseAdapter): Promise<void>;
 /**
@@ -98,6 +99,7 @@ export interface DumpRow {
     indexed: number;
     source: string;
     last_seen_at: number | null;
+    indexnow_synced_at: number | null;
 }
 /**
  * Export database as compressed dump (base64 gzip) using batched streaming
@@ -106,5 +108,7 @@ export interface DumpRow {
 export declare function exportDbDump(db: DatabaseAdapter): Promise<string>;
 /**
  * Import dump into database
+ * Sets indexed=1, last_seen_at=indexed_at, indexnow_synced_at=indexed_at
+ * (pages from dump were already indexed and synced during build, no need to re-notify)
  */
 export declare function importDbDump(db: DatabaseAdapter, rows: DumpRow[]): Promise<void>;

package/dist/runtime/server/db/shared.js

@@ -35,33 +35,6 @@ export async function initSchema(db) {
     "INSERT OR REPLACE INTO _ai_ready_info (id, version) VALUES (?, ?)",
     ["schema", SCHEMA_VERSION]
   );
-  if (!import.meta.prerender && !import.meta.dev) {
-    await restoreFromDumpIfEmpty(db);
-  }
-}
-async function restoreFromDumpIfEmpty(db) {
-  const row = await db.first("SELECT COUNT(*) as count FROM ai_ready_pages");
-  if (row && row.count > 0)
-    return;
-  try {
-    let dumpData = null;
-    const cfEnv = globalThis.__env__;
-    if (cfEnv?.ASSETS) {
-      const response = await cfEnv.ASSETS.fetch("https://placeholder/__ai-ready/pages.dump");
-      if (response.ok)
-        dumpData = await response.text();
-    }
-    if (!dumpData) {
-      dumpData = await globalThis.$fetch("/__ai-ready/pages.dump", {
-        responseType: "text"
-      });
-    }
-    if (!dumpData)
-      return;
-    const rows = await decompressFromBase64(dumpData);
-    await importDbDump(db, rows);
-  } catch {
-  }
 }
 async function checkSchemaVersion(db) {
   const info = await db.first(
@@ -147,7 +120,7 @@ export async function exportDbDump(db) {
   let first = true;
   while (true) {
     const rows = await db.all(`
-      SELECT route, route_key, title, description, markdown, headings, keywords, content_hash, updated_at, indexed_at, is_error, indexed, source, last_seen_at
+      SELECT route, route_key, title, description, markdown, headings, keywords, content_hash, updated_at, indexed_at, is_error, indexed, source, last_seen_at, indexnow_synced_at
       FROM ai_ready_pages
       ORDER BY route
       LIMIT ${DUMP_BATCH_SIZE} OFFSET ${offset}
@@ -179,8 +152,8 @@ export async function importDbDump(db, rows) {
   for (const row of rows) {
     const source = row.source || "prerender";
     await db.exec(`
-      INSERT OR REPLACE INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, content_hash, updated_at, indexed_at, is_error, indexed, source, last_seen_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, NULL)
-    `, [row.route, row.route_key, row.title, row.description, row.markdown, row.headings, row.keywords, row.content_hash || null, row.updated_at, row.indexed_at, row.is_error, source]);
+      INSERT OR REPLACE INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, content_hash, updated_at, indexed_at, is_error, indexed, source, last_seen_at, indexnow_synced_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)
+    `, [row.route, row.route_key, row.title, row.description, row.markdown, row.headings, row.keywords, row.content_hash || null, row.updated_at, row.indexed_at, row.is_error, source, row.indexed_at, row.indexed_at]);
   }
 }

package/dist/runtime/server/routes/__ai-ready/indexnow.post.js

@@ -14,5 +14,5 @@ export default eventHandler(async (event) => {
     }
   }
   const limit = query.limit ? Number(query.limit) : 100;
-  return syncToIndexNow(event, limit);
+  return syncToIndexNow(event, limit, { useWaitUntil: true });
 });

package/dist/runtime/server/routes/__ai-ready-debug.get.d.ts

@@ -26,13 +26,18 @@ interface DebugInfo {
         total: number;
         indexed: number;
         pending: number;
-        sitemapSeededAt: string | null;
+        errors: number;
     };
     indexNow?: {
         pending: number;
         totalSubmitted: number;
         lastSubmittedAt: string | null;
         lastError: string | null;
+        backoff?: {
+            until: string;
+            minutesRemaining: number;
+            attempt: number;
+        } | null;
     };
     indexNowLog?: Array<{
         id: number;
@@ -58,6 +63,13 @@ interface DebugInfo {
         pageCount: number;
         source: string;
     };
+    buildInfo?: {
+        storedBuildId: string | null;
+        dumpBuildId: string | null;
+        dumpPageCount: number | null;
+        isStale: boolean;
+        dumpCreatedAt: string | null;
+    };
     virtualModules: {
         pageDataModule: {
             available: boolean;

package/dist/runtime/server/routes/__ai-ready-debug.get.js

@@ -1,6 +1,7 @@
 import { createError, eventHandler, setHeader } from "h3";
 import { useRuntimeConfig } from "nitropack/runtime";
-import { countPages, countPagesNeedingIndexNowSync, getIndexNowLog, getIndexNowStats, getRecentCronRuns, getSitemapSeededAt, queryPages } from "../db/queries.js";
+import { useDatabase } from "../db/index.js";
+import { countPages, countPagesNeedingIndexNowSync, getIndexNowLog, getIndexNowStats, getRecentCronRuns, queryPages } from "../db/queries.js";
 export default eventHandler(async (event) => {
   const runtimeConfig = useRuntimeConfig(event)["nuxt-ai-ready"];
   if (!runtimeConfig.debug) {
@@ -107,19 +108,20 @@ export default eventHandler(async (event) => {
   let indexNowInfo;
   let indexNowLogInfo;
   let cronRunsInfo;
+  let buildInfo;
   if (!isDev && !isPrerender && !dbError) {
     try {
-      const [total, pending, sitemapSeededAt, cronRuns] = await Promise.all([
+      const [total, pending, errors, cronRuns] = await Promise.all([
         countPages(event),
         countPages(event, { where: { pending: true } }),
-        getSitemapSeededAt(event),
+        countPages(event, { where: { hasError: true } }),
         getRecentCronRuns(event, 20)
       ]);
       runtimeSyncInfo = {
         total,
         indexed: total - pending,
         pending,
-        sitemapSeededAt: sitemapSeededAt ? new Date(sitemapSeededAt).toISOString() : null
+        errors
       };
       cronRunsInfo = cronRuns.map((run) => ({
         id: run.id,
@@ -139,11 +141,26 @@ export default eventHandler(async (event) => {
           getIndexNowStats(event),
           getIndexNowLog(event, 20)
         ]);
+        const db2 = await useDatabase(event);
+        const backoffRow = await db2.first("SELECT value FROM _ai_ready_info WHERE id = ?", ["indexnow_backoff"]);
+        let backoffInfo = null;
+        if (backoffRow) {
+          const parsed = JSON.parse(backoffRow.value);
+          const now = Date.now();
+          if (parsed.until > now) {
+            backoffInfo = {
+              until: new Date(parsed.until).toISOString(),
+              minutesRemaining: Math.ceil((parsed.until - now) / 6e4),
+              attempt: parsed.attempt
+            };
+          }
+        }
         indexNowInfo = {
           pending: indexNowPending,
           totalSubmitted: indexNowStats.totalSubmitted,
           lastSubmittedAt: indexNowStats.lastSubmittedAt ? new Date(indexNowStats.lastSubmittedAt).toISOString() : null,
-          lastError: indexNowStats.lastError
+          lastError: indexNowStats.lastError,
+          backoff: backoffInfo
         };
         indexNowLogInfo = indexNowLogEntries.map((entry) => ({
           id: entry.id,
@@ -153,6 +170,33 @@ export default eventHandler(async (event) => {
           error: entry.error
         }));
       }
+      const db = await useDatabase(event);
+      const storedRow = await db.first("SELECT value FROM _ai_ready_info WHERE id = ?", ["build_id"]);
+      const storedBuildId = storedRow?.value || null;
+      let dumpMeta = null;
+      const cfEnv2 = event.context?.cloudflare?.env;
+      if (cfEnv2?.ASSETS?.fetch) {
+        const metaResponse = await cfEnv2.ASSETS.fetch(new Request("https://assets.local/__ai-ready/pages.meta.json")).catch(() => null);
+        if (metaResponse?.ok)
+          dumpMeta = await metaResponse.json().catch(() => null);
+      }
+      if (!dumpMeta) {
+        dumpMeta = await globalThis.$fetch("/__ai-ready/pages.meta.json").catch(() => null);
+      }
+      buildInfo = {
+        storedBuildId,
+        dumpBuildId: dumpMeta?.buildId || null,
+        dumpPageCount: dumpMeta?.pageCount || null,
+        isStale: dumpMeta ? storedBuildId !== dumpMeta.buildId : false,
+        dumpCreatedAt: dumpMeta?.createdAt || null
+      };
+      if (buildInfo.isStale) {
+        issues.push(`Build ID mismatch: DB has "${storedBuildId || "none"}", dump has "${dumpMeta?.buildId}"`);
+        suggestions.push("Cron will mark pages pending on next run, or manually trigger /__ai-ready/cron");
+      } else if (!storedBuildId && dumpMeta) {
+        issues.push("No build ID stored in DB - data may need restore");
+        suggestions.push("Wait for cron to run, or manually trigger /__ai-ready/cron");
+      }
     } catch (err) {
       issues.push(`Runtime stats error: ${err.message || String(err)}`);
     }
@@ -186,6 +230,7 @@ export default eventHandler(async (event) => {
     indexNow: indexNowInfo,
     indexNowLog: indexNowLogInfo,
     cronRuns: cronRunsInfo,
+    buildInfo,
     pageData: {
       source,
       pageCount: pages.length,

package/dist/runtime/server/utils/checkStale.d.ts

@@ -0,0 +1,20 @@
+import type { H3Event } from 'h3';
+export interface BuildMeta {
+    buildId: string;
+    pageCount: number;
+    createdAt: string;
+}
+export interface StaleCheckResult {
+    action: 'none' | 'restored' | 'marked_pending';
+    buildId?: string;
+    dbCount: number;
+    dumpCount?: number;
+    reason?: string;
+}
+/**
+ * Check if data is stale and handle restore/mark-pending
+ * Called at start of cron - handles:
+ * 1. Empty DB → restore from dump
+ * 2. Build ID changed → mark all pages pending for recheck
+ */
+export declare function checkAndHandleStale(event: H3Event): Promise<StaleCheckResult>;

package/dist/runtime/server/utils/checkStale.js

@@ -0,0 +1,100 @@
+import { useRuntimeConfig } from "nitropack/runtime";
+import { useDatabase } from "../db/index.js";
+import { countPages } from "../db/queries.js";
+import { decompressFromBase64, importDbDump } from "../db/shared.js";
+import { logger } from "../logger.js";
+async function fetchBuildMeta(event) {
+  const cfEnv = event.context?.cloudflare?.env;
+  if (cfEnv?.ASSETS?.fetch) {
+    const response = await cfEnv.ASSETS.fetch(new Request("https://assets.local/__ai-ready/pages.meta.json")).catch(() => null);
+    if (response?.ok) {
+      return response.json().catch(() => null);
+    }
+  }
+  return globalThis.$fetch("/__ai-ready/pages.meta.json").catch(() => null);
+}
+async function fetchDump(event) {
+  const cfEnv = event.context?.cloudflare?.env;
+  let dumpData = null;
+  if (cfEnv?.ASSETS?.fetch) {
+    const response = await cfEnv.ASSETS.fetch(new Request("https://assets.local/__ai-ready/pages.dump")).catch(() => null);
+    if (response?.ok) {
+      dumpData = await response.text();
+    }
+  }
+  if (!dumpData) {
+    dumpData = await globalThis.$fetch("/__ai-ready/pages.dump", { responseType: "text" }).catch(() => null);
+  }
+  if (!dumpData)
+    return null;
+  return decompressFromBase64(dumpData);
+}
+async function getStoredBuildId(event) {
+  const db = await useDatabase(event);
+  const row = await db.first("SELECT value FROM _ai_ready_info WHERE id = ?", ["build_id"]);
+  return row?.value || null;
+}
+async function setStoredBuildId(event, buildId) {
+  const db = await useDatabase(event);
+  await db.exec("INSERT OR REPLACE INTO _ai_ready_info (id, value) VALUES (?, ?)", ["build_id", buildId]);
+}
+async function markAllPagesPending(event) {
+  const db = await useDatabase(event);
+  await db.exec("UPDATE ai_ready_pages SET indexed = 0 WHERE is_error = 0");
+  return countPages(event, { where: { pending: true } });
+}
+export async function checkAndHandleStale(event) {
+  const config = useRuntimeConfig()["nuxt-ai-ready"];
+  const debug = config.debug;
+  const dbCount = await countPages(event);
+  const storedBuildId = await getStoredBuildId(event);
+  const meta = await fetchBuildMeta(event);
+  if (debug) {
+    logger.info(`[stale-check] DB count: ${dbCount}, stored buildId: ${storedBuildId || "none"}, dump buildId: ${meta?.buildId || "none"}`);
+  }
+  if (!meta) {
+    if (debug)
+      logger.info("[stale-check] No build metadata found, skipping stale check");
+    return { action: "none", dbCount, reason: "no_dump_metadata" };
+  }
+  if (dbCount === 0) {
+    if (debug)
+      logger.info("[stale-check] DB empty, restoring from dump...");
+    const rows = await fetchDump(event);
+    if (!rows) {
+      logger.warn("[stale-check] Failed to fetch dump for restore");
+      return { action: "none", dbCount: 0, dumpCount: meta.pageCount, reason: "dump_fetch_failed" };
+    }
+    const db = await useDatabase(event);
+    await importDbDump(db, rows);
+    await setStoredBuildId(event, meta.buildId);
+    logger.info(`[stale-check] Restored ${rows.length} pages from dump (buildId: ${meta.buildId})`);
+    return {
+      action: "restored",
+      buildId: meta.buildId,
+      dbCount: rows.length,
+      dumpCount: meta.pageCount
+    };
+  }
+  if (storedBuildId !== meta.buildId) {
+    if (debug)
+      logger.info(`[stale-check] Build ID changed (${storedBuildId} \u2192 ${meta.buildId}), marking pages pending...`);
+    const pendingCount = await markAllPagesPending(event);
+    await setStoredBuildId(event, meta.buildId);
+    logger.info(`[stale-check] Marked ${pendingCount} pages as pending for recheck (buildId: ${meta.buildId})`);
+    return {
+      action: "marked_pending",
+      buildId: meta.buildId,
+      dbCount,
+      dumpCount: meta.pageCount
+    };
+  }
+  if (debug)
+    logger.info("[stale-check] Build ID unchanged, no action needed");
+  return {
+    action: "none",
+    buildId: meta.buildId,
+    dbCount,
+    dumpCount: meta.pageCount
+  };
+}

package/dist/runtime/server/utils/indexnow.d.ts

@@ -1,4 +1,5 @@
 import type { H3Event } from 'h3';
+export type { BuildMeta, IndexNowSubmitResult, PageHashMeta } from '../../../shared/indexnow.js';
 export interface IndexNowResult {
     success: boolean;
     submitted: number;
@@ -7,18 +8,23 @@ export interface IndexNowResult {
     backoff?: boolean;
 }
 /**
- * Submit URLs to IndexNow API
+ * Submit URLs to IndexNow API with fallback on rate limit
+ * Wrapper around shared implementation with runtime-specific fetch
  */
 export declare function submitToIndexNow(routes: string[], config: {
     key: string;
-    host?: string;
 }, siteUrl: string): Promise<{
     success: boolean;
     error?: string;
+    host?: string;
 }>;
+export interface SyncToIndexNowOptions {
+    /** Use waitUntil for background DB updates (default: false in cron, true otherwise) */
+    useWaitUntil?: boolean;
+}
 /**
  * Submit pending pages to IndexNow
  * Queries DB for pages needing sync, submits, marks as synced
  * Implements exponential backoff on 429 rate limit errors
  */
-export declare function syncToIndexNow(event: H3Event | undefined, limit?: number): Promise<IndexNowResult>;
+export declare function syncToIndexNow(event: H3Event | undefined, limit?: number, options?: SyncToIndexNowOptions): Promise<IndexNowResult>;

package/dist/runtime/server/utils/indexnow.js

@@ -1,11 +1,12 @@
 import { getSiteConfig } from "#site-config/server/composables";
 import { useRuntimeConfig } from "nitropack/runtime";
+import { submitToIndexNow as submitToIndexNowShared } from "../../../shared/indexnow";
 import { useDatabase } from "../db/index.js";
 import {
+  batchIndexNowUpdate,
   countPagesNeedingIndexNowSync,
   getPagesNeedingIndexNowSync,
   logIndexNowSubmission,
-  markIndexNowSynced,
   updateIndexNowStats
 } from "../db/queries.js";
 import { logger } from "../logger.js";
@@ -31,34 +32,25 @@ async function setBackoffInfo(event, info) {
   }
 }
 export async function submitToIndexNow(routes, config, siteUrl) {
-  if (!siteUrl) {
-    return { success: false, error: "Site URL not configured" };
-  }
-  const host = config.host || "api.indexnow.org";
-  const endpoint = `https://${host}/indexnow`;
-  const urlList = routes.map(
-    (route) => route.startsWith("http") ? route : `${siteUrl}${route}`
-  );
-  const body = {
-    host: new URL(siteUrl).host,
-    key: config.key,
-    keyLocation: `${siteUrl}/${config.key}.txt`,
-    urlList
+  const runtimeFetch = async (input, init) => {
+    const url = typeof input === "string" ? input : input.toString();
+    const body = init?.body ? JSON.parse(init.body) : void 0;
+    const result = await $fetch.raw(url, {
+      method: init?.method,
+      headers: init?.headers,
+      body
+    }).catch((err) => ({ _error: err.message }));
+    if (result && "_error" in result) {
+      return { ok: false, status: 500, statusText: result._error };
+    }
+    return { ok: result.status >= 200 && result.status < 300, status: result.status };
   };
-  logger.debug(`[indexnow] Submitting ${urlList.length} URLs to ${endpoint}`);
-  const response = await $fetch(endpoint, {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body
-  }).catch((err) => ({ error: err.message }));
-  if (response && typeof response === "object" && "error" in response) {
-    logger.warn(`[indexnow] Submission failed: ${response.error}`);
-    return { success: false, error: response.error };
-  }
-  logger.debug(`[indexnow] Successfully submitted ${urlList.length} URLs`);
-  return { success: true };
+  return submitToIndexNowShared(routes, config.key, siteUrl, {
+    fetchFn: runtimeFetch,
+    logger
+  });
 }
-export async function syncToIndexNow(event, limit = 100) {
+export async function syncToIndexNow(event, limit = 100, options) {
   const config = useRuntimeConfig(event)["nuxt-ai-ready"];
   const siteConfig = getSiteConfig(event);
   if (!config.indexNowKey) {
@@ -71,43 +63,55 @@ export async function syncToIndexNow(event, limit = 100) {
   if (backoff && Date.now() < backoff.until) {
     const waitMinutes = Math.ceil((backoff.until - Date.now()) / 6e4);
     logger.debug(`[indexnow] In backoff period, ${waitMinutes}m remaining`);
-    const remaining2 = await countPagesNeedingIndexNowSync(event);
+    const remaining = await countPagesNeedingIndexNowSync(event);
     return {
       success: false,
       submitted: 0,
-      remaining: remaining2,
+      remaining,
       error: `Rate limited, retry in ${waitMinutes}m`,
       backoff: true
     };
   }
-  const pages = await getPagesNeedingIndexNowSync(event, limit);
+  const [totalPending, pages] = await Promise.all([
+    countPagesNeedingIndexNowSync(event),
+    getPagesNeedingIndexNowSync(event, limit)
+  ]);
   if (pages.length === 0) {
     return { success: true, submitted: 0, remaining: 0 };
   }
   const routes = pages.map((p) => p.route);
   const result = await submitToIndexNow(routes, { key: config.indexNowKey }, siteConfig.url);
-  if (config.debug) {
-    await logIndexNowSubmission(event, routes.length, result.success, result.error);
-  }
-  if (result.success) {
-    await setBackoffInfo(event, null);
-    await markIndexNowSynced(event, routes);
-    await updateIndexNowStats(event, routes.length);
-  } else {
-    await updateIndexNowStats(event, 0, result.error);
-    if (result.error?.includes("429")) {
-      const attempt = backoff ? Math.min(backoff.attempt + 1, BACKOFF_MINUTES.length - 1) : 0;
-      const backoffMinutes = BACKOFF_MINUTES[attempt] ?? 60;
-      const until = Date.now() + backoffMinutes * 60 * 1e3;
-      await setBackoffInfo(event, { until, attempt });
-      logger.warn(`[indexnow] Rate limited, backing off for ${backoffMinutes}m (attempt ${attempt + 1})`);
+  const dbUpdates = async () => {
+    if (config.debug) {
+      await logIndexNowSubmission(event, routes.length, result.success, result.error);
+    }
+    if (result.success) {
+      await setBackoffInfo(event, null);
+      await batchIndexNowUpdate(event, routes, routes.length);
+      logger.debug(`[indexnow] DB updated: ${routes.length} pages marked synced via ${result.host}`);
+    } else {
+      await updateIndexNowStats(event, 0, result.error);
+      if (result.error?.includes("429")) {
+        const attempt = backoff ? Math.min(backoff.attempt + 1, BACKOFF_MINUTES.length - 1) : 0;
+        const backoffMinutes = BACKOFF_MINUTES[attempt] ?? 60;
+        const until = Date.now() + backoffMinutes * 60 * 1e3;
+        await setBackoffInfo(event, { until, attempt });
+        logger.warn(`[indexnow] Rate limited, backing off for ${backoffMinutes}m (attempt ${attempt + 1})`);
+      }
     }
+  };
+  if (options?.useWaitUntil && event?.waitUntil) {
+    event.waitUntil(dbUpdates().catch(
+      (err) => logger.error(`[indexnow] Background DB update failed: ${err.message}`)
+    ));
+  } else {
+    await dbUpdates();
   }
-  const remaining = await countPagesNeedingIndexNowSync(event);
+  const submitted = result.success ? routes.length : 0;
   return {
     success: result.success,
-    submitted: result.success ? routes.length : 0,
-    remaining,
+    submitted,
+    remaining: Math.max(0, totalPending - submitted),
     error: result.error,
     backoff: !result.success && result.error?.includes("429")
   };

package/dist/runtime/server/utils/runCron.d.ts

@@ -1,6 +1,12 @@
 import type { H3Event } from 'h3';
+import type { StaleCheckResult } from './checkStale.js';
 export interface CronResult {
     runId?: number | null;
+    stale?: StaleCheckResult;
+    sitemap?: {
+        seeded: number;
+        pruned: number;
+    };
     index?: {
         indexed: number;
         remaining: number;

package/dist/runtime/server/utils/runCron.js

@@ -1,7 +1,10 @@
 import { useEvent, useRuntimeConfig } from "nitropack/runtime";
-import { cleanupOldCronRuns, completeCronRun, startCronRun } from "../db/queries.js";
+import { cleanupOldCronRuns, completeCronRun, pruneStaleRoutes, seedRoutes, startCronRun } from "../db/queries.js";
+import { logger } from "../logger.js";
 import { batchIndexPages } from "./batchIndex.js";
+import { checkAndHandleStale } from "./checkStale.js";
 import { syncToIndexNow } from "./indexnow.js";
+import { fetchSitemapUrls } from "./sitemap.js";
 function getEvent(providedEvent) {
   if (providedEvent)
     return providedEvent;
@@ -20,8 +23,34 @@ export async function runCron(providedEvent, options) {
     return {};
   }
   const config = useRuntimeConfig()["nuxt-ai-ready"];
+  const debug = config.debug;
+  const startTime = Date.now();
   const results = {};
   const allErrors = [];
+  if (debug) {
+    logger.info(`[cron] Starting cron run (batchSize: ${options?.batchSize ?? config.runtimeSync.batchSize}, indexNow: ${!!config.indexNowKey})`);
+  }
+  if (config.runtimeSync.enabled) {
+    results.stale = await checkAndHandleStale(event).catch((err) => {
+      console.warn("[ai-ready:cron] Stale check failed:", err.message);
+      allErrors.push(`stale-check: ${err.message}`);
+      return { action: "none", dbCount: 0, reason: err.message };
+    });
+    if (debug && results.stale) {
+      logger.info(`[cron] Stale check: ${results.stale.action} (db: ${results.stale.dbCount}, dump: ${results.stale.dumpCount ?? "n/a"})`);
+    }
+  }
+  if (config.runtimeSync.enabled) {
+    const sitemapResult = await seedFromSitemap(event, config, debug).catch((err) => {
+      console.warn("[ai-ready:cron] Sitemap seeding failed:", err.message);
+      allErrors.push(`sitemap: ${err.message}`);
+      return { seeded: 0, pruned: 0 };
+    });
+    results.sitemap = sitemapResult;
+    if (debug && (sitemapResult.seeded > 0 || sitemapResult.pruned > 0)) {
+      logger.info(`[cron] Sitemap: seeded ${sitemapResult.seeded}, pruned ${sitemapResult.pruned}`);
+    }
+  }
   const runId = await startCronRun(event);
   results.runId = runId;
   if (config.runtimeSync.enabled) {
@@ -39,6 +68,9 @@ export async function runCron(providedEvent, options) {
     if (indexResult.errors.length > 0) {
       allErrors.push(...indexResult.errors);
     }
+    if (debug) {
+      logger.info(`[cron] Index: ${indexResult.indexed} pages (${indexResult.remaining} remaining${indexResult.errors.length > 0 ? `, ${indexResult.errors.length} errors` : ""})`);
+    }
   }
   if (config.indexNowKey) {
     const indexNowResult = await syncToIndexNow(event, 100).catch((err) => {
@@ -53,6 +85,10 @@ export async function runCron(providedEvent, options) {
     if (indexNowResult.error) {
       allErrors.push(`IndexNow: ${indexNowResult.error}`);
     }
+    if (debug) {
+      const status = indexNowResult.error ? `error: ${indexNowResult.error}` : `${indexNowResult.submitted} submitted (${indexNowResult.remaining} remaining)`;
+      logger.info(`[cron] IndexNow: ${status}`);
+    }
   }
   if (runId) {
     await completeCronRun(event, runId, {
@@ -64,5 +100,39 @@ export async function runCron(providedEvent, options) {
     });
     await cleanupOldCronRuns(event, 50);
   }
+  if (debug) {
+    const duration = Date.now() - startTime;
+    const parts = [];
+    if (results.stale?.action !== "none")
+      parts.push(results.stale?.action);
+    if (results.index?.indexed)
+      parts.push(`${results.index.indexed} indexed`);
+    if (results.indexNow?.submitted)
+      parts.push(`${results.indexNow.submitted} submitted to IndexNow`);
+    if (allErrors.length > 0)
+      parts.push(`${allErrors.length} errors`);
+    logger.info(`[cron] Complete in ${duration}ms${parts.length > 0 ? `: ${parts.join(", ")}` : ""}`);
+  }
   return results;
 }
+async function seedFromSitemap(event, config, debug) {
+  const { pruneTtl } = config.runtimeSync;
+  const urls = await fetchSitemapUrls(event);
+  if (urls.length === 0) {
+    if (debug)
+      logger.info("[cron] Sitemap: no URLs found");
+    return { seeded: 0, pruned: 0 };
+  }
+  if (debug)
+    logger.info(`[cron] Sitemap: found ${urls.length} URLs`);
+  const routes = urls.map((u) => {
+    const url = new URL(u.loc);
+    return url.pathname;
+  }).filter((route) => !route.includes("."));
+  const seeded = await seedRoutes(event, routes);
+  let pruned = 0;
+  if (pruneTtl > 0) {
+    pruned = await pruneStaleRoutes(event, pruneTtl);
+  }
+  return { seeded, pruned };
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "0.7.15",
+  "version": "0.8.0",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",

package/dist/runtime/server/plugins/sitemap-seeder.d.ts

@@ -1,2 +0,0 @@
-declare const _default: import("nitropack/types").NitroAppPlugin;
-export default _default;

package/dist/runtime/server/plugins/sitemap-seeder.js

@@ -1,58 +0,0 @@
-import { defineNitroPlugin, useRuntimeConfig } from "nitropack/runtime";
-import { getSitemapSeededAt, pruneStaleRoutes, seedRoutes, setSitemapSeededAt } from "../db/queries.js";
-import { logger } from "../logger.js";
-import { fetchSitemapUrls } from "../utils/sitemap.js";
-let seeding = null;
-export default defineNitroPlugin((nitro) => {
-  if (import.meta.prerender)
-    return;
-  nitro.hooks.hook("request", async (event) => {
-    if (!seeding) {
-      seeding = seedFromSitemap(event).catch((err) => {
-        if (err.message?.includes("D1 binding") || err.message?.includes("[d1]")) {
-          const config = useRuntimeConfig()["nuxt-ai-ready"];
-          if (config.debug) {
-            const debugInfo = {
-              hasGlobalEnv: !!globalThis.__env__,
-              globalEnvKeys: globalThis.__env__ ? Object.keys(globalThis.__env__) : [],
-              hasCloudflareContext: !!event.context?.cloudflare,
-              error: err.message
-            };
-            logger.info("[sitemap-seeder] D1 binding debug:", JSON.stringify(debugInfo, null, 2));
-          }
-          logger.info("[sitemap-seeder] Skipping runtime seeding - using static pages data");
-        } else {
-          logger.error("[sitemap-seeder] Failed to seed:", err);
-        }
-      });
-    }
-  });
-});
-async function seedFromSitemap(event) {
-  const config = useRuntimeConfig()["nuxt-ai-ready"];
-  const { ttl, pruneTtl } = config.runtimeSync;
-  const seededAt = await getSitemapSeededAt(event);
-  if (seededAt && ttl > 0) {
-    const age = (Date.now() - seededAt) / 1e3;
-    if (age < ttl) {
-      logger.debug(`[sitemap-seeder] Sitemap fresh (${Math.round(age)}s old), skipping`);
-      return;
-    }
-  }
-  const urls = await fetchSitemapUrls(event);
-  if (urls.length === 0) {
-    return;
-  }
-  const routes = urls.map((u) => {
-    const url = new URL(u.loc);
-    return url.pathname;
-  }).filter((route) => !route.includes("."));
-  await seedRoutes(event, routes);
-  await setSitemapSeededAt(event, Date.now());
-  logger.info(`[sitemap-seeder] Seeded ${routes.length} routes from sitemap`);
-  if (pruneTtl > 0) {
-    const pruned = await pruneStaleRoutes(event, pruneTtl);
-    if (pruned > 0)
-      logger.info(`[sitemap-seeder] Pruned ${pruned} stale routes`);
-  }
-}