nuxt-ai-ready 2.5.1 → 2.5.3

package/dist/cli.mjs

@@ -320,6 +320,8 @@ const main = defineCommand({
           }
         } else if (res.skipped) {
           consola.info(`Skipped: ${colors.yellow(res.route)} (still fresh)`);
+        } else if (res.gone) {
+          consola.success(`Removed: ${colors.green(res.route)} (HTTP 410)`);
         } else {
           throw new Error(`Failed to reindex ${res.route ?? args.route}${res.error ? `: ${res.error}` : ""}`);
         }

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "2.5.1",
+  "version": "2.5.3",
   "builder": {
     "@nuxt/module-builder": "1.0.3",
     "unbuild": "3.6.1"

package/dist/runtime/server/db/drizzle/queries.js

@@ -706,7 +706,6 @@ export async function seedRoutes(event, routes, options = {}) {
           indexed = CASE WHEN ai_ready_pages.is_error = 1 THEN 0 ELSE ai_ready_pages.indexed END
         WHERE ai_ready_pages.last_seen_at IS NULL
           OR ai_ready_pages.last_seen_at < excluded.last_seen_at - ${refreshWindowMs}
-          OR ai_ready_pages.is_error = 1
           OR ai_ready_pages.locale IS DISTINCT FROM excluded.locale
       `,
       params

package/dist/runtime/server/db/queries.d.ts

@@ -147,6 +147,8 @@ export declare function seedRoutes(event: H3Event | undefined, routes: Array<str
     locale?: string;
     url?: string;
 }>, options?: SeedRoutesOptions): Promise<number>;
+/** Remove a page after its route returns HTTP 410. */
+export declare function prunePage(event: H3Event | undefined, route: string): Promise<void>;
 /**
  * Prune routes not seen in sitemap for longer than threshold
  * Only prunes routes with source='runtime' (never prerendered pages)

package/dist/runtime/server/db/queries.js

@@ -403,7 +403,6 @@ export async function seedRoutes(event, routes, options = {}) {
           indexed = CASE WHEN ai_ready_pages.is_error = 1 THEN 0 ELSE ai_ready_pages.indexed END
         WHERE ai_ready_pages.last_seen_at IS NULL
           OR ai_ready_pages.last_seen_at < excluded.last_seen_at - ${refreshWindowMs}
-          OR ai_ready_pages.is_error = 1
           OR ai_ready_pages.locale IS DISTINCT FROM excluded.locale
       `,
       params
@@ -412,6 +411,12 @@ export async function seedRoutes(event, routes, options = {}) {
   await db.batch(stmts);
   return byRoute.size;
 }
+export async function prunePage(event, route) {
+  const db = await getDb(event);
+  if (!db)
+    return;
+  await db.exec("DELETE FROM ai_ready_pages WHERE route = ?", [normalizeRoute(route)]);
+}
 export async function pruneStaleRoutes(event, staleThresholdSeconds, protectedSince, refreshWindowMs = SEED_REFRESH_WINDOW_MS) {
   const db = await getDb(event);
   if (!db)

package/dist/runtime/server/db/seed.d.ts

@@ -24,8 +24,8 @@ export declare const SEED_REFRESH_WINDOW_MS: number;
 export declare function resolveSeedRefreshWindowMs(pruneTtlSeconds: number): number;
 export interface SeedRoutesOptions {
     /**
-     * An existing row is rewritten only when its `last_seen_at` is older than
-     * this, when it is an error row, or when its locale changed.
+     * An existing row is rewritten when its `last_seen_at` is older than this
+     * or its locale changed. Error rows retry after this interval.
      * @default SEED_REFRESH_WINDOW_MS
      */
     refreshWindowMs?: number;

package/dist/runtime/server/middleware/markdown.js

@@ -128,13 +128,13 @@ ${contentPage.markdown}`);
   if (!response.ok && response.status !== 404)
     return response;
   if (response.status === 404) {
-    setMarkdownHeaders(event, ctx);
+    setMarkdownHeaders(event, ctx, response.headers);
     setResponseStatus(event, 404);
     return notFoundMarkdown(ctx, canonicalUrl, buildFrontmatter);
   }
   const contentType = response.headers.get("content-type") || "";
   if (!contentType.includes("text/html")) {
-    setMarkdownHeaders(event, ctx);
+    setMarkdownHeaders(event, ctx, response.headers);
     setResponseStatus(event, 404);
     return notFoundMarkdown(ctx, canonicalUrl, buildFrontmatter);
   }
@@ -169,6 +169,6 @@ ${contentPage.markdown}`);
       additionalFrontmatter
     }
   );
-  setMarkdownHeaders(event, ctx);
+  setMarkdownHeaders(event, ctx, response.headers);
   return finalizeMarkdown(result.markdown);
 });

package/dist/runtime/server/routes/__ai-ready/reindex.post.js

@@ -10,6 +10,8 @@ export default eventHandler(async (event) => {
   }
   const force = query.force !== "false" && query.force !== "0";
   const result = await indexPageByRoute(route, event, { force });
+  if (result.gone)
+    return { route, indexed: false, gone: true };
   if (!result.success) {
     setResponseStatus(event, 502);
     return { route, indexed: false, error: result.error ?? `Failed to index ${route}` };

package/dist/runtime/server/utils/batchIndex.js

@@ -36,12 +36,14 @@ export async function batchIndexPages(event, options = {}) {
           markFailedAsError: true,
           force: true
         });
-        return { route: page.route, success: result.success };
+        return { route: page.route, result };
       })
     );
     for (const result of results) {
       processed++;
-      if (result.success) {
+      if (result.result.gone) {
+        logger.debug(`[batchIndex] Pruned gone route: ${result.route}`);
+      } else if (result.result.success) {
         indexed++;
         logger.debug(`[batchIndex] Indexed: ${result.route}`);
       } else {

package/dist/runtime/server/utils/checkStale.d.ts

@@ -21,7 +21,7 @@ export declare const STALE_CHECK_INTERVAL_MS: number;
 /**
  * Check if data is stale and handle restore/mark-pending
  * Called at start of cron - handles:
- * 1. Empty DB → restore from dump
+ * 1. Empty DB with a different build ID → restore from dump
  * 2. Build ID changed → compare hashes, only mark changed pages pending, add new pages from dump
  * Skips HTTP fetch if checked within 5 minutes and DB is populated
  */

package/dist/runtime/server/utils/checkStale.js

@@ -78,7 +78,7 @@ export async function checkAndHandleStale(event) {
       logger.info("[stale-check] No build metadata found, skipping stale check");
     return { action: "none", dbCount, reason: "no_dump_metadata" };
   }
-  if (dbCount === 0) {
+  if (dbCount === 0 && storedBuildId !== meta.buildId) {
     if (debug)
       logger.info("[stale-check] DB empty, restoring from dump...");
     const rows = await fetchDump(event);

package/dist/runtime/server/utils/indexPage.d.ts

@@ -9,7 +9,7 @@ export interface IndexPageOptions {
     /** Mark failed fetches as errors in DB to prevent retry loops */
     markFailedAsError?: boolean;
 }
-export interface IndexPageResult {
+interface IndexedPageResult {
     success: boolean;
     /** True if page was already fresh and skipped */
     skipped?: boolean;
@@ -31,6 +31,12 @@ export interface IndexPageResult {
     /** Error message if failed */
     error?: string;
 }
+export type IndexPageResult = IndexedPageResult & ({
+    success: false;
+    gone: true;
+} | {
+    gone?: never;
+});
 /**
  * Manually index a page's HTML content into database
  * Use this from custom plugins or API routes to trigger indexing
@@ -41,3 +47,4 @@ export declare function indexPage(route: string, html: string, options?: IndexPa
  * Convenience wrapper around indexPage that handles fetching
  */
 export declare function indexPageByRoute(route: string, event: H3Event | undefined, options?: IndexPageOptions): Promise<IndexPageResult>;
+export {};

package/dist/runtime/server/utils/indexPage.js

@@ -1,5 +1,5 @@
 import { fetchWithEvent, useNitroApp, useRuntimeConfig } from "#nuxtseo/nitro";
-import { getPageIndexState, upsertPage } from "../db/queries.js";
+import { getPageIndexState, prunePage, upsertPage } from "../db/queries.js";
 import { computeContentHash } from "../db/shared.js";
 import { logger } from "../logger.js";
 import { convertHtmlToMarkdown } from "../utils.js";
@@ -79,18 +79,26 @@ export async function indexPage(route, html, options = {}, event) {
 }
 export async function indexPageByRoute(route, event, options = {}) {
   logger.debug(`[indexPageByRoute] Fetching HTML for ${route} (timeout: 10000ms)`);
-  const html = await (event ? fetchWithEvent(event, route, {
+  const fetched = await (event ? fetchWithEvent(event, route, {
     headers: { accept: "text/html", [INTERNAL_HEADER]: "1" },
     timeout: 1e4
   }) : globalThis.$fetch(route, {
     headers: { accept: "text/html", [INTERNAL_HEADER]: "1" },
     timeout: 1e4
     // 10s timeout per page (must fit within CF worker limit)
-  })).catch((err) => {
+  })).then((html2) => ({ _tag: "Html", html: html2 })).catch((err) => {
+    const status = err.status ?? err.statusCode ?? err.response?.status;
+    if (status === 410)
+      return { _tag: "Gone" };
     logger.warn(`[indexPageByRoute] Failed to fetch ${route}:`, err.message);
-    return null;
+    return { _tag: "Failed" };
   });
-  if (html) {
+  if (fetched._tag === "Gone") {
+    await prunePage(event, route);
+    return { success: false, gone: true };
+  }
+  const html = fetched._tag === "Html" ? fetched.html : null;
+  if (typeof html === "string" && html) {
     logger.debug(`[indexPageByRoute] Fetched ${route} (${html.length} bytes)`);
   }
   if (!html || typeof html !== "string") {

package/dist/runtime/server/utils/negotiation-response.d.ts

@@ -17,7 +17,7 @@ type NegotiationResponse = Awaited<ReturnType<typeof sendRedirect>> | undefined;
 export declare function setLinkHeader(event: H3Event, ctx: NegotiationContext, variant: 'html' | 'markdown'): void;
 export declare function setStatusAwareHeader(event: H3Event, ctx: NegotiationContext, variant: 'html' | 'markdown'): void;
 export declare function setUncacheableHeaders(event: H3Event): void;
-export declare function setMarkdownHeaders(event: H3Event, ctx: NegotiationContext): void;
+export declare function setMarkdownHeaders(event: H3Event, ctx: NegotiationContext, sourceHeaders?: Headers): void;
 /**
  * Site config is set up by a Nitro middleware. The early negotiation handler
  * runs in front of that middleware, so it must run the setup itself before it

package/dist/runtime/server/utils/negotiation-response.js

@@ -1,6 +1,6 @@
 import { createNitroRouteRuleMatcher } from "nuxtseo-shared/server";
 import { localAgentSkillArtifacts } from "#ai-ready-virtual/agent-skills.mjs";
-import { appendHeader, createError, getRequestHost, getResponseHeader, sendRedirect, setHeader } from "#nuxtseo/h3";
+import { appendHeader, createError, getHeader, getRequestHost, getResponseHeader, sendRedirect, setHeader } from "#nuxtseo/h3";
 import { useRuntimeConfig } from "#nuxtseo/nitro";
 import { withSiteUrl } from "#site-config/server/composables/utils";
 import initSiteConfig from "#site-config/server/middleware/init";
@@ -42,9 +42,24 @@ export function setUncacheableHeaders(event) {
       setHeader(event, header, "no-store");
   }
 }
-export function setMarkdownHeaders(event, ctx) {
+export function setMarkdownHeaders(event, ctx, sourceHeaders) {
   setHeader(event, "content-type", "text/markdown; charset=utf-8");
   setLinkHeader(event, ctx, "markdown");
+  const privateResponse = [
+    "cache-control",
+    "cdn-cache-control",
+    "cloudflare-cdn-cache-control",
+    "netlify-cdn-cache-control",
+    "vercel-cdn-cache-control",
+    "surrogate-control"
+  ].some((header) => {
+    const value = `${getResponseHeader(event, header) || ""},${sourceHeaders?.get(header) || ""}`;
+    return /(?:^|,)\s*(?:private|no-store|no-cache)\s*(?:,|=|$)/i.test(value);
+  });
+  if (getHeader(event, "cookie") || getHeader(event, "authorization") || getResponseHeader(event, "set-cookie") || sourceHeaders?.has("set-cookie") || privateResponse) {
+    setUncacheableHeaders(event);
+    return;
+  }
   const cacheHeaders = ctx.config.markdownCacheHeaders;
   if (cacheHeaders) {
     const { maxAge, swr } = cacheHeaders;

package/node_modules/mdream/node_modules/@emnapi/core/node_modules/@emnapi/wasi-threads/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021-present Toyobayashi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/node_modules/mdream/node_modules/@emnapi/core/node_modules/@emnapi/wasi-threads/README.md

@@ -0,0 +1,203 @@
+# @emnapi/wasi-threads
+
+This package makes [wasi-threads proposal](https://github.com/WebAssembly/wasi-threads) based WASI modules work in Node.js and browser.
+
+## Quick Start
+
+`index.html`
+
+```html
+<script src="./node_modules/@tybys/wasm-util/dist/wasm-util.js"></script>
+<script src="./node_modules/@emnapi/wasi-threads/dist/wasi-threads.js"></script>
+<script src="./index.js"></script>
+```
+
+If your application will block browser main thread (for example `pthread_join`), please run it in worker instead.
+
+```html
+<script>
+  // pthread_join (Atomics.wait) cannot be called in browser main thread
+  new Worker('./index.js')
+</script>
+```
+
+`index.js`
+
+```js
+const ENVIRONMENT_IS_NODE =
+  typeof process === 'object' && process !== null &&
+  typeof process.versions === 'object' && process.versions !== null &&
+  typeof process.versions.node === 'string';
+
+(function (main) {
+  if (ENVIRONMENT_IS_NODE) {
+    main(require)
+  } else {
+    if (typeof importScripts === 'function') {
+      importScripts('./node_modules/@tybys/wasm-util/dist/wasm-util.js')
+      importScripts('./node_modules/@emnapi/wasi-threads/dist/wasi-threads.js')
+    }
+    const nodeWasi = { WASI: globalThis.wasmUtil.WASI }
+    const nodeWorkerThreads = {
+      Worker: globalThis.Worker
+    }
+    const _require = function (request) {
+      if (request === 'node:wasi' || request === 'wasi') return nodeWasi
+      if (request === 'node:worker_threads' || request === 'worker_threads') return nodeWorkerThreads
+      if (request === '@emnapi/wasi-threads') return globalThis.wasiThreads
+      throw new Error('Can not find module: ' + request)
+    }
+    main(_require)
+  }
+})(async function (require) {
+  const { WASI } = require('wasi')
+  const { Worker } = require('worker_threads')
+  const { WASIThreads } = require('@emnapi/wasi-threads')
+
+  const wasi = new WASI({
+    version: 'preview1'
+  })
+  const wasiThreads = new WASIThreads({
+    wasi,
+
+    /**
+     * avoid Atomics.wait() deadlock during thread creation in browser
+     * see https://emscripten.org/docs/tools_reference/settings_reference.html#pthread-pool-size
+     */
+    reuseWorker: ENVIRONMENT_IS_NODE
+      ? false
+      : {
+          size: 4 /** greater than actual needs (2) */,
+          strict: true
+        },
+
+    /**
+     * Synchronous thread creation
+     * pthread_create will not return until thread worker actually starts
+     */
+    waitThreadStart: typeof window === 'undefined' ? 1000 : false,
+
+    onCreateWorker: () => {
+      return new Worker('./worker.js', {
+        execArgv: ['--experimental-wasi-unstable-preview1']
+      })
+    }
+  })
+  const memory = new WebAssembly.Memory({
+    initial: 16777216 / 65536,
+    maximum: 2147483648 / 65536,
+    shared: true
+  })
+  let input
+  const file = 'path/to/your/wasi-module.wasm'
+  try {
+    input = require('fs').readFileSync(require('path').join(__dirname, file))
+  } catch (err) {
+    const response = await fetch(file)
+    input = await response.arrayBuffer()
+  }
+  let { module, instance } = await WebAssembly.instantiate(input, {
+    env: { memory },
+    wasi_snapshot_preview1: wasi.wasiImport,
+    ...wasiThreads.getImportObject()
+  })
+
+  wasiThreads.setup(instance, module, memory)
+  await wasiThreads.preloadWorkers()
+
+  if (typeof instance.exports._start === 'function') {
+    return wasi.start(instance)
+  } else {
+    wasi.initialize(instance)
+    // instance.exports.exported_wasm_function()
+  }
+})
+```
+
+`worker.js`
+
+```js
+(function (main) {
+  const ENVIRONMENT_IS_NODE =
+    typeof process === 'object' && process !== null &&
+    typeof process.versions === 'object' && process.versions !== null &&
+    typeof process.versions.node === 'string'
+
+  if (ENVIRONMENT_IS_NODE) {
+    const _require = function (request) {
+      return require(request)
+    }
+
+    const _init = function () {
+      const nodeWorkerThreads = require('worker_threads')
+      const parentPort = nodeWorkerThreads.parentPort
+
+      parentPort.on('message', (data) => {
+        globalThis.onmessage({ data })
+      })
+
+      Object.assign(globalThis, {
+        self: globalThis,
+        require,
+        Worker: nodeWorkerThreads.Worker,
+        importScripts: function (f) {
+          (0, eval)(require('fs').readFileSync(f, 'utf8') + '//# sourceURL=' + f)
+        },
+        postMessage: function (msg) {
+          parentPort.postMessage(msg)
+        }
+      })
+    }
+
+    main(_require, _init)
+  } else {
+    importScripts('./node_modules/@tybys/wasm-util/dist/wasm-util.js')
+    importScripts('./node_modules/@emnapi/wasi-threads/dist/wasi-threads.js')
+
+    const nodeWasi = { WASI: globalThis.wasmUtil.WASI }
+    const _require = function (request) {
+      if (request === '@emnapi/wasi-threads') return globalThis.wasiThreads
+      if (request === 'node:wasi' || request === 'wasi') return nodeWasi
+      throw new Error('Can not find module: ' + request)
+    }
+    const _init = function () {}
+    main(_require, _init)
+  }
+})(function main (require, init) {
+  init()
+
+  const { WASI } = require('wasi')
+  const { ThreadMessageHandler, WASIThreads } = require('@emnapi/wasi-threads')
+
+  const handler = new ThreadMessageHandler({
+    async onLoad ({ wasmModule, wasmMemory }) {
+      const wasi = new WASI({
+        version: 'preview1'
+      })
+
+      const wasiThreads = new WASIThreads({
+        wasi,
+        childThread: true
+      })
+
+      const originalInstance = await WebAssembly.instantiate(wasmModule, {
+        env: {
+          memory: wasmMemory,
+        },
+        wasi_snapshot_preview1: wasi.wasiImport,
+        ...wasiThreads.getImportObject()
+      })
+
+      // must call `initialize` instead of `start` in child thread
+      const instance = wasiThreads.initialize(originalInstance, wasmModule, wasmMemory)
+
+      return { module: wasmModule, instance }
+    }
+  })
+
+  globalThis.onmessage = function (e) {
+    handler.handle(e)
+    // handle other messages
+  }
+})
+```