nuxt-ai-ready 1.5.4 → 1.5.5

package/dist/module.json

@@ -4,7 +4,7 @@
     "nuxt": ">=4.0.0"
   },
   "configKey": "aiReady",
-  "version": "1.5.4",
+  "version": "1.5.5",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/module.mjs

@@ -1,7 +1,7 @@
 import { createHash, randomBytes } from 'node:crypto';
-import { mkdir, writeFile, appendFile, stat, access } from 'node:fs/promises';
+import { mkdir, writeFile, appendFile, stat, access, readFile } from 'node:fs/promises';
 import { join, dirname } from 'node:path';
-import { useLogger, useNuxt, addTypeTemplate, addTemplate, defineNuxtModule, createResolver, hasNuxtModule, addServerHandler, addPlugin, addServerPlugin } from '@nuxt/kit';
+import { useLogger, useNuxt, addTypeTemplate, addTemplate, defineNuxtModule, createResolver, hasNuxtModule, addServerPlugin, addServerHandler, addPlugin } from '@nuxt/kit';
 import defu from 'defu';
 import { installNuxtSiteConfig, useSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
 import { setupDevToolsUI } from 'nuxtseo-shared/devtools';
@@ -10,7 +10,7 @@ import { readPackageJSON, resolvePackageJSON } from 'pkg-types';
 import { parseSitemapXml } from '@nuxtjs/sitemap/utils';
 import { colorize } from 'consola/utils';
 import { withBase } from 'ufo';
-import { toMarkdownPath } from '../dist/runtime/markdown-path.js';
+import { normalizePagePath, toMarkdownPath } from '../dist/runtime/markdown-path.js';
 import { initSchema, computeContentHash, insertPage, queryAllPages, exportDbDump } from '../dist/runtime/server/db/shared.js';
 import { comparePageHashes, submitToIndexNowShared } from '../dist/runtime/server/utils/indexnow-shared.js';
 import { buildLlmsFullTxtHeader, formatPageForLlmsFullTxt } from '../dist/runtime/server/utils/llms-full.js';
@@ -125,14 +125,18 @@ function resolveRouteLocale(route, i18n) {
   return matched ? matched.code : i18n.defaultLocale;
 }
 async function processMarkdownRoute(state, nuxt, route, parsed, lastmod, options) {
-  const { markdown, title, description, headings, keywords, updatedAt: metaUpdatedAt } = parsed;
+  route = normalizePagePath(route);
+  const { title, description, headings, keywords, updatedAt: metaUpdatedAt } = parsed;
   let updatedAt = (lastmod instanceof Date ? lastmod.toISOString() : lastmod) || (/* @__PURE__ */ new Date()).toISOString();
   if (metaUpdatedAt) {
     const parsedDate = new Date(metaUpdatedAt);
     if (!Number.isNaN(parsedDate.getTime()))
       updatedAt = parsedDate.toISOString();
   }
-  await nuxt.hooks.callHook("ai-ready:page:markdown", { route, markdown, title, description, headings });
+  const hookContext = { ...parsed, route };
+  await nuxt.hooks.callHook("ai-ready:page:markdown", hookContext);
+  const { markdown } = hookContext;
+  parsed.markdown = markdown;
   if (state.db) {
     const contentHash = await computeContentHash(markdown);
     await insertPage(state.db, {
@@ -147,7 +151,7 @@ async function processMarkdownRoute(state, nuxt, route, parsed, lastmod, options
       locale: resolveRouteLocale(route, state.i18n)
     });
   }
-  if (state.llmsFullTxtPath && !options?.skipLlmsFullTxt) {
+  if (state.llmsFullTxtPath && !options?.skipLlmsFullTxt && markdown.trim()) {
     const pageContent = formatPageForLlmsFullTxt(route, title, description, markdown, state.siteInfo?.url);
     logger.debug(`Appending to llms-full.txt: ${route} (${(pageContent.length / 1024).toFixed(1)}kb)`);
     await appendFile(state.llmsFullTxtPath, pageContent, "utf-8");
@@ -157,7 +161,7 @@ async function processMarkdownRoute(state, nuxt, route, parsed, lastmod, options
 async function processSitemapEntry(state, nuxt, nitro, entry) {
   const loc = typeof entry === "string" ? entry : entry.loc;
   const lastmod = typeof entry === "string" ? void 0 : entry.lastmod;
-  const route = loc.startsWith("http") ? new URL(loc).pathname : loc;
+  const route = normalizePagePath(loc.startsWith("http") ? new URL(loc).pathname : loc);
   if (route.split("/").some((segment) => segment.startsWith("_"))) {
     return { crawled: false, skipped: true };
   }
@@ -566,6 +570,37 @@ async function detectI18n(opts = {}) {
   return toRuntimeI18nConfig(auto);
 }
 
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function ensureStaticHeader(contents, route, name, value) {
+  const eol = contents.includes("\r\n") ? "\r\n" : "\n";
+  const routePattern = new RegExp(`^${escapeRegExp(route)}[\\t ]*\\r?$`, "gm");
+  const routeMatches = [...contents.matchAll(routePattern)];
+  const routeMatch = routeMatches.at(-1);
+  if (!routeMatch || routeMatch.index === void 0) {
+    const separator = contents.length === 0 ? "" : contents.endsWith(`${eol}${eol}`) ? "" : contents.endsWith(eol) ? eol : `${eol}${eol}`;
+    return `${contents}${separator}${route}${eol}  ${name}: ${value}${eol}`;
+  }
+  const routeLineEnd = routeMatch.index + routeMatch[0].length;
+  const blockStart = contents[routeLineEnd] === "\n" ? routeLineEnd + 1 : routeLineEnd;
+  const nextBlockPattern = /^(?![\t ]|\r?$).+/gm;
+  nextBlockPattern.lastIndex = blockStart;
+  const nextBlock = nextBlockPattern.exec(contents);
+  const blockEnd = nextBlock?.index ?? contents.length;
+  const block = contents.slice(blockStart, blockEnd);
+  const escapedName = escapeRegExp(name);
+  const existingHeaderPattern = new RegExp(
+    `^[\\t ]+(?:${escapedName}[\\t ]*:|![\\t ]*${escapedName}[\\t ]*\\r?$)`,
+    "im"
+  );
+  if (existingHeaderPattern.test(block)) {
+    return contents;
+  }
+  const prefix = blockStart === routeLineEnd ? eol : "";
+  return `${contents.slice(0, blockStart)}${prefix}  ${name}: ${value}${eol}${contents.slice(blockStart)}`;
+}
+
 const module$1 = defineNuxtModule({
   meta: {
     name: "nuxt-ai-ready",
@@ -682,6 +717,8 @@ const module$1 = defineNuxtModule({
       robotsOpts.groups = groups;
       const group = {
         userAgent: "*",
+        // Preserve nuxt-robots' default wildcard rule so the injected group remains valid.
+        disallow: [""],
         contentSignal: [`ai-train=${config.contentSignal.aiTrain ? "yes" : "no"}`, `search=${config.contentSignal.search ? "yes" : "no"}`, `ai-input=${config.contentSignal.aiInput ? "yes" : "no"}`]
       };
       if (config.contentSignal.contentUsage !== false)
@@ -944,6 +981,7 @@ export async function lookupContentPage(event, path) {
       i18n: i18nConfig,
       ftsTokenizer
     };
+    addServerPlugin(resolve("./runtime/server/plugins/html-capture.prerender"));
     addServerHandler({
       middleware: true,
       handler: resolve("./runtime/server/middleware/markdown.prerender")
@@ -1012,20 +1050,29 @@ export async function lookupContentPage(event, path) {
       icon: "carbon:ai-label"
     }, resolve, nuxt);
     nuxt.options.nitro.routeRules = nuxt.options.nitro.routeRules || {};
-    nuxt.options.nitro.routeRules["/llms.txt"] = { headers: { "Content-Type": "text/plain; charset=utf-8" } };
-    nuxt.options.nitro.routeRules["/llms-full.txt"] = { headers: { "Content-Type": "text/plain; charset=utf-8" } };
+    for (const route of ["/llms.txt", "/llms-full.txt"]) {
+      nuxt.options.nitro.routeRules[route] = defu(
+        nuxt.options.nitro.routeRules[route],
+        { headers: { "Content-Type": "text/plain; charset=utf-8" } }
+      );
+    }
     nuxt.hooks.hook("nitro:build:before", (nitro) => {
       nitro.hooks.hook("compiled", async () => {
         const headersPath = join(nitro.options.output.publicDir, "_headers");
         logger.debug(`Checking for _headers file: ${headersPath}`);
         const exists = await access(headersPath).then(() => true).catch(() => false);
         if (exists) {
-          logger.debug(`Appending .md charset header to _headers`);
-          await appendFile(headersPath, `
-/*.md
-  Content-Type: text/markdown; charset=utf-8
-`);
-          logger.debug("Appended .md charset header to _headers");
+          const headers = await readFile(headersPath, "utf8");
+          const mergedHeaders = ensureStaticHeader(
+            headers,
+            "/*.md",
+            "Content-Type",
+            "text/markdown; charset=utf-8"
+          );
+          if (mergedHeaders !== headers) {
+            await writeFile(headersPath, mergedHeaders);
+            logger.debug("Merged .md charset header into _headers");
+          }
         }
       });
     });

package/dist/runtime/markdown-path.d.ts

@@ -1 +1,2 @@
+export declare function normalizePagePath(path: string): string;
 export declare function toMarkdownPath(path: string): string;

package/dist/runtime/markdown-path.js

@@ -1,6 +1,10 @@
 const RE_TRAILING_SLASHES = /\/+$/;
+export function normalizePagePath(path) {
+  return path.replace(RE_TRAILING_SLASHES, "") || "/";
+}
 export function toMarkdownPath(path) {
-  if (path === "/")
+  const normalizedPath = normalizePagePath(path);
+  if (normalizedPath === "/")
     return "/index.md";
-  return `${path.replace(RE_TRAILING_SLASHES, "")}.md`;
+  return `${normalizedPath}.md`;
 }

package/dist/runtime/server/middleware/markdown.prerender.js

@@ -6,6 +6,7 @@ import { convertHtmlToMarkdown, extractLastUpdated, getMarkdownRenderInfo } from
 import { tryGetContentMarkdown } from "../utils/content.js";
 import { buildFrontmatter } from "../utils/frontmatter.js";
 import { extractKeywords } from "../utils/keywords.js";
+import { consumePrerenderedHtml } from "../utils/prerender-html.js";
 function extractHeadingsFromMarkdown(markdown) {
   const headings = [];
   for (const m of markdown.matchAll(/^(#{1,6}) ([^\n]+)$/gm)) {
@@ -50,26 +51,32 @@ ${contentPage.markdown}`;
       updatedAt: lastUpdated2
     });
   }
-  logger.debug(`[markdown.prerender] Fetching HTML for ${path}`);
-  const response = await event.fetch(path, { signal: AbortSignal.timeout(3e4) }).catch((err) => {
-    if (err?.name === "TimeoutError" || err?.name === "AbortError") {
-      throw createError({
-        statusCode: 504,
-        statusMessage: "Gateway Timeout",
-        message: `Timed out fetching HTML for ${path}`
+  let html = consumePrerenderedHtml(path);
+  if (html) {
+    logger.debug(`[markdown.prerender] Reusing prerendered HTML for ${path} (${html.length} bytes)`);
+  } else {
+    logger.debug(`[markdown.prerender] Fetching HTML for ${path}`);
+    const response = await event.fetch(path, { signal: AbortSignal.timeout(3e4) }).catch((err) => {
+      if (err?.name === "TimeoutError" || err?.name === "AbortError") {
+        throw createError({
+          statusCode: 504,
+          statusMessage: "Gateway Timeout",
+          message: `Timed out fetching HTML for ${path}`
+        });
+      }
+      throw err;
+    });
+    if (!response.ok) {
+      return createError({
+        statusCode: response.status,
+        statusMessage: response.statusText,
+        message: `Failed to fetch HTML for ${path}`
       });
     }
-    throw err;
-  });
-  if (!response.ok) {
-    return createError({
-      statusCode: response.status,
-      statusMessage: response.statusText,
-      message: `Failed to fetch HTML for ${path}`
-    });
+    html = await response.text();
+    logger.debug(`[markdown.prerender] Fetched HTML for ${path} (${html.length} bytes)`);
+    consumePrerenderedHtml(path);
   }
-  const html = await response.text();
-  logger.debug(`[markdown.prerender] Fetched HTML for ${path} (${html.length} bytes)`);
   if (html.includes("__NUXT_ERROR__") || html.includes("nuxt-error-page")) {
     return createError({
       statusCode: 404,

package/dist/runtime/server/plugins/html-capture.prerender.d.ts

@@ -0,0 +1,2 @@
+import type { NitroApp } from 'nitropack/types';
+export default function htmlCapturePlugin(nitroApp: NitroApp): void;

package/dist/runtime/server/plugins/html-capture.prerender.js

@@ -0,0 +1,10 @@
+import { storePrerenderedHtml } from "../utils/prerender-html.js";
+export default function htmlCapturePlugin(nitroApp) {
+  if (!import.meta.prerender)
+    return;
+  nitroApp.hooks.hook("render:response", (response, { event }) => {
+    if (typeof response.body !== "string" || !String(response.headers?.["content-type"] || "").includes("text/html"))
+      return;
+    storePrerenderedHtml(event.path, response.body);
+  });
+}

package/dist/runtime/server/utils/prerender-html.d.ts

@@ -0,0 +1,2 @@
+export declare function storePrerenderedHtml(path: string, html: string): void;
+export declare function consumePrerenderedHtml(path: string): string | undefined;

package/dist/runtime/server/utils/prerender-html.js

@@ -0,0 +1,13 @@
+const htmlByPath = /* @__PURE__ */ new Map();
+function normalizeHtmlCachePath(path) {
+  return path.replace(/\/+$/, "") || "/";
+}
+export function storePrerenderedHtml(path, html) {
+  htmlByPath.set(normalizeHtmlCachePath(path), html);
+}
+export function consumePrerenderedHtml(path) {
+  const key = normalizeHtmlCachePath(path);
+  const html = htmlByPath.get(key);
+  htmlByPath.delete(key);
+  return html;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "1.5.4",
+  "version": "1.5.5",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -57,13 +57,13 @@
     }
   },
   "dependencies": {
-    "@mdream/js": "^1.4.1",
-    "@nuxt/kit": "^4.4.8",
+    "@mdream/js": "^1.5.2",
+    "@nuxt/kit": "^4.5.0",
     "citty": "^0.2.2",
     "consola": "^3.4.2",
     "defu": "^6.1.7",
     "drizzle-orm": "^0.45.2",
-    "mdream": "^1.4.1",
+    "mdream": "^1.5.2",
     "nuxt-site-config": "^4.1.1",
     "nuxtseo-shared": "^5.3.2",
     "pathe": "^2.0.3",
@@ -74,13 +74,13 @@
   },
   "devDependencies": {
     "@antfu/eslint-config": "^9.1.0",
-    "@arethetypeswrong/cli": "^0.18.4",
+    "@arethetypeswrong/cli": "^0.18.5",
     "@libsql/client": "^0.17.4",
     "@nuxt/content": "^3.15.0",
     "@nuxt/module-builder": "^1.0.2",
     "@nuxt/test-utils": "^4.0.3",
     "@nuxtjs/eslint-config-typescript": "^12.1.0",
-    "@nuxtjs/mcp-toolkit": "^0.17.2",
+    "@nuxtjs/mcp-toolkit": "^0.18.0",
     "@nuxtjs/robots": "^6.1.2",
     "@nuxtjs/sitemap": "^8.2.2",
     "@types/better-sqlite3": "^7.6.13",
@@ -89,12 +89,12 @@
     "@vueuse/nuxt": "^14.3.0",
     "better-sqlite3": "^12.11.1",
     "bumpp": "^11.1.0",
-    "eslint": "^10.6.0",
+    "eslint": "^10.7.0",
     "eslint-plugin-harlanzw": "^0.17.0",
-    "execa": "^9.6.1",
+    "execa": "^10.0.0",
     "happy-dom": "^20.10.6",
     "nitropack": "^2.13.4",
-    "nuxt": "^4.4.8",
+    "nuxt": "^4.5.0",
     "nuxt-site-config": "^4.1.1",
     "nuxtseo-layer-devtools": "^5.3.2",
     "playwright": "^1.61.1",
@@ -103,10 +103,10 @@
     "typescript": "^6.0.3",
     "unbuild": "^3.6.1",
     "vitest": "^4.1.10",
-    "vue": "^3.5.39",
-    "vue-router": "^5.1.0",
-    "vue-tsc": "^3.3.6",
-    "wrangler": "^4.107.0",
+    "vue": "^3.5.40",
+    "vue-router": "^5.2.0",
+    "vue-tsc": "^3.3.7",
+    "wrangler": "^4.112.0",
     "zod": "^4.4.3"
   },
   "scripts": {