nuxt-ai-ready 2.0.1 → 2.2.0

package/dist/runtime/server/utils/sitemap-md.js

@@ -0,0 +1,65 @@
+import { toMarkdownPath } from "../../markdown-path.js";
+import { toDeployedRoute } from "../../route-path.js";
+export const SITEMAP_MD_ROUTE = "/sitemap.md";
+function topSegment(route) {
+  return route.split("/").filter(Boolean)[0] || "";
+}
+function escapeTitle(title) {
+  return title.replaceAll("[", "\\[").replaceAll("]", "\\]");
+}
+function formatEntry(entry, resolveHref) {
+  const title = escapeTitle(entry.title?.trim() || entry.route);
+  const href = resolveHref(entry.route);
+  if (entry.updatedAt) {
+    const parsed = new Date(entry.updatedAt);
+    if (!Number.isNaN(parsed.getTime()))
+      return `- [${title}](${href} "${parsed.toISOString()}")`;
+  }
+  return `- [${title}](${href})`;
+}
+export function buildSitemapMd(entries, options = {}) {
+  const resolveHref = options.resolveHref || toMarkdownPath;
+  const groups = /* @__PURE__ */ new Map();
+  for (const entry of [...entries].sort((a, b) => a.route.localeCompare(b.route))) {
+    const key = topSegment(entry.route);
+    const group = groups.get(key);
+    if (group)
+      group.push(entry);
+    else
+      groups.set(key, [entry]);
+  }
+  const keys = [...groups.keys()].sort((a, b) => {
+    if (a === "")
+      return -1;
+    if (b === "")
+      return 1;
+    return a.localeCompare(b);
+  });
+  const parts = [`# ${options.siteName || "Site"} Sitemap`, "", "All pages in Markdown format."];
+  for (const key of keys) {
+    parts.push("", `## ${key || "Root"}`);
+    for (const entry of groups.get(key))
+      parts.push(formatEntry(entry, resolveHref));
+  }
+  return `${parts.join("\n")}
+`;
+}
+export function appendSitemapSection(markdown, sitemapHref = SITEMAP_MD_ROUTE) {
+  const section = `## Sitemap
+
+See the full [sitemap](${sitemapHref}) for all pages.`;
+  const trimmed = markdown.trimEnd();
+  if (trimmed.endsWith(section))
+    return `${trimmed}
+`;
+  return `${trimmed}
+
+${section}
+`;
+}
+export function isSitemapMdRequest(path, baseURL, enabled) {
+  if (!enabled)
+    return false;
+  const withoutQuery = path.split("?")[0] || path;
+  return withoutQuery === SITEMAP_MD_ROUTE || withoutQuery === toDeployedRoute(SITEMAP_MD_ROUTE, baseURL);
+}

package/dist/runtime/types.d.ts

@@ -149,6 +149,19 @@ export interface ModuleOptions {
      * Structured llms.txt configuration
      */
     llmsTxt?: LlmsTxtConfig;
+    /**
+     * Generate a Markdown sitemap at `/sitemap.md` from indexed pages and end
+     * every Markdown page with a `## Sitemap` section linking to it.
+     * @default true
+     */
+    sitemapMd?: boolean;
+    /**
+     * Advertise llms.txt as `rel="describedby"` from HTML and Markdown
+     * responses, as the llms.txt v2 spec requires. Emits an HTML `<link>` tag
+     * and a Link header entry pointing at the llms.txt route.
+     * @default true
+     */
+    describedby?: boolean;
     /**
      * Content Signal Directives
      */
@@ -231,7 +244,7 @@ export interface ModuleOptions {
     llmsTxtCacheSeconds?: number;
     /**
      * Database configuration for page storage
-     * Supports SQLite, LibSQL/Turso, Cloudflare D1, and Neon Postgres
+     * Supports SQLite, LibSQL/Turso, Cloudflare D1, Neon, and PostgreSQL
      *
      * Storage stays off until a runtime feature or page tool needs it.
      * Set to `false` (or `{ type: 'none' }`) to force it off.
@@ -247,9 +260,10 @@ export interface ModuleOptions {
          * - 'bun': Bun SQLite via bun:sqlite (auto-detected on Bun) [experimental]
          * - 'libsql': Turso/LibSQL [experimental]
          * - 'neon': Vercel Postgres via Neon serverless (auto-detected on Vercel with POSTGRES_URL) [experimental]
+         * - 'postgres': PostgreSQL via Postgres.js [experimental]
          * @default 'sqlite' when a requested feature needs storage
          */
-        type?: 'none' | 'sqlite' | 'bun' | 'd1' | 'libsql' | 'neon';
+        type?: 'none' | 'sqlite' | 'bun' | 'd1' | 'libsql' | 'neon' | 'postgres';
         /**
          * SQLite filename (relative to rootDir or absolute)
          * @default '.data/ai-ready/pages.db'
@@ -257,12 +271,12 @@ export interface ModuleOptions {
         filename?: string;
         /**
          * D1 binding name for Cloudflare Workers/Pages
-         * @default 'AI_READY_DB'
+         * @default 'DB'
          */
         bindingName?: string;
         /**
-         * Database URL for LibSQL/Turso or Neon/Vercel Postgres
-         * For Vercel: auto-reads from POSTGRES_URL env var
+         * Database URL for LibSQL/Turso, Neon, or PostgreSQL
+         * PostgreSQL drivers also read POSTGRES_URL or DATABASE_URL.
          */
         url?: string;
         /**
@@ -271,7 +285,7 @@ export interface ModuleOptions {
         authToken?: string;
     };
     /**
-     * Enable scheduled cron task (runs every minute)
+     * Enable scheduled cron task (runs every 5 minutes)
      * When true, automatically enables runtimeSync for background indexing
      */
     cron?: boolean;
@@ -321,7 +335,7 @@ export interface ModuleOptions {
         ttl?: number;
         /**
          * Pages to index per batch
-         * @default 20
+         * @default 50
          */
         batchSize?: number;
         /**

package/dist/shared/{nuxt-ai-ready.D8qbnNrv.mjs → nuxt-ai-ready.D5URd0TD.mjs}

@@ -8,9 +8,12 @@ import { installNuxtSiteConfig, useSiteConfig, withSiteUrl } from 'nuxt-site-con
 import { resolveNuxtContentVersion, renderNitroTypeAugmentations, setupNitroRuntimeCompatibility } from 'nuxtseo-shared/kit';
 import { readPackageJSON, resolvePackageJSON } from 'pkg-types';
 import { fileURLToPath, pathToFileURL } from 'node:url';
+import { isReservedPath, normalizePagePath, toMarkdownPath } from '../../dist/runtime/markdown-path.js';
+import { SITEMAP_MD_ROUTE } from '../../dist/runtime/server/utils/sitemap-md.js';
 import { MCP_SERVER_CARD_MEDIA_TYPE, AI_CATALOG_MEDIA_TYPE } from '../../dist/runtime/server/utils/discovery-response.js';
 import { isAbsolute, join as join$1 } from 'pathe';
 import { resolveI18nConfig, toRuntimeI18nConfig as toRuntimeI18nConfig$1 } from 'nuxtseo-shared/i18n';
+import { toDeployedRoute } from '../../dist/runtime/route-path.js';
 
 const SELECT_PAGE_DATA = "SELECT route, title, description, markdown, headings, keywords, updated_at, is_error, locale FROM ai_ready_pages";
 function createBuildPageDataVirtual(options) {
@@ -175,7 +178,7 @@ function resolveModuleEntryUrl(url) {
 
 const MARKDOWN_LINK_AVAILABILITY_FILE = "markdown-link-availability.json";
 
-const declarativeWebMcpTypes = `
+const webmcpTypes = `
 import type { WebMcpToolsContext } from 'nuxt-ai-ready/webmcp'
 
 declare module '#app' {
@@ -184,37 +187,6 @@ declare module '#app' {
     'ai-ready:webmcp:tools': (context: WebMcpToolsContext) => void | Promise<void>
   }
 }
-
-declare module '@vue/runtime-dom' {
-  interface HTMLAttributes {
-    /** Names the tool this form exposes to agents. */
-    toolname?: string
-    /** Describes what submitting this form does. */
-    tooldescription?: string
-    /** Describes this field as a tool parameter. */
-    toolparamdescription?: string
-    /** Submit the form as soon as an agent fills it in. */
-    toolautosubmit?: boolean | ''
-  }
-}
-
-declare global {
-  interface SubmitEvent {
-    /** Whether an agent triggered this submit through a tool call. */
-    readonly agentInvoked?: boolean
-    /** Return a result to the agent. Call preventDefault() first. */
-    respondWith?: (result: Promise<unknown>) => void
-  }
-
-  interface WebMcpToolEvent extends Event {
-    readonly toolName: string
-  }
-
-  interface WindowEventMap {
-    toolactivated: WebMcpToolEvent
-    toolcancel: WebMcpToolEvent
-  }
-}
 `;
 function registerTypeTemplates(ctx) {
   const nitroTypes = renderNitroTypeAugmentations(ctx.nitroCompatibility, {
@@ -231,7 +203,7 @@ import type { MarkdownContext, MarkdownSourceContext, PageIndexedContext } from
 import type { MdreamOptions } from 'mdream'
 
 ${nitroTypes}
-${ctx.config.webmcp ? declarativeWebMcpTypes : ""}
+${ctx.config.webmcp ? webmcpTypes : ""}
 export {}
 `
   });
@@ -598,7 +570,7 @@ function resolveDatabaseConfig(input) {
       database: { _tag: "Enabled", type, bindingName: config.bindingName || "DB" }
     };
   }
-  if (type === "neon") {
+  if (type === "neon" || type === "postgres") {
     return {
       _tag: "Resolved",
       logs,
@@ -892,6 +864,57 @@ function ensureStaticHeader(contents, route, name, value) {
   return `${contents.slice(0, blockStart)}${prefix}  ${name}: ${value}${eol}${contents.slice(blockStart)}`;
 }
 
+const RE_MD_EXT = /\.md$/;
+function isStaticMarkdownSourceRoute(route) {
+  if (route.includes("*") || route.includes(":"))
+    return false;
+  const path = route.split("?")[0] || route;
+  if (isReservedPath(path))
+    return false;
+  const lastSegment = path.split("/").pop() || "";
+  return !lastSegment.includes(".");
+}
+function staticDescribedbyEntry(baseURL) {
+  return `<${encodeURI(toDeployedRoute("/llms.txt", baseURL))}>; rel="describedby"`;
+}
+function buildStaticMarkdownLinkHeader(route, baseURL, describedby) {
+  const htmlRoute = encodeURI(toDeployedRoute(normalizePagePath(route), baseURL));
+  const parts = [
+    `<${htmlRoute}>; rel="alternate"; type="text/html"`,
+    `<${htmlRoute}>; rel="canonical"`
+  ];
+  if (describedby)
+    parts.push(staticDescribedbyEntry(baseURL));
+  return parts.join(", ");
+}
+function pageRouteFromMarkdownTwin(fileName) {
+  if (!fileName?.endsWith(".md") || fileName === SITEMAP_MD_ROUTE)
+    return null;
+  const pageRoute = normalizePagePath(fileName.replace(RE_MD_EXT, ""));
+  if (pageRoute === "/index")
+    return "/";
+  return isStaticMarkdownSourceRoute(pageRoute) ? pageRoute : null;
+}
+function prerenderedMarkdownHeaderRules(prerenderedRoutes, baseURL, describedby) {
+  const rules = /* @__PURE__ */ new Map();
+  for (const entry of prerenderedRoutes) {
+    const pageRoute = pageRouteFromMarkdownTwin(entry.fileName);
+    if (pageRoute === null)
+      continue;
+    const mdRoute = toMarkdownPath(pageRoute);
+    if (!rules.has(mdRoute)) {
+      rules.set(mdRoute, {
+        route: mdRoute,
+        headers: {
+          "Content-Type": "text/markdown; charset=utf-8",
+          "Link": buildStaticMarkdownLinkHeader(pageRoute, baseURL, describedby)
+        }
+      });
+    }
+  }
+  return [...rules.values()];
+}
+
 const DEFAULT_MAX_OUTPUT_CHARS = 1500;
 const DEFAULT_LIST_LIMIT = 20;
 const DEFAULT_SEARCH_LIMIT = 10;
@@ -1216,7 +1239,12 @@ ${details}`);
         userAgent: "*",
         // Preserve nuxt-robots' default wildcard rule so the injected group remains valid.
         disallow: [""],
-        contentSignal: [`ai-train=${config.contentSignal.aiTrain ? "yes" : "no"}`, `search=${config.contentSignal.search ? "yes" : "no"}`, `ai-input=${config.contentSignal.aiInput ? "yes" : "no"}`]
+        // Object form renders one comma-separated line; an array renders one line per entry, which validators reject.
+        contentSignal: {
+          "ai-train": config.contentSignal.aiTrain ? "yes" : "no",
+          "search": config.contentSignal.search ? "yes" : "no",
+          "ai-input": config.contentSignal.aiInput ? "yes" : "no"
+        }
       };
       if (config.contentSignal.contentUsage !== false)
         group.contentUsage = [`train-ai=${config.contentSignal.aiTrain ? "y" : "n"}`];
@@ -1579,12 +1607,13 @@ export const logger = createModuleLogger('nuxt-ai-ready', ${!!config.debug})
         bun: "#ai-ready/server/db/drizzle/providers/bun",
         d1: "#ai-ready/server/db/drizzle/providers/d1",
         libsql: "#ai-ready/server/db/drizzle/providers/libsql",
-        neon: "#ai-ready/server/db/drizzle/providers/neon"
+        neon: "#ai-ready/server/db/drizzle/providers/neon",
+        postgres: "#ai-ready/server/db/drizzle/providers/postgres"
       };
       nitroConfig.virtual["#ai-ready-virtual/db-provider.mjs"] = database._tag === "Enabled" ? `export { createClient } from '${providerMap[database.type] || providerMap.sqlite}'` : `export function createClient() {
   throw new Error('[nuxt-ai-ready] The database is disabled. Set \`aiReady.database\` to store pages at runtime.')
 }`;
-      const schemaPath = database._tag === "Enabled" && database.type === "neon" ? "#ai-ready/server/db/schema/postgres" : "#ai-ready/server/db/schema/sqlite";
+      const schemaPath = database._tag === "Enabled" && (database.type === "neon" || database.type === "postgres") ? "#ai-ready/server/db/schema/postgres" : "#ai-ready/server/db/schema/sqlite";
       nitroConfig.virtual["#ai-ready-virtual/db-schema.mjs"] = `export * from '${schemaPath}'`;
       nitroConfig.virtual["#ai-ready-virtual/devtools-meta.mjs"] = `export const devtoolsMeta = ${JSON.stringify({
         contentSignal: config.contentSignal || false,
@@ -1599,6 +1628,8 @@ export const logger = createModuleLogger('nuxt-ai-ready', ${!!config.debug})
       debugCron: config.debugCron || false,
       contentNegotiation: config.contentNegotiation === void 0 ? "auto" : config.contentNegotiation ? "enabled" : "disabled",
       mdreamOptions: config.mdreamOptions || {},
+      sitemapMd: config.sitemapMd !== false,
+      describedby: config.describedby !== false,
       markdownCacheHeaders: defu(config.markdownCacheHeaders, {
         maxAge: 3600,
         swr: true
@@ -1642,6 +1673,9 @@ export const logger = createModuleLogger('nuxt-ai-ready', ${!!config.debug})
     });
     addServerHandler({ route: "/llms.txt", handler: resolve("./runtime/server/routes/llms.txt.get"), lazy: true });
     addServerHandler({ route: "/llms-full.txt", handler: resolve("./runtime/server/routes/llms-full.txt.get"), lazy: true });
+    if (config.sitemapMd !== false) {
+      addServerHandler({ route: SITEMAP_MD_ROUTE, handler: resolve("./runtime/server/routes/sitemap.md.get"), lazy: true });
+    }
     if (agentSkillsResult._tag === "Enabled") {
       addServerHandler({ route: AGENT_SKILLS_INDEX_ROUTE, handler: resolve("./runtime/server/routes/agent-skills-index"), lazy: true });
       for (const route of Object.keys(agentSkillsResult.localArtifacts)) {
@@ -1688,7 +1722,9 @@ export const logger = createModuleLogger('nuxt-ai-ready', ${!!config.debug})
         addServerHandler({ route: "/__ai-ready/pages", handler: resolve("./runtime/server/routes/__ai-ready/pages.get"), lazy: true });
       }
     }
-    addServerHandler({ route: "/__ai-ready__/debug.json", handler: resolve("./runtime/server/routes/__ai-ready/devtools.get"), lazy: true });
+    if (nuxt.options.dev || config.debug) {
+      addServerHandler({ route: "/__ai-ready__/debug.json", handler: resolve("./runtime/server/routes/__ai-ready/devtools.get"), lazy: true });
+    }
     if (config.debug) {
       addServerHandler({ route: "/__ai-ready-debug", handler: resolve("./runtime/server/routes/__ai-ready-debug.get"), lazy: true });
     }
@@ -1697,6 +1733,7 @@ export const logger = createModuleLogger('nuxt-ai-ready', ${!!config.debug})
       addServerHandler({ route: "/__ai-ready/poll", method: "post", handler: resolve("./runtime/server/routes/__ai-ready/poll.post"), lazy: true });
       addServerHandler({ route: "/__ai-ready/prune", method: "post", handler: resolve("./runtime/server/routes/__ai-ready/prune.post"), lazy: true });
       addServerHandler({ route: "/__ai-ready/restore", method: "post", handler: resolve("./runtime/server/routes/__ai-ready/restore.post"), lazy: true });
+      addServerHandler({ route: "/__ai-ready/reindex", method: "post", handler: resolve("./runtime/server/routes/__ai-ready/reindex.post"), lazy: true });
       addServerPlugin(resolve("./runtime/server/plugins/sitemap-seeder"));
     }
     if (config.cron && !nuxt.options.dev) {
@@ -1734,19 +1771,50 @@ export const logger = createModuleLogger('nuxt-ai-ready', ${!!config.debug})
     for (const route of ["/llms.txt", "/llms-full.txt"]) {
       extendRouteRules(route, { headers: { "Content-Type": "text/plain; charset=utf-8" } });
     }
+    if (config.sitemapMd !== false) {
+      extendRouteRules(SITEMAP_MD_ROUTE, { headers: { "Content-Type": "text/markdown; charset=utf-8" } });
+    }
+    const staticBaseURL = nuxt.options.app.baseURL || "/";
+    for (const route of nuxt.options.nitro.prerender?.routes || []) {
+      if (!isStaticMarkdownSourceRoute(route))
+        continue;
+      extendRouteRules(toMarkdownPath(route), {
+        headers: {
+          "Content-Type": "text/markdown; charset=utf-8",
+          "Link": buildStaticMarkdownLinkHeader(route, staticBaseURL, config.describedby !== false)
+        }
+      });
+    }
     nuxt.hooks.hook("nitro:build:before", (nitro) => {
+      nitro.hooks.hook("prerender:done", () => {
+        for (const { route, headers } of prerenderedMarkdownHeaderRules(
+          nitro._prerenderedRoutes || [],
+          staticBaseURL,
+          config.describedby !== false
+        )) {
+          nitro.options.routeRules[route] = defu({ headers }, nitro.options.routeRules[route]);
+        }
+      });
       nitro.hooks.hook("compiled", async () => {
         const headersPath = join(nitro.options.output.publicDir, "_headers");
         logger.debug(`Checking for _headers file: ${headersPath}`);
         const exists = await access(headersPath).then(() => true).catch(() => false);
         if (exists) {
           const headers = await readFile(headersPath, "utf8");
-          const mergedHeaders = ensureStaticHeader(
+          let mergedHeaders = ensureStaticHeader(
             headers,
             "/*.md",
             "Content-Type",
             "text/markdown; charset=utf-8"
           );
+          if (config.describedby !== false) {
+            mergedHeaders = ensureStaticHeader(
+              mergedHeaders,
+              "/*.md",
+              "Link",
+              staticDescribedbyEntry(nitro.options.baseURL || "/")
+            );
+          }
           if (mergedHeaders !== headers) {
             await writeFile(headersPath, mergedHeaders);
             logger.debug("Merged .md charset header into _headers");

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nuxt-ai-ready",
   "type": "module",
-  "version": "2.0.1",
+  "version": "2.2.0",
   "description": "Best practice AI & LLM discoverability for Nuxt sites.",
   "author": {
     "name": "Harlan Wilton",
@@ -48,7 +48,8 @@
     "@libsql/client": "^0.14.0",
     "@neondatabase/serverless": "^1.0.0",
     "@nuxtjs/sitemap": ">=8.3.1",
-    "better-sqlite3": "^11.0.0 || ^12.0.0 || ^13.0.0"
+    "better-sqlite3": "^11.0.0 || ^12.0.0 || ^13.0.0",
+    "postgres": "^3.4.9"
   },
   "peerDependenciesMeta": {
     "@libsql/client": {
@@ -59,20 +60,23 @@
     },
     "better-sqlite3": {
       "optional": true
+    },
+    "postgres": {
+      "optional": true
     }
   },
   "dependencies": {
-    "@mdream/js": "^1.7.0",
+    "@mdream/js": "^1.7.1",
     "@nuxt/kit": "^4.5.2",
     "citty": "^0.2.2",
     "consola": "^3.4.2",
     "defu": "^6.1.7",
     "drizzle-orm": "1.0.0-rc.4",
-    "mdream": "^1.7.0",
+    "mdream": "^1.7.1",
     "nuxt-site-config": "^4.2.3",
     "nuxtseo-shared": "^5.3.14",
     "pathe": "^2.0.3",
-    "pkg-types": "^2.3.1",
+    "pkg-types": "^2.3.2",
     "site-config-stack": "^4.2.3",
     "sitemapd": "^0.2.2",
     "ufo": "^1.6.4",
@@ -80,43 +84,44 @@
     "yaml": "^2.9.0"
   },
   "devDependencies": {
-    "@antfu/eslint-config": "^9.3.0",
+    "@antfu/eslint-config": "^9.5.1",
     "@arethetypeswrong/cli": "^0.18.5",
-    "@harlan-zw/comark-content": "0.1.4",
-    "@libsql/client": "^0.17.4",
-    "@nuxt/content": "^3.15.2",
+    "@harlan-zw/comark-content": "^0.1.5",
+    "@libsql/client": "^0.18.0",
+    "@nuxt/content": "^3.16.0",
     "@nuxt/module-builder": "^1.0.3",
-    "@nuxt/test-utils": "^4.1.0",
+    "@nuxt/test-utils": "^4.2.0",
     "@nuxtjs/eslint-config-typescript": "^12.1.0",
     "@nuxtjs/mcp-toolkit": "^0.19.0",
-    "@nuxtjs/robots": "^6.1.5",
-    "@nuxtjs/sitemap": "^8.4.0",
+    "@nuxtjs/robots": "^6.2.0",
+    "@nuxtjs/sitemap": "^8.5.0",
     "@types/better-sqlite3": "^9.6.0",
-    "@vitest/coverage-v8": "^4.1.11",
-    "@vue/test-utils": "^2.4.11",
+    "@vitest/coverage-v8": "^5.0.0",
+    "@vue/test-utils": "^2.5.0",
     "@vueuse/nuxt": "^14.4.0",
     "better-sqlite3": "^13.0.3",
-    "bumpp": "^12.2.1",
-    "eslint": "^10.8.1",
-    "eslint-plugin-harlanzw": "^0.20.0",
+    "bumpp": "^12.3.0",
+    "eslint": "^10.10.0",
+    "eslint-plugin-harlanzw": "^0.21.1",
     "execa": "^10.0.1",
     "h3": "^1.15.11",
-    "happy-dom": "^20.11.2",
+    "happy-dom": "^20.14.0",
     "nitropack": "^2.13.4",
     "nuxt": "^4.5.2",
     "nuxt-site-config": "^4.2.3",
     "nuxtseo-layer-devtools": "^5.3.14",
-    "playwright": "^1.62.1",
-    "playwright-core": "^1.62.1",
+    "playwright": "^1.63.0",
+    "playwright-core": "^1.63.0",
+    "postgres": "^3.4.9",
     "tinyglobby": "^0.2.17",
     "typescript": "6.0.3",
     "unbuild": "^3.6.1",
-    "vitest": "^4.1.11",
-    "vue": "^3.5.41",
-    "vue-router": "^5.2.0",
-    "vue-tsc": "^3.3.10",
-    "wrangler": "^4.124.0",
-    "zod": "^4.4.3"
+    "vitest": "^5.0.0",
+    "vue": "^3.5.42",
+    "vue-router": "^5.3.1",
+    "vue-tsc": "^3.3.11",
+    "wrangler": "^4.129.0",
+    "zod": "^4.5.4"
   },
   "scripts": {
     "lint": "eslint .",