nuxt-ai-ready 3.3.0 → 3.3.1

package/dist/runtime/server/db/drizzle/queries.js

@@ -3,7 +3,7 @@ import { cronRuns, info, pages, sitemaps } from "#ai-ready-virtual/db-schema.mjs
 import { useRuntimeConfig } from "#nuxtseo/nitro";
 import { parseSitemapCrawlState, serializeSitemapCrawlState } from "../../utils/sitemap-crawl-state.js";
 import { resolveFtsTokenizer as validateFtsTokenizer } from "../schema-sql.js";
-import { deriveLocale, SEED_REFRESH_WINDOW_MS } from "../seed.js";
+import { deriveLocale, parseSourceLastmod, SEED_REFRESH_WINDOW_MS } from "../seed.js";
 import { LIKE_ESCAPE, likeSubstring, maxRowsPerInsert, toFtsMatchQuery } from "../shared.js";
 import { useDrizzle } from "./client.js";
 import { getRawExecutor, useRawDb } from "./raw.js";
@@ -234,9 +234,11 @@ export async function deleteInfoValue(event, key) {
   const client = await useDrizzle(event);
   await client.db.delete(info).where(eq(info.id, key));
 }
-const SQLITE_SCHEMA_VERSION = "v2.3.0-drizzle";
+const LEGACY_SQLITE_SCHEMA_VERSION = "v2.3.0-drizzle";
+const SQLITE_SCHEMA_VERSION = "v2.4.0-drizzle-source-lastmod";
 const LEGACY_POSTGRES_INTEGER_SCHEMA_VERSION = "v2.3.0-drizzle";
-const POSTGRES_SCHEMA_VERSION = "v2.3.0-drizzle-postgres-bigint";
+const LEGACY_POSTGRES_SCHEMA_VERSION = "v2.3.0-drizzle-postgres-bigint";
+const POSTGRES_SCHEMA_VERSION = "v2.4.0-drizzle-postgres-source-lastmod";
 export async function initSchema(event) {
   const client = await useDrizzle(event);
   const tokenizer = resolveFtsTokenizer(event);
@@ -249,6 +251,11 @@ export async function initSchema(event) {
     return;
   if (client.dialect === "postgres" && currentVersion === LEGACY_POSTGRES_INTEGER_SCHEMA_VERSION) {
     await migratePostgresIntegerTimestamps(client);
+    await migrateSourceLastmod(client, schemaVersion);
+    return;
+  }
+  if (tokenizerMatches && (client.dialect === "postgres" ? currentVersion === LEGACY_POSTGRES_SCHEMA_VERSION : currentVersion === LEGACY_SQLITE_SCHEMA_VERSION)) {
+    await migrateSourceLastmod(client, schemaVersion);
     return;
   }
   if (currentVersion) {
@@ -341,10 +348,16 @@ async function migratePostgresIntegerTimestamps(client) {
     { sql: "ALTER TABLE ai_ready_sitemaps ALTER COLUMN last_crawled_at TYPE BIGINT USING last_crawled_at::BIGINT" },
     {
       sql: "UPDATE _ai_ready_info SET version = ? WHERE id = 'schema'",
-      params: [POSTGRES_SCHEMA_VERSION]
+      params: [LEGACY_POSTGRES_SCHEMA_VERSION]
     }
   ]);
 }
+async function migrateSourceLastmod(client, schemaVersion) {
+  await getRawExecutor(client).batch([
+    { sql: "ALTER TABLE ai_ready_pages ADD COLUMN source_lastmod TEXT" },
+    { sql: "UPDATE _ai_ready_info SET version = ? WHERE id = 'schema'", params: [schemaVersion] }
+  ]);
+}
 async function createSQLiteTables(client, ftsTokenizer) {
   const statements = [
     sql`CREATE TABLE IF NOT EXISTS ai_ready_pages (
@@ -358,6 +371,7 @@ async function createSQLiteTables(client, ftsTokenizer) {
       keywords TEXT NOT NULL DEFAULT '[]',
       content_hash TEXT,
       updated_at TEXT NOT NULL,
+      source_lastmod TEXT,
       indexed_at INTEGER NOT NULL,
       is_error INTEGER NOT NULL DEFAULT 0,
       indexed INTEGER NOT NULL DEFAULT 0,
@@ -445,6 +459,7 @@ async function createPostgresTables(client) {
       content_hash TEXT,
       updated_at TEXT NOT NULL,
       indexed_at BIGINT NOT NULL,
+      source_lastmod TEXT,
       is_error INTEGER NOT NULL DEFAULT 0,
       indexed INTEGER NOT NULL DEFAULT 0,
       source TEXT NOT NULL DEFAULT 'prerender',
@@ -685,29 +700,34 @@ export async function seedRoutes(event, routes, options = {}) {
     byRoute.set(route, {
       route,
       routeKey: normalizeRouteKey(route),
-      locale: deriveLocale(event, route, explicitLocale, entryUrl)
+      locale: deriveLocale(event, route, explicitLocale, entryUrl),
+      sourceLastmod: parseSourceLastmod(typeof entry === "string" ? void 0 : entry.lastmod)
     });
   }
-  const rowsPerInsert = maxRowsPerInsert(4);
+  const rowsPerInsert = maxRowsPerInsert(5);
   const db = await useRawDb(event);
   const entries = [...byRoute.values()];
   const stmts = [];
   for (let i = 0; i < entries.length; i += rowsPerInsert) {
     const batch = entries.slice(i, i + rowsPerInsert);
-    const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', 0, 0, 0, 'runtime', ?, ?)`).join(", ");
-    const params = batch.flatMap((r) => [r.route, r.routeKey, nowMs, r.locale]);
+    const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', ?, 0, 0, 0, 'runtime', ?, ?)`).join(", ");
+    const params = batch.flatMap((r) => [r.route, r.routeKey, r.sourceLastmod, nowMs, r.locale]);
     stmts.push({
       sql: `
-        INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at, locale)
+        INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, source_lastmod, indexed_at, is_error, indexed, source, last_seen_at, locale)
         VALUES ${valuesSql}
         ON CONFLICT(route) DO UPDATE SET
           last_seen_at = excluded.last_seen_at,
           locale = excluded.locale,
+          source_lastmod = COALESCE(excluded.source_lastmod, ai_ready_pages.source_lastmod),
           is_error = 0,
-          indexed = CASE WHEN ai_ready_pages.is_error = 1 THEN 0 ELSE ai_ready_pages.indexed END
+          indexed = CASE WHEN ai_ready_pages.is_error = 1
+            OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
+            THEN 0 ELSE ai_ready_pages.indexed END
         WHERE ai_ready_pages.last_seen_at IS NULL
           OR ai_ready_pages.last_seen_at < excluded.last_seen_at - ${refreshWindowMs}
           OR ai_ready_pages.locale IS DISTINCT FROM excluded.locale
+          OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
       `,
       params
     });

package/dist/runtime/server/db/queries.d.ts

@@ -1,6 +1,6 @@
 import type { SitemapCrawlState } from '../utils/sitemap-crawl-state.js';
 import type { AiReadyDatabaseEvent } from './context.js';
-import type { SeedRoutesOptions } from './seed.js';
+import type { SeedRoute, SeedRoutesOptions } from './seed.js';
 export type { SeedRoutesOptions };
 export { resolveSeedRefreshWindowMs, SEED_REFRESH_WINDOW_MS } from './seed.js';
 export interface PageRow {
@@ -145,11 +145,7 @@ export declare function getPageHash(event: AiReadyDatabaseEvent | undefined, rou
 /**
  * Seed routes from sitemap (insert with indexed=0 if not exists)
  */
-export declare function seedRoutes(event: AiReadyDatabaseEvent | undefined, routes: Array<string | {
-    route: string;
-    locale?: string;
-    url?: string;
-}>, options?: SeedRoutesOptions): Promise<number>;
+export declare function seedRoutes(event: AiReadyDatabaseEvent | undefined, routes: SeedRoute[], options?: SeedRoutesOptions): Promise<number>;
 /** Remove a page after its route returns HTTP 410. */
 export declare function prunePage(event: AiReadyDatabaseEvent | undefined, route: string): Promise<void>;
 /**

package/dist/runtime/server/db/queries.js

@@ -3,7 +3,7 @@ import { useEvent, useRuntimeConfig } from "#nuxtseo/nitro";
 import { logger } from "../logger.js";
 import { checkAndHandleStale } from "../utils/checkStale.js";
 import { parseSitemapCrawlState, serializeSitemapCrawlState } from "../utils/sitemap-crawl-state.js";
-import { deriveLocale, SEED_REFRESH_WINDOW_MS } from "./seed.js";
+import { deriveLocale, parseSourceLastmod, SEED_REFRESH_WINDOW_MS } from "./seed.js";
 import { LIKE_ESCAPE, likeSubstring, maxRowsPerInsert, normalizeRoute, normalizeRouteKey, toFtsMatchQuery } from "./shared.js";
 export { resolveSeedRefreshWindowMs, SEED_REFRESH_WINDOW_MS } from "./seed.js";
 function getEventFromContext(providedEvent) {
@@ -409,26 +409,31 @@ export async function seedRoutes(event, routes, options = {}) {
     byRoute.set(route, {
       route,
       routeKey: normalizeRouteKey(route),
-      locale: deriveLocale(event, route, explicitLocale, entryUrl)
+      locale: deriveLocale(event, route, explicitLocale, entryUrl),
+      sourceLastmod: parseSourceLastmod(typeof entry === "string" ? void 0 : entry.lastmod)
     });
   }
-  const rowsPerInsert = maxRowsPerInsert(4);
+  const rowsPerInsert = maxRowsPerInsert(5);
   const stmts = [];
   for (const batch of chunk([...byRoute.values()], rowsPerInsert)) {
-    const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', 0, 0, 0, 'runtime', ?, ?)`).join(", ");
-    const params = batch.flatMap((r) => [r.route, r.routeKey, nowMs, r.locale]);
+    const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', ?, 0, 0, 0, 'runtime', ?, ?)`).join(", ");
+    const params = batch.flatMap((r) => [r.route, r.routeKey, r.sourceLastmod, nowMs, r.locale]);
     stmts.push({
       sql: `
-        INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at, locale)
+        INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, source_lastmod, indexed_at, is_error, indexed, source, last_seen_at, locale)
         VALUES ${valuesSql}
         ON CONFLICT(route) DO UPDATE SET
           last_seen_at = excluded.last_seen_at,
           locale = excluded.locale,
+          source_lastmod = COALESCE(excluded.source_lastmod, ai_ready_pages.source_lastmod),
           is_error = 0,
-          indexed = CASE WHEN ai_ready_pages.is_error = 1 THEN 0 ELSE ai_ready_pages.indexed END
+          indexed = CASE WHEN ai_ready_pages.is_error = 1
+            OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
+            THEN 0 ELSE ai_ready_pages.indexed END
         WHERE ai_ready_pages.last_seen_at IS NULL
           OR ai_ready_pages.last_seen_at < excluded.last_seen_at - ${refreshWindowMs}
           OR ai_ready_pages.locale IS DISTINCT FROM excluded.locale
+          OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
       `,
       params
     });

package/dist/runtime/server/db/schema/postgres.d.ts

@@ -152,6 +152,21 @@ export declare const pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
             identity: undefined;
             generated: undefined;
         }>;
+        sourceLastmod: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").PgTextBuilder<[string, ...string[]]>, {
+            name: string;
+            tableName: "ai_ready_pages";
+            dataType: "string";
+            data: string;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            identity: undefined;
+            generated: undefined;
+        }>;
         indexedAt: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgBigInt53Builder>, {
             name: string;
             tableName: "ai_ready_pages";
@@ -721,6 +736,21 @@ export declare const schema: {
                 identity: undefined;
                 generated: undefined;
             }>;
+            sourceLastmod: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").PgTextBuilder<[string, ...string[]]>, {
+                name: string;
+                tableName: "ai_ready_pages";
+                dataType: "string";
+                data: string;
+                driverParam: string;
+                notNull: false;
+                hasDefault: false;
+                isPrimaryKey: false;
+                isAutoincrement: false;
+                hasRuntimeDefault: false;
+                enumValues: undefined;
+                identity: undefined;
+                generated: undefined;
+            }>;
             indexedAt: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgBigInt53Builder>, {
                 name: string;
                 tableName: "ai_ready_pages";

package/dist/runtime/server/db/schema/postgres.js

@@ -10,6 +10,7 @@ export const pages = pgTable("ai_ready_pages", {
   keywords: text("keywords").notNull().default("[]"),
   contentHash: text("content_hash"),
   updatedAt: text("updated_at").notNull(),
+  sourceLastmod: text("source_lastmod"),
   indexedAt: bigint("indexed_at", { mode: "number" }).notNull(),
   isError: integer("is_error").notNull().default(0),
   indexed: integer("indexed").notNull().default(0),

package/dist/runtime/server/db/schema/sqlite.d.ts

@@ -162,6 +162,22 @@ export declare const pages: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
             identity: undefined;
             generated: undefined;
         }, {}>;
+        sourceLastmod: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "ai_ready_pages";
+            dataType: "string";
+            data: string;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
         indexedAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
             name: string;
             tableName: "ai_ready_pages";
@@ -767,6 +783,22 @@ export declare const schema: {
                 identity: undefined;
                 generated: undefined;
             }, {}>;
+            sourceLastmod: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+                name: string;
+                tableName: "ai_ready_pages";
+                dataType: "string";
+                data: string;
+                driverParam: string;
+                notNull: false;
+                hasDefault: false;
+                isPrimaryKey: false;
+                isAutoincrement: false;
+                hasRuntimeDefault: false;
+                enumValues: [string, ...string[]];
+                baseColumn: never;
+                identity: undefined;
+                generated: undefined;
+            }, {}>;
             indexedAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
                 name: string;
                 tableName: "ai_ready_pages";

package/dist/runtime/server/db/schema/sqlite.js

@@ -10,6 +10,7 @@ export const pages = sqliteTable("ai_ready_pages", {
   keywords: text("keywords").notNull().default("[]"),
   contentHash: text("content_hash"),
   updatedAt: text("updated_at").notNull(),
+  sourceLastmod: text("source_lastmod"),
   indexedAt: integer("indexed_at").notNull(),
   isError: integer("is_error").notNull().default(0),
   indexed: integer("indexed").notNull().default(0),

package/dist/runtime/server/db/schema-sql.d.ts

@@ -2,7 +2,7 @@
  * SQLite schema definitions for build-time prerendering
  * Used by shared.ts during nuxi generate/build
  */
-export declare const SCHEMA_VERSION = "v2.3.0";
+export declare const SCHEMA_VERSION = "v2.4.0";
 /**
  * Allowed FTS5 tokenizer values. The configured value gets interpolated into the
  * `CREATE VIRTUAL TABLE … tokenize='…'` DDL, so we restrict it to a known-safe

package/dist/runtime/server/db/schema-sql.js

@@ -1,4 +1,4 @@
-export const SCHEMA_VERSION = "v2.3.0";
+export const SCHEMA_VERSION = "v2.4.0";
 export const SUPPORTED_FTS_TOKENIZERS = [
   "unicode61 remove_diacritics 2",
   "trigram"
@@ -24,6 +24,7 @@ export function buildSchemaSql(opts = {}) {
       keywords TEXT NOT NULL DEFAULT '[]',
       content_hash TEXT,
       updated_at TEXT NOT NULL,
+      source_lastmod TEXT,
       indexed_at INTEGER NOT NULL,
       is_error INTEGER NOT NULL DEFAULT 0,
       indexed INTEGER NOT NULL DEFAULT 0,

package/dist/runtime/server/db/seed.d.ts

@@ -30,3 +30,11 @@ export interface SeedRoutesOptions {
      */
     refreshWindowMs?: number;
 }
+export type SeedRoute = string | {
+    route: string;
+    locale?: string;
+    url?: string;
+    lastmod?: string | Date;
+};
+/** Parse source dates once, so equivalent W3C date representations compare equal. */
+export declare function parseSourceLastmod(input: string | Date | undefined): string | null;

package/dist/runtime/server/db/seed.js

@@ -39,3 +39,19 @@ export function resolveSeedRefreshWindowMs(pruneTtlSeconds) {
     return SEED_REFRESH_WINDOW_MS;
   return Math.min(SEED_REFRESH_WINDOW_MS, Math.floor(pruneTtlSeconds * 1e3 / 2));
 }
+export function parseSourceLastmod(input) {
+  if (input === void 0)
+    return null;
+  if (typeof input === "string" && (input.length > 64 || !/^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2}))?$/.test(input)))
+    return null;
+  const date = new Date(input);
+  if (!Number.isFinite(date.getTime()))
+    return null;
+  if (typeof input === "string") {
+    const calendarDate = input.slice(0, 10);
+    const parsedCalendarDate = new Date(calendarDate);
+    if (!Number.isFinite(parsedCalendarDate.getTime()) || parsedCalendarDate.toISOString().slice(0, 10) !== calendarDate)
+      return null;
+  }
+  return date.toISOString();
+}

package/dist/runtime/server/db/shared.d.ts

@@ -1,8 +1,8 @@
 /**
  * Compute content hash for change detection (first 16 chars of SHA-256).
- * Hash the page body only. Frontmatter holds per-conversion fields
- * (`canonical_url`, `last_updated`) that differ between the build and runtime
- * paths, so hashing it marks unchanged pages as changed.
+ * Hash the page body and image metadata. Other frontmatter holds fields
+ * (`canonical_url`, `last_updated`) that can differ between conversion paths.
+ * Those fields stay excluded so unchanged pages keep the same hash.
  */
 export declare function computeContentHash(markdown: string): Promise<string>;
 export interface DatabaseAdapter {

package/dist/runtime/server/db/shared.js

@@ -10,7 +10,25 @@ function stripPageFrontmatter(markdown) {
   if (document.errors.length || !isMap(document.contents))
     return markdown;
   const isPageFrontmatter = document.contents.items.some((item) => isScalar(item.key) && PAGE_FRONTMATTER_KEYS.has(String(item.key.value)));
-  return isPageFrontmatter ? markdown.slice(match[0].length) : markdown;
+  if (!isPageFrontmatter)
+    return markdown;
+  const body = markdown.slice(match[0].length);
+  const meta = document.get("meta", true);
+  const images = [];
+  if (isMap(meta)) {
+    for (const { key, value } of meta.items) {
+      if (!isScalar(key) || typeof key.value !== "string" || !isScalar(value))
+        continue;
+      if (key.value !== "og:image" && !key.value.startsWith("og:image:") && key.value !== "twitter:image" && !key.value.startsWith("twitter:image:"))
+        continue;
+      if (typeof value.value === "string" || typeof value.value === "number" && Number.isFinite(value.value))
+        images.push([key.value, value.value]);
+    }
+  }
+  if (!images.length)
+    return body;
+  images.sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
+  return JSON.stringify([images, body.trim()]);
 }
 export async function computeContentHash(markdown) {
   const encoder = new TextEncoder();

package/dist/runtime/server/plugins/sitemap-seeder.js

@@ -2,8 +2,10 @@ import { useRuntimeConfig } from "#nuxtseo/nitro";
 import { trackDrizzleWork } from "../db/drizzle/client.js";
 import { getPageLastmods, getSitemapLastCrawledAt, markSitemapSeeded, resolveSeedRefreshWindowMs, seedRoutes } from "../db/queries.js";
 import { logger } from "../logger.js";
+import { markIndexedSitemapLastmods } from "../utils/sitemap-lastmod.js";
 import { mapSitemapRoutes } from "../utils/sitemap-routes.js";
 const DIAGNOSTICS_KEY = "_aiReadySitemapDiagnostics";
+const INDEXED_LASTMODS_KEY = "_aiReadyIndexedSitemapLastmods";
 function isDebug(_event) {
   return !!useRuntimeConfig()["nuxt-ai-ready"]?.debug;
 }
@@ -66,6 +68,7 @@ export default function sitemapSeederPlugin(nitroApp) {
     const routeToUrl = mapSitemapRoutes(urls);
     if (routeToUrl.size === 0)
       return;
+    const routes = [...routeToUrl.entries()].map(([route, url]) => ({ route, url: url.loc, lastmod: url.lastmod }));
     logger.debug(`[sitemap-seeder] Processing ${routeToUrl.size} routes from ${sitemapName}`);
     const record = isDebug(event) ? (m) => recordDiagnostic(event, m) : void 0;
     const readStart = Date.now();
@@ -75,6 +78,9 @@ export default function sitemapSeederPlugin(nitroApp) {
       const lastmod = lastmods.get(route);
       if (lastmod && !url.lastmod) {
         url.lastmod = lastmod;
+        const context = event.context;
+        const enrichedUrls = context[INDEXED_LASTMODS_KEY] ??= /* @__PURE__ */ new Set();
+        enrichedUrls.add(url.loc);
         enriched++;
       }
     }
@@ -92,7 +98,6 @@ export default function sitemapSeederPlugin(nitroApp) {
     if (Date.now() - lastSeed < SEED_INTERVAL_MS)
       return;
     lastSeedAt.set(sitemapName, Date.now());
-    const routes = [...routeToUrl.entries()].map(([route, url]) => ({ route, url: url.loc }));
     const urlCount = urls.length;
     const seed = async () => {
       const seedStart = Date.now();
@@ -120,6 +125,9 @@ export default function sitemapSeederPlugin(nitroApp) {
     }
   });
   nitroApp.hooks.hook("sitemap:output", (ctx) => {
+    const enrichedUrls = ctx.event.context[INDEXED_LASTMODS_KEY];
+    if (enrichedUrls?.size)
+      ctx.sitemap = markIndexedSitemapLastmods(ctx.sitemap, enrichedUrls);
     const diagnostics = ctx.event.context[DIAGNOSTICS_KEY];
     if (!diagnostics?.length)
       return;

package/dist/runtime/server/routes/llms.txt.get.js

@@ -20,13 +20,15 @@ function getBuildLlmsTxtCached(maxAge) {
   return cached;
 }
 export default eventHandler(async (event) => {
-  assertMethod(event, "GET", true);
   const runtimeConfig = useRuntimeConfig(event);
+  const path = toLogicalRoute(event.path.split("?")[0], runtimeConfig.app.baseURL);
+  if (path !== "/llms.txt" && !/^\/llms\/[^/]+\.txt$/i.test(path))
+    return;
+  assertMethod(event, "GET", true);
   const config = runtimeConfig["nuxt-ai-ready"];
   const cacheSeconds = config.llmsTxtCacheSeconds;
   const cacheEnabled = !import.meta.dev && !import.meta.prerender && cacheSeconds > 0;
   const files = cacheEnabled ? await getBuildLlmsTxtCached(cacheSeconds)(event) : await buildLlmsTxtFiles(event);
-  const path = toLogicalRoute(event.path.split("?")[0], runtimeConfig.app.baseURL);
   const content = files[path];
   if (content === void 0)
     throw createError({ statusCode: 404, statusMessage: "llms.txt file not found" });

package/dist/runtime/server/utils/llms-full.js

@@ -1,15 +1,41 @@
 import { joinURL } from "ufo";
+import { isMap, isScalar, parseDocument, stringify } from "yaml";
 import { normalizeLlmsTxtConfig } from "../../llms-txt-format.js";
-const RE_FRONTMATTER = /^---\r?\n[\s\S]*?\r?\n---\r?\n*/;
+import { escapeMarkdownHtml } from "../../markdown-text.js";
+const RE_FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---\r?\n*/;
 const RE_INLINE_WHITESPACE = /\s+/g;
+function imageMetadataFence(frontmatter) {
+  const document = parseDocument(frontmatter);
+  const meta = document.get("meta", true);
+  if (document.errors.length || !isMap(meta))
+    return;
+  const images = {};
+  for (const { key, value } of meta.items) {
+    if (!isScalar(key) || typeof key.value !== "string" || !isScalar(value))
+      continue;
+    if (key.value !== "og:image" && !key.value.startsWith("og:image:") && key.value !== "twitter:image" && !key.value.startsWith("twitter:image:"))
+      continue;
+    if (typeof value.value === "string" || typeof value.value === "number" && Number.isFinite(value.value))
+      images[key.value] = value.value;
+  }
+  if (!Object.keys(images).length)
+    return;
+  return `\`\`\`yaml
+${stringify({ meta: images }, { lineWidth: 0 }).trimEnd()}
+\`\`\``;
+}
 export function formatPageForLlmsFullTxt(route, title, description, markdown, siteUrl) {
   const canonicalUrl = siteUrl ? joinURL(siteUrl, route) : route;
-  const pageTitle = (title && title !== route ? title : route).trim().replace(RE_INLINE_WHITESPACE, " ");
-  const pageDescription = description.trim().replace(RE_INLINE_WHITESPACE, " ");
+  const pageTitle = escapeMarkdownHtml((title && title !== route ? title : route).trim().replace(RE_INLINE_WHITESPACE, " "));
+  const pageDescription = escapeMarkdownHtml(description.trim().replace(RE_INLINE_WHITESPACE, " "));
   const content = markdown.replace(RE_FRONTMATTER, "").trim();
   const parts = ["---", "", `- **Page:** ${pageTitle}`, `- **Source:** ${canonicalUrl}`];
   if (pageDescription)
     parts.push(`- **Description:** ${pageDescription}`);
+  const frontmatter = markdown.match(RE_FRONTMATTER)?.[1];
+  const images = frontmatter && imageMetadataFence(frontmatter);
+  if (images)
+    parts.push("", images);
   parts.push("");
   if (content) {
     parts.push(content);
@@ -19,9 +45,9 @@ export function formatPageForLlmsFullTxt(route, title, description, markdown, si
 `;
 }
 export function buildLlmsFullTxtHeader(siteInfo, llmsTxtConfig) {
-  const parts = [`# ${siteInfo?.name || siteInfo?.url || "Site"}`];
+  const parts = [`# ${escapeMarkdownHtml(siteInfo?.name || siteInfo?.url || "Site")}`];
   if (siteInfo?.description)
-    parts.push("", `> ${siteInfo.description}`);
+    parts.push("", `> ${escapeMarkdownHtml(siteInfo.description)}`);
   if (siteInfo?.url)
     parts.push("", `Canonical Origin: ${siteInfo.url}`);
   if (llmsTxtConfig) {

package/dist/runtime/server/utils/negotiation-response.js

@@ -52,6 +52,9 @@ export function setUncacheableHeaders(event) {
 }
 export function setMarkdownHeaders(event, ctx, sourceHeaders) {
   setHeader(event, "content-type", "text/markdown; charset=utf-8");
+  const sourcePolicy = sourceHeaders?.get("content-signal");
+  if (sourcePolicy !== null && sourcePolicy !== void 0 && getResponseHeader(event, "content-signal") === void 0)
+    setHeader(event, "content-signal", sourcePolicy);
   const vary = [getResponseHeader(event, "vary"), sourceHeaders?.get("vary")].filter(Boolean).join(", ");
   setHeader(event, "vary", mergeVaryHeader(vary, "Accept"));
   setLinkHeader(event, ctx, "markdown");

package/dist/runtime/server/utils/runCron.js

@@ -170,7 +170,7 @@ async function pingSitemap(event, config, sitemapIntervalMinutes, debug) {
     route: nextSitemap.route,
     state: nextSitemap.crawlState
   });
-  const routes = [...mapSitemapRoutes(result.urls).entries()].map(([route, url]) => ({ route, url: url.loc }));
+  const routes = [...mapSitemapRoutes(result.urls).entries()].map(([route, url]) => ({ route, url: url.loc, lastmod: url.lastmod }));
   if (routes.length > 0)
     await seedRoutes(event, routes, { refreshWindowMs });
   if (result._tag === "failed") {

package/dist/runtime/server/utils/sitemap-lastmod.d.ts

@@ -0,0 +1,4 @@
+/** Mark dates added by this module without changing the public sitemap date. */
+export declare function markIndexedSitemapLastmods(xml: string, enrichedUrls: ReadonlySet<string>): string;
+/** The source crawl must never treat our indexed date as a CMS change. */
+export declare function stripIndexedSitemapLastmods(body: Uint8Array): Uint8Array;

package/dist/runtime/server/utils/sitemap-lastmod.js

@@ -0,0 +1,19 @@
+const INDEXED_LASTMOD_MARKER = "<!-- nuxt-ai-ready:indexed-lastmod -->";
+export function markIndexedSitemapLastmods(xml, enrichedUrls) {
+  const escapedUrls = new Set([...enrichedUrls].map((url) => url.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&apos;")));
+  return xml.replace(/<url>([\s\S]*?)<\/url>/g, (entry, body) => {
+    const loc = body.match(/<loc>([^<]*)<\/loc>/)?.[1];
+    return loc && escapedUrls.has(loc) ? entry.replace("<lastmod>", `${INDEXED_LASTMOD_MARKER}<lastmod>`) : entry;
+  });
+}
+export function stripIndexedSitemapLastmods(body) {
+  let xml;
+  try {
+    xml = new TextDecoder("utf-8", { fatal: true }).decode(body);
+  } catch {
+    return body;
+  }
+  if (!xml.includes(INDEXED_LASTMOD_MARKER))
+    return body;
+  return new TextEncoder().encode(xml.replace(/<!-- nuxt-ai-ready:indexed-lastmod -->\s*<lastmod>[^<]*<\/lastmod>/g, ""));
+}

package/dist/runtime/server/utils/sitemap-md.js

@@ -1,11 +1,12 @@
 import { toMarkdownPath } from "../../markdown-path.js";
+import { escapeMarkdownHtml } from "../../markdown-text.js";
 import { toDeployedRoute } from "../../route-path.js";
 export const SITEMAP_MD_ROUTE = "/sitemap.md";
 function topSegment(route) {
   return route.split("/").filter(Boolean)[0] || "";
 }
 function escapeTitle(title) {
-  return title.replaceAll("[", "\\[").replaceAll("]", "\\]");
+  return escapeMarkdownHtml(title).replaceAll("[", "\\[").replaceAll("]", "\\]");
 }
 function formatEntry(entry, resolveHref) {
   const title = escapeTitle(entry.title?.trim() || entry.route);
@@ -35,7 +36,7 @@ export function buildSitemapMd(entries, options = {}) {
       return 1;
     return a.localeCompare(b);
   });
-  const parts = [`# ${options.siteName || "Site"} Sitemap`, "", "All pages in Markdown format."];
+  const parts = [`# ${escapeMarkdownHtml(options.siteName || "Site")} Sitemap`, "", "All pages in Markdown format."];
   for (const key of keys) {
     parts.push("", `## ${key || "Root"}`);
     for (const entry of groups.get(key))

package/dist/runtime/server/utils/sitemap.js

@@ -7,6 +7,7 @@ import { logger } from "../logger.js";
 import { getCfEnv, hasAssets } from "./cloudflare.js";
 import { createUniversalContext } from "./context.js";
 import { fetchRawWithEvent } from "./fetch.js";
+import { stripIndexedSitemapLastmods } from "./sitemap-lastmod.js";
 const FETCH_TIMEOUT = 15e3;
 const SITEMAP_FALLBACK_ORIGIN = "http://localhost";
 const SITEMAP_READER_LIMITS = {
@@ -221,11 +222,12 @@ function createLocalSitemapLoader(event, usePublicAsset, loadedBodies) {
         bytesRead: read.result.bytesRead
       };
     }
-    loadedBodies.set(request.url, read.result.body);
+    const sourceBody = stripIndexedSitemapLastmods(read.result.body);
+    loadedBodies.set(request.url, sourceBody);
     return {
       _tag: "body",
       url: request.url,
-      body: read.result.body
+      body: sourceBody
     };
   };
 }