nuxt-ai-ready 3.2.0 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/chunks/agent-skills.mjs +2 -1
  2. package/dist/chunks/prerender.mjs +197 -28
  3. package/dist/cli.mjs +259 -0
  4. package/dist/module.d.mts +1 -0
  5. package/dist/module.json +1 -1
  6. package/dist/module.mjs +2 -1
  7. package/dist/runtime/llms-txt-files.d.ts +30 -0
  8. package/dist/runtime/llms-txt-files.js +196 -0
  9. package/dist/runtime/llms-txt-format.js +5 -4
  10. package/dist/runtime/llms-txt-utils.d.ts +3 -2
  11. package/dist/runtime/llms-txt-utils.js +85 -224
  12. package/dist/runtime/markdown-text.d.ts +2 -0
  13. package/dist/runtime/markdown-text.js +3 -0
  14. package/dist/runtime/server/db/drizzle/queries.d.ts +2 -6
  15. package/dist/runtime/server/db/drizzle/queries.js +30 -10
  16. package/dist/runtime/server/db/queries.d.ts +5 -6
  17. package/dist/runtime/server/db/queries.js +40 -9
  18. package/dist/runtime/server/db/schema/postgres.d.ts +30 -0
  19. package/dist/runtime/server/db/schema/postgres.js +1 -0
  20. package/dist/runtime/server/db/schema/sqlite.d.ts +32 -0
  21. package/dist/runtime/server/db/schema/sqlite.js +1 -0
  22. package/dist/runtime/server/db/schema-sql.d.ts +1 -1
  23. package/dist/runtime/server/db/schema-sql.js +2 -1
  24. package/dist/runtime/server/db/seed.d.ts +8 -0
  25. package/dist/runtime/server/db/seed.js +16 -0
  26. package/dist/runtime/server/db/shared.d.ts +3 -3
  27. package/dist/runtime/server/db/shared.js +19 -1
  28. package/dist/runtime/server/middleware/markdown.js +1 -1
  29. package/dist/runtime/server/plugins/sitemap-seeder.js +9 -1
  30. package/dist/runtime/server/routes/llms-full.txt.get.js +2 -1
  31. package/dist/runtime/server/routes/llms.txt.get.js +21 -11
  32. package/dist/runtime/server/routes/sitemap.md.get.js +2 -1
  33. package/dist/runtime/server/utils/llms-full.js +31 -5
  34. package/dist/runtime/server/utils/markdown-request.js +2 -5
  35. package/dist/runtime/server/utils/negotiation-response.js +3 -0
  36. package/dist/runtime/server/utils/runCron.js +1 -1
  37. package/dist/runtime/server/utils/sitemap-lastmod.d.ts +4 -0
  38. package/dist/runtime/server/utils/sitemap-lastmod.js +19 -0
  39. package/dist/runtime/server/utils/sitemap-md.js +3 -2
  40. package/dist/runtime/server/utils/sitemap.d.ts +1 -0
  41. package/dist/runtime/server/utils/sitemap.js +26 -12
  42. package/dist/runtime/server/utils.js +50 -2
  43. package/dist/runtime/types.d.ts +13 -1
  44. package/dist/shared/{nuxt-ai-ready.DH9mSayy.mjs → nuxt-ai-ready.CKXM1jnE.mjs} +40 -12
  45. package/package.json +3 -3
  46. package/skills/nuxt-ai-ready/SKILL.md +34 -1
@@ -1,6 +1,6 @@
1
1
  import type { SitemapCrawlState } from '../../utils/sitemap-crawl-state.js';
2
2
  import type { AiReadyDatabaseEvent } from '../context.js';
3
- import type { SeedRoutesOptions } from '../seed.js';
3
+ import type { SeedRoute, SeedRoutesOptions } from '../seed.js';
4
4
  export interface PageInput {
5
5
  route: string;
6
6
  title: string;
@@ -182,9 +182,5 @@ export declare function resetSitemapErrors(event: AiReadyDatabaseEvent | undefin
182
182
  * guard each run bumped `last_seen_at` on every row, which cost millions of
183
183
  * D1 row writes a month. Keep the two layers in sync.
184
184
  */
185
- export declare function seedRoutes(event: AiReadyDatabaseEvent | undefined, routes: Array<string | {
186
- route: string;
187
- locale?: string;
188
- url?: string;
189
- }>, options?: SeedRoutesOptions): Promise<number>;
185
+ export declare function seedRoutes(event: AiReadyDatabaseEvent | undefined, routes: SeedRoute[], options?: SeedRoutesOptions): Promise<number>;
190
186
  export {};
@@ -3,7 +3,7 @@ import { cronRuns, info, pages, sitemaps } from "#ai-ready-virtual/db-schema.mjs
3
3
  import { useRuntimeConfig } from "#nuxtseo/nitro";
4
4
  import { parseSitemapCrawlState, serializeSitemapCrawlState } from "../../utils/sitemap-crawl-state.js";
5
5
  import { resolveFtsTokenizer as validateFtsTokenizer } from "../schema-sql.js";
6
- import { deriveLocale, SEED_REFRESH_WINDOW_MS } from "../seed.js";
6
+ import { deriveLocale, parseSourceLastmod, SEED_REFRESH_WINDOW_MS } from "../seed.js";
7
7
  import { LIKE_ESCAPE, likeSubstring, maxRowsPerInsert, toFtsMatchQuery } from "../shared.js";
8
8
  import { useDrizzle } from "./client.js";
9
9
  import { getRawExecutor, useRawDb } from "./raw.js";
@@ -234,9 +234,11 @@ export async function deleteInfoValue(event, key) {
234
234
  const client = await useDrizzle(event);
235
235
  await client.db.delete(info).where(eq(info.id, key));
236
236
  }
237
- const SQLITE_SCHEMA_VERSION = "v2.3.0-drizzle";
237
+ const LEGACY_SQLITE_SCHEMA_VERSION = "v2.3.0-drizzle";
238
+ const SQLITE_SCHEMA_VERSION = "v2.4.0-drizzle-source-lastmod";
238
239
  const LEGACY_POSTGRES_INTEGER_SCHEMA_VERSION = "v2.3.0-drizzle";
239
- const POSTGRES_SCHEMA_VERSION = "v2.3.0-drizzle-postgres-bigint";
240
+ const LEGACY_POSTGRES_SCHEMA_VERSION = "v2.3.0-drizzle-postgres-bigint";
241
+ const POSTGRES_SCHEMA_VERSION = "v2.4.0-drizzle-postgres-source-lastmod";
240
242
  export async function initSchema(event) {
241
243
  const client = await useDrizzle(event);
242
244
  const tokenizer = resolveFtsTokenizer(event);
@@ -249,6 +251,11 @@ export async function initSchema(event) {
249
251
  return;
250
252
  if (client.dialect === "postgres" && currentVersion === LEGACY_POSTGRES_INTEGER_SCHEMA_VERSION) {
251
253
  await migratePostgresIntegerTimestamps(client);
254
+ await migrateSourceLastmod(client, schemaVersion);
255
+ return;
256
+ }
257
+ if (tokenizerMatches && (client.dialect === "postgres" ? currentVersion === LEGACY_POSTGRES_SCHEMA_VERSION : currentVersion === LEGACY_SQLITE_SCHEMA_VERSION)) {
258
+ await migrateSourceLastmod(client, schemaVersion);
252
259
  return;
253
260
  }
254
261
  if (currentVersion) {
@@ -341,10 +348,16 @@ async function migratePostgresIntegerTimestamps(client) {
341
348
  { sql: "ALTER TABLE ai_ready_sitemaps ALTER COLUMN last_crawled_at TYPE BIGINT USING last_crawled_at::BIGINT" },
342
349
  {
343
350
  sql: "UPDATE _ai_ready_info SET version = ? WHERE id = 'schema'",
344
- params: [POSTGRES_SCHEMA_VERSION]
351
+ params: [LEGACY_POSTGRES_SCHEMA_VERSION]
345
352
  }
346
353
  ]);
347
354
  }
355
+ async function migrateSourceLastmod(client, schemaVersion) {
356
+ await getRawExecutor(client).batch([
357
+ { sql: "ALTER TABLE ai_ready_pages ADD COLUMN source_lastmod TEXT" },
358
+ { sql: "UPDATE _ai_ready_info SET version = ? WHERE id = 'schema'", params: [schemaVersion] }
359
+ ]);
360
+ }
348
361
  async function createSQLiteTables(client, ftsTokenizer) {
349
362
  const statements = [
350
363
  sql`CREATE TABLE IF NOT EXISTS ai_ready_pages (
@@ -358,6 +371,7 @@ async function createSQLiteTables(client, ftsTokenizer) {
358
371
  keywords TEXT NOT NULL DEFAULT '[]',
359
372
  content_hash TEXT,
360
373
  updated_at TEXT NOT NULL,
374
+ source_lastmod TEXT,
361
375
  indexed_at INTEGER NOT NULL,
362
376
  is_error INTEGER NOT NULL DEFAULT 0,
363
377
  indexed INTEGER NOT NULL DEFAULT 0,
@@ -445,6 +459,7 @@ async function createPostgresTables(client) {
445
459
  content_hash TEXT,
446
460
  updated_at TEXT NOT NULL,
447
461
  indexed_at BIGINT NOT NULL,
462
+ source_lastmod TEXT,
448
463
  is_error INTEGER NOT NULL DEFAULT 0,
449
464
  indexed INTEGER NOT NULL DEFAULT 0,
450
465
  source TEXT NOT NULL DEFAULT 'prerender',
@@ -685,29 +700,34 @@ export async function seedRoutes(event, routes, options = {}) {
685
700
  byRoute.set(route, {
686
701
  route,
687
702
  routeKey: normalizeRouteKey(route),
688
- locale: deriveLocale(event, route, explicitLocale, entryUrl)
703
+ locale: deriveLocale(event, route, explicitLocale, entryUrl),
704
+ sourceLastmod: parseSourceLastmod(typeof entry === "string" ? void 0 : entry.lastmod)
689
705
  });
690
706
  }
691
- const rowsPerInsert = maxRowsPerInsert(4);
707
+ const rowsPerInsert = maxRowsPerInsert(5);
692
708
  const db = await useRawDb(event);
693
709
  const entries = [...byRoute.values()];
694
710
  const stmts = [];
695
711
  for (let i = 0; i < entries.length; i += rowsPerInsert) {
696
712
  const batch = entries.slice(i, i + rowsPerInsert);
697
- const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', 0, 0, 0, 'runtime', ?, ?)`).join(", ");
698
- const params = batch.flatMap((r) => [r.route, r.routeKey, nowMs, r.locale]);
713
+ const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', ?, 0, 0, 0, 'runtime', ?, ?)`).join(", ");
714
+ const params = batch.flatMap((r) => [r.route, r.routeKey, r.sourceLastmod, nowMs, r.locale]);
699
715
  stmts.push({
700
716
  sql: `
701
- INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at, locale)
717
+ INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, source_lastmod, indexed_at, is_error, indexed, source, last_seen_at, locale)
702
718
  VALUES ${valuesSql}
703
719
  ON CONFLICT(route) DO UPDATE SET
704
720
  last_seen_at = excluded.last_seen_at,
705
721
  locale = excluded.locale,
722
+ source_lastmod = COALESCE(excluded.source_lastmod, ai_ready_pages.source_lastmod),
706
723
  is_error = 0,
707
- indexed = CASE WHEN ai_ready_pages.is_error = 1 THEN 0 ELSE ai_ready_pages.indexed END
724
+ indexed = CASE WHEN ai_ready_pages.is_error = 1
725
+ OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
726
+ THEN 0 ELSE ai_ready_pages.indexed END
708
727
  WHERE ai_ready_pages.last_seen_at IS NULL
709
728
  OR ai_ready_pages.last_seen_at < excluded.last_seen_at - ${refreshWindowMs}
710
729
  OR ai_ready_pages.locale IS DISTINCT FROM excluded.locale
730
+ OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
711
731
  `,
712
732
  params
713
733
  });
@@ -1,6 +1,6 @@
1
1
  import type { SitemapCrawlState } from '../utils/sitemap-crawl-state.js';
2
2
  import type { AiReadyDatabaseEvent } from './context.js';
3
- import type { SeedRoutesOptions } from './seed.js';
3
+ import type { SeedRoute, SeedRoutesOptions } from './seed.js';
4
4
  export type { SeedRoutesOptions };
5
5
  export { resolveSeedRefreshWindowMs, SEED_REFRESH_WINDOW_MS } from './seed.js';
6
6
  export interface PageRow {
@@ -34,6 +34,9 @@ export interface PageEntry {
34
34
  export interface PageData extends PageEntry {
35
35
  markdown: string;
36
36
  }
37
+ export type PageMetadata = Pick<PageEntry, 'route' | 'title' | 'description' | 'isError' | 'locale'>;
38
+ /** Read only admitted page metadata. Each query fits D1's 100 parameter limit. */
39
+ export declare function queryPageMetadata(event: AiReadyDatabaseEvent | undefined, routes: readonly string[]): Promise<PageMetadata[]>;
37
40
  export interface SearchResult {
38
41
  route: string;
39
42
  title: string;
@@ -142,11 +145,7 @@ export declare function getPageHash(event: AiReadyDatabaseEvent | undefined, rou
142
145
  /**
143
146
  * Seed routes from sitemap (insert with indexed=0 if not exists)
144
147
  */
145
- export declare function seedRoutes(event: AiReadyDatabaseEvent | undefined, routes: Array<string | {
146
- route: string;
147
- locale?: string;
148
- url?: string;
149
- }>, options?: SeedRoutesOptions): Promise<number>;
148
+ export declare function seedRoutes(event: AiReadyDatabaseEvent | undefined, routes: SeedRoute[], options?: SeedRoutesOptions): Promise<number>;
150
149
  /** Remove a page after its route returns HTTP 410. */
151
150
  export declare function prunePage(event: AiReadyDatabaseEvent | undefined, route: string): Promise<void>;
152
151
  /**
@@ -3,7 +3,7 @@ import { useEvent, useRuntimeConfig } from "#nuxtseo/nitro";
3
3
  import { logger } from "../logger.js";
4
4
  import { checkAndHandleStale } from "../utils/checkStale.js";
5
5
  import { parseSitemapCrawlState, serializeSitemapCrawlState } from "../utils/sitemap-crawl-state.js";
6
- import { deriveLocale, SEED_REFRESH_WINDOW_MS } from "./seed.js";
6
+ import { deriveLocale, parseSourceLastmod, SEED_REFRESH_WINDOW_MS } from "./seed.js";
7
7
  import { LIKE_ESCAPE, likeSubstring, maxRowsPerInsert, normalizeRoute, normalizeRouteKey, toFtsMatchQuery } from "./shared.js";
8
8
  export { resolveSeedRefreshWindowMs, SEED_REFRESH_WINDOW_MS } from "./seed.js";
9
9
  function getEventFromContext(providedEvent) {
@@ -80,6 +80,7 @@ async function getPrerenderDb() {
80
80
  const isErrorQuery = sql.includes("is_error = 1") || params.includes(1) && sql.includes("is_error");
81
81
  const excludeErrors = sql.includes("is_error = 0");
82
82
  const includeMarkdown = wantsMarkdown(sql);
83
+ const admittedRoutes = sql.includes("WHERE route IN (") ? new Set(params) : void 0;
83
84
  if (isErrorQuery) {
84
85
  return [...errorRoutes].map((route) => ({
85
86
  route,
@@ -94,8 +95,8 @@ async function getPrerenderDb() {
94
95
  locale: ""
95
96
  }));
96
97
  }
97
- const filtered = excludeErrors ? pages.filter((p) => !errorRoutes.has(p.route)) : pages;
98
- return filtered.map((p) => ({
98
+ const filtered = pages.filter((p) => (!excludeErrors || !errorRoutes.has(p.route)) && (!admittedRoutes || admittedRoutes.has(p.route)));
99
+ const rows = filtered.map((p) => ({
99
100
  route: p.route,
100
101
  title: p.title,
101
102
  description: p.description,
@@ -107,6 +108,13 @@ async function getPrerenderDb() {
107
108
  indexed: 1,
108
109
  locale: p.locale || ""
109
110
  }));
111
+ if (admittedRoutes && !excludeErrors) {
112
+ for (const route of errorRoutes) {
113
+ if (admittedRoutes.has(route))
114
+ rows.push({ route, title: "", description: "", headings: [], keywords: "[]", updated_at: "", is_error: 1, indexed: 0, locale: "" });
115
+ }
116
+ }
117
+ return rows;
110
118
  },
111
119
  first: async (sql, params = []) => {
112
120
  if (sql.includes("WHERE route = ?")) {
@@ -136,6 +144,24 @@ async function getPrerenderDb() {
136
144
  }
137
145
  };
138
146
  }
147
+ export async function queryPageMetadata(event, routes) {
148
+ if (!routes.length)
149
+ return [];
150
+ const db = await getDb(event);
151
+ if (!db)
152
+ return [];
153
+ const unique = [...new Set(routes)];
154
+ const pages = [];
155
+ for (let i = 0; i < unique.length; i += 100) {
156
+ const batch = unique.slice(i, i + 100);
157
+ const rows = await db.all(
158
+ `SELECT route, title, description, is_error, locale FROM ai_ready_pages WHERE route IN (${batch.map(() => "?").join(", ")})`,
159
+ batch
160
+ );
161
+ pages.push(...rows.map((row) => ({ route: row.route, title: row.title, description: row.description, isError: row.is_error === 1, locale: row.locale || "" })));
162
+ }
163
+ return pages;
164
+ }
139
165
  function buildWhereClause(where) {
140
166
  const conditions = [];
141
167
  const params = [];
@@ -383,26 +409,31 @@ export async function seedRoutes(event, routes, options = {}) {
383
409
  byRoute.set(route, {
384
410
  route,
385
411
  routeKey: normalizeRouteKey(route),
386
- locale: deriveLocale(event, route, explicitLocale, entryUrl)
412
+ locale: deriveLocale(event, route, explicitLocale, entryUrl),
413
+ sourceLastmod: parseSourceLastmod(typeof entry === "string" ? void 0 : entry.lastmod)
387
414
  });
388
415
  }
389
- const rowsPerInsert = maxRowsPerInsert(4);
416
+ const rowsPerInsert = maxRowsPerInsert(5);
390
417
  const stmts = [];
391
418
  for (const batch of chunk([...byRoute.values()], rowsPerInsert)) {
392
- const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', 0, 0, 0, 'runtime', ?, ?)`).join(", ");
393
- const params = batch.flatMap((r) => [r.route, r.routeKey, nowMs, r.locale]);
419
+ const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', ?, 0, 0, 0, 'runtime', ?, ?)`).join(", ");
420
+ const params = batch.flatMap((r) => [r.route, r.routeKey, r.sourceLastmod, nowMs, r.locale]);
394
421
  stmts.push({
395
422
  sql: `
396
- INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at, locale)
423
+ INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, source_lastmod, indexed_at, is_error, indexed, source, last_seen_at, locale)
397
424
  VALUES ${valuesSql}
398
425
  ON CONFLICT(route) DO UPDATE SET
399
426
  last_seen_at = excluded.last_seen_at,
400
427
  locale = excluded.locale,
428
+ source_lastmod = COALESCE(excluded.source_lastmod, ai_ready_pages.source_lastmod),
401
429
  is_error = 0,
402
- indexed = CASE WHEN ai_ready_pages.is_error = 1 THEN 0 ELSE ai_ready_pages.indexed END
430
+ indexed = CASE WHEN ai_ready_pages.is_error = 1
431
+ OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
432
+ THEN 0 ELSE ai_ready_pages.indexed END
403
433
  WHERE ai_ready_pages.last_seen_at IS NULL
404
434
  OR ai_ready_pages.last_seen_at < excluded.last_seen_at - ${refreshWindowMs}
405
435
  OR ai_ready_pages.locale IS DISTINCT FROM excluded.locale
436
+ OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
406
437
  `,
407
438
  params
408
439
  });
@@ -152,6 +152,21 @@ export declare const pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
152
152
  identity: undefined;
153
153
  generated: undefined;
154
154
  }>;
155
+ sourceLastmod: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").PgTextBuilder<[string, ...string[]]>, {
156
+ name: string;
157
+ tableName: "ai_ready_pages";
158
+ dataType: "string";
159
+ data: string;
160
+ driverParam: string;
161
+ notNull: false;
162
+ hasDefault: false;
163
+ isPrimaryKey: false;
164
+ isAutoincrement: false;
165
+ hasRuntimeDefault: false;
166
+ enumValues: undefined;
167
+ identity: undefined;
168
+ generated: undefined;
169
+ }>;
155
170
  indexedAt: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgBigInt53Builder>, {
156
171
  name: string;
157
172
  tableName: "ai_ready_pages";
@@ -721,6 +736,21 @@ export declare const schema: {
721
736
  identity: undefined;
722
737
  generated: undefined;
723
738
  }>;
739
+ sourceLastmod: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").PgTextBuilder<[string, ...string[]]>, {
740
+ name: string;
741
+ tableName: "ai_ready_pages";
742
+ dataType: "string";
743
+ data: string;
744
+ driverParam: string;
745
+ notNull: false;
746
+ hasDefault: false;
747
+ isPrimaryKey: false;
748
+ isAutoincrement: false;
749
+ hasRuntimeDefault: false;
750
+ enumValues: undefined;
751
+ identity: undefined;
752
+ generated: undefined;
753
+ }>;
724
754
  indexedAt: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgBigInt53Builder>, {
725
755
  name: string;
726
756
  tableName: "ai_ready_pages";
@@ -10,6 +10,7 @@ export const pages = pgTable("ai_ready_pages", {
10
10
  keywords: text("keywords").notNull().default("[]"),
11
11
  contentHash: text("content_hash"),
12
12
  updatedAt: text("updated_at").notNull(),
13
+ sourceLastmod: text("source_lastmod"),
13
14
  indexedAt: bigint("indexed_at", { mode: "number" }).notNull(),
14
15
  isError: integer("is_error").notNull().default(0),
15
16
  indexed: integer("indexed").notNull().default(0),
@@ -162,6 +162,22 @@ export declare const pages: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
162
162
  identity: undefined;
163
163
  generated: undefined;
164
164
  }, {}>;
165
+ sourceLastmod: import("drizzle-orm/sqlite-core").SQLiteColumn<{
166
+ name: string;
167
+ tableName: "ai_ready_pages";
168
+ dataType: "string";
169
+ data: string;
170
+ driverParam: string;
171
+ notNull: false;
172
+ hasDefault: false;
173
+ isPrimaryKey: false;
174
+ isAutoincrement: false;
175
+ hasRuntimeDefault: false;
176
+ enumValues: [string, ...string[]];
177
+ baseColumn: never;
178
+ identity: undefined;
179
+ generated: undefined;
180
+ }, {}>;
165
181
  indexedAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
166
182
  name: string;
167
183
  tableName: "ai_ready_pages";
@@ -767,6 +783,22 @@ export declare const schema: {
767
783
  identity: undefined;
768
784
  generated: undefined;
769
785
  }, {}>;
786
+ sourceLastmod: import("drizzle-orm/sqlite-core").SQLiteColumn<{
787
+ name: string;
788
+ tableName: "ai_ready_pages";
789
+ dataType: "string";
790
+ data: string;
791
+ driverParam: string;
792
+ notNull: false;
793
+ hasDefault: false;
794
+ isPrimaryKey: false;
795
+ isAutoincrement: false;
796
+ hasRuntimeDefault: false;
797
+ enumValues: [string, ...string[]];
798
+ baseColumn: never;
799
+ identity: undefined;
800
+ generated: undefined;
801
+ }, {}>;
770
802
  indexedAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
771
803
  name: string;
772
804
  tableName: "ai_ready_pages";
@@ -10,6 +10,7 @@ export const pages = sqliteTable("ai_ready_pages", {
10
10
  keywords: text("keywords").notNull().default("[]"),
11
11
  contentHash: text("content_hash"),
12
12
  updatedAt: text("updated_at").notNull(),
13
+ sourceLastmod: text("source_lastmod"),
13
14
  indexedAt: integer("indexed_at").notNull(),
14
15
  isError: integer("is_error").notNull().default(0),
15
16
  indexed: integer("indexed").notNull().default(0),
@@ -2,7 +2,7 @@
2
2
  * SQLite schema definitions for build-time prerendering
3
3
  * Used by shared.ts during nuxi generate/build
4
4
  */
5
- export declare const SCHEMA_VERSION = "v2.3.0";
5
+ export declare const SCHEMA_VERSION = "v2.4.0";
6
6
  /**
7
7
  * Allowed FTS5 tokenizer values. The configured value gets interpolated into the
8
8
  * `CREATE VIRTUAL TABLE … tokenize='…'` DDL, so we restrict it to a known-safe
@@ -1,4 +1,4 @@
1
- export const SCHEMA_VERSION = "v2.3.0";
1
+ export const SCHEMA_VERSION = "v2.4.0";
2
2
  export const SUPPORTED_FTS_TOKENIZERS = [
3
3
  "unicode61 remove_diacritics 2",
4
4
  "trigram"
@@ -24,6 +24,7 @@ export function buildSchemaSql(opts = {}) {
24
24
  keywords TEXT NOT NULL DEFAULT '[]',
25
25
  content_hash TEXT,
26
26
  updated_at TEXT NOT NULL,
27
+ source_lastmod TEXT,
27
28
  indexed_at INTEGER NOT NULL,
28
29
  is_error INTEGER NOT NULL DEFAULT 0,
29
30
  indexed INTEGER NOT NULL DEFAULT 0,
@@ -30,3 +30,11 @@ export interface SeedRoutesOptions {
30
30
  */
31
31
  refreshWindowMs?: number;
32
32
  }
33
+ export type SeedRoute = string | {
34
+ route: string;
35
+ locale?: string;
36
+ url?: string;
37
+ lastmod?: string | Date;
38
+ };
39
+ /** Parse source dates once, so equivalent W3C date representations compare equal. */
40
+ export declare function parseSourceLastmod(input: string | Date | undefined): string | null;
@@ -39,3 +39,19 @@ export function resolveSeedRefreshWindowMs(pruneTtlSeconds) {
39
39
  return SEED_REFRESH_WINDOW_MS;
40
40
  return Math.min(SEED_REFRESH_WINDOW_MS, Math.floor(pruneTtlSeconds * 1e3 / 2));
41
41
  }
42
+ export function parseSourceLastmod(input) {
43
+ if (input === void 0)
44
+ return null;
45
+ if (typeof input === "string" && (input.length > 64 || !/^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2}))?$/.test(input)))
46
+ return null;
47
+ const date = new Date(input);
48
+ if (!Number.isFinite(date.getTime()))
49
+ return null;
50
+ if (typeof input === "string") {
51
+ const calendarDate = input.slice(0, 10);
52
+ const parsedCalendarDate = new Date(calendarDate);
53
+ if (!Number.isFinite(parsedCalendarDate.getTime()) || parsedCalendarDate.toISOString().slice(0, 10) !== calendarDate)
54
+ return null;
55
+ }
56
+ return date.toISOString();
57
+ }
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Compute content hash for change detection (first 16 chars of SHA-256).
3
- * Hash the page body only. Frontmatter holds per-conversion fields
4
- * (`canonical_url`, `last_updated`) that differ between the build and runtime
5
- * paths, so hashing it marks unchanged pages as changed.
3
+ * Hash the page body and image metadata. Other frontmatter holds fields
4
+ * (`canonical_url`, `last_updated`) that can differ between conversion paths.
5
+ * Those fields stay excluded so unchanged pages keep the same hash.
6
6
  */
7
7
  export declare function computeContentHash(markdown: string): Promise<string>;
8
8
  export interface DatabaseAdapter {
@@ -10,7 +10,25 @@ function stripPageFrontmatter(markdown) {
10
10
  if (document.errors.length || !isMap(document.contents))
11
11
  return markdown;
12
12
  const isPageFrontmatter = document.contents.items.some((item) => isScalar(item.key) && PAGE_FRONTMATTER_KEYS.has(String(item.key.value)));
13
- return isPageFrontmatter ? markdown.slice(match[0].length) : markdown;
13
+ if (!isPageFrontmatter)
14
+ return markdown;
15
+ const body = markdown.slice(match[0].length);
16
+ const meta = document.get("meta", true);
17
+ const images = [];
18
+ if (isMap(meta)) {
19
+ for (const { key, value } of meta.items) {
20
+ if (!isScalar(key) || typeof key.value !== "string" || !isScalar(value))
21
+ continue;
22
+ if (key.value !== "og:image" && !key.value.startsWith("og:image:") && key.value !== "twitter:image" && !key.value.startsWith("twitter:image:"))
23
+ continue;
24
+ if (typeof value.value === "string" || typeof value.value === "number" && Number.isFinite(value.value))
25
+ images.push([key.value, value.value]);
26
+ }
27
+ }
28
+ if (!images.length)
29
+ return body;
30
+ images.sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
31
+ return JSON.stringify([images, body.trim()]);
14
32
  }
15
33
  export async function computeContentHash(markdown) {
16
34
  const encoder = new TextEncoder();
@@ -20,7 +20,7 @@ function notFoundMarkdown(ctx, canonicalUrl, build) {
20
20
  ``,
21
21
  `- [Sitemap](${resolveUrl("/sitemap.xml")})`,
22
22
  `- [llms.txt](${resolveUrl("/llms.txt")})`,
23
- `- [llms-full.txt](${resolveUrl("/llms-full.txt")})`,
23
+ ...config.llmsFullTxt === false ? [] : [`- [llms-full.txt](${resolveUrl("/llms-full.txt")})`],
24
24
  ``
25
25
  ].join("\n");
26
26
  const i18n = config.i18n;
@@ -2,8 +2,10 @@ import { useRuntimeConfig } from "#nuxtseo/nitro";
2
2
  import { trackDrizzleWork } from "../db/drizzle/client.js";
3
3
  import { getPageLastmods, getSitemapLastCrawledAt, markSitemapSeeded, resolveSeedRefreshWindowMs, seedRoutes } from "../db/queries.js";
4
4
  import { logger } from "../logger.js";
5
+ import { markIndexedSitemapLastmods } from "../utils/sitemap-lastmod.js";
5
6
  import { mapSitemapRoutes } from "../utils/sitemap-routes.js";
6
7
  const DIAGNOSTICS_KEY = "_aiReadySitemapDiagnostics";
8
+ const INDEXED_LASTMODS_KEY = "_aiReadyIndexedSitemapLastmods";
7
9
  function isDebug(_event) {
8
10
  return !!useRuntimeConfig()["nuxt-ai-ready"]?.debug;
9
11
  }
@@ -66,6 +68,7 @@ export default function sitemapSeederPlugin(nitroApp) {
66
68
  const routeToUrl = mapSitemapRoutes(urls);
67
69
  if (routeToUrl.size === 0)
68
70
  return;
71
+ const routes = [...routeToUrl.entries()].map(([route, url]) => ({ route, url: url.loc, lastmod: url.lastmod }));
69
72
  logger.debug(`[sitemap-seeder] Processing ${routeToUrl.size} routes from ${sitemapName}`);
70
73
  const record = isDebug(event) ? (m) => recordDiagnostic(event, m) : void 0;
71
74
  const readStart = Date.now();
@@ -75,6 +78,9 @@ export default function sitemapSeederPlugin(nitroApp) {
75
78
  const lastmod = lastmods.get(route);
76
79
  if (lastmod && !url.lastmod) {
77
80
  url.lastmod = lastmod;
81
+ const context = event.context;
82
+ const enrichedUrls = context[INDEXED_LASTMODS_KEY] ??= /* @__PURE__ */ new Set();
83
+ enrichedUrls.add(url.loc);
78
84
  enriched++;
79
85
  }
80
86
  }
@@ -92,7 +98,6 @@ export default function sitemapSeederPlugin(nitroApp) {
92
98
  if (Date.now() - lastSeed < SEED_INTERVAL_MS)
93
99
  return;
94
100
  lastSeedAt.set(sitemapName, Date.now());
95
- const routes = [...routeToUrl.entries()].map(([route, url]) => ({ route, url: url.loc }));
96
101
  const urlCount = urls.length;
97
102
  const seed = async () => {
98
103
  const seedStart = Date.now();
@@ -120,6 +125,9 @@ export default function sitemapSeederPlugin(nitroApp) {
120
125
  }
121
126
  });
122
127
  nitroApp.hooks.hook("sitemap:output", (ctx) => {
128
+ const enrichedUrls = ctx.event.context[INDEXED_LASTMODS_KEY];
129
+ if (enrichedUrls?.size)
130
+ ctx.sitemap = markIndexedSitemapLastmods(ctx.sitemap, enrichedUrls);
123
131
  const diagnostics = ctx.event.context[DIAGNOSTICS_KEY];
124
132
  if (!diagnostics?.length)
125
133
  return;
@@ -1,4 +1,4 @@
1
- import { eventHandler, sendIterable, setHeader, setResponseHeader } from "#nuxtseo/h3";
1
+ import { assertMethod, eventHandler, sendIterable, setHeader, setResponseHeader } from "#nuxtseo/h3";
2
2
  import { useRuntimeConfig } from "#nuxtseo/nitro";
3
3
  import { getSiteConfig } from "#site-config/server/composables";
4
4
  import { withSiteUrl } from "#site-config/server/composables/utils";
@@ -7,6 +7,7 @@ import { toDeployedRoute } from "../../route-path.js";
7
7
  import { countPages, streamPages } from "../db/queries.js";
8
8
  import { buildLlmsFullTxtHeader, formatPageForLlmsFullTxt } from "../utils/llms-full.js";
9
9
  export default eventHandler(async (event) => {
10
+ assertMethod(event, "GET", true);
10
11
  if (import.meta.prerender) {
11
12
  setHeader(event, "Content-Type", "text/plain; charset=utf-8");
12
13
  return "# llms-full.txt\n\nThis file is generated during prerender.";
@@ -1,29 +1,39 @@
1
- import { eventHandler, setHeader } from "#nuxtseo/h3";
1
+ import { assertMethod, createError, eventHandler, getRequestURL, setHeader } from "#nuxtseo/h3";
2
2
  import { defineCachedFunction, useRuntimeConfig } from "#nuxtseo/nitro";
3
3
  import { publicCacheControl } from "../../cache-control.js";
4
- import { buildLlmsTxt } from "../../llms-txt-utils.js";
4
+ import { buildLlmsTxtFiles } from "../../llms-txt-utils.js";
5
+ import { toLogicalRoute } from "../../route-path.js";
5
6
  const cachedBuilders = /* @__PURE__ */ new Map();
6
7
  function getBuildLlmsTxtCached(maxAge) {
7
8
  let cached = cachedBuilders.get(maxAge);
8
9
  if (!cached) {
9
- cached = defineCachedFunction(buildLlmsTxt, {
10
- name: "llms-txt",
10
+ cached = defineCachedFunction(buildLlmsTxtFiles, {
11
+ name: "llms-txt-files",
11
12
  group: "ai-ready",
12
13
  maxAge,
13
- swr: true
14
+ swr: true,
15
+ // Share one discovery snapshot between root and child files, separated by origin.
16
+ getKey: (event) => getRequestURL(event, { xForwardedHost: true, xForwardedProto: true }).origin
14
17
  });
15
18
  cachedBuilders.set(maxAge, cached);
16
19
  }
17
20
  return cached;
18
21
  }
19
22
  export default eventHandler(async (event) => {
20
- const runtimeConfig = useRuntimeConfig(event)["nuxt-ai-ready"];
21
- const cacheSeconds = runtimeConfig.llmsTxtCacheSeconds;
22
- const cacheEnabled = !import.meta.dev && cacheSeconds > 0;
23
- const content = cacheEnabled ? await getBuildLlmsTxtCached(cacheSeconds)(event) : await buildLlmsTxt(event);
23
+ const runtimeConfig = useRuntimeConfig(event);
24
+ const path = toLogicalRoute(event.path.split("?")[0], runtimeConfig.app.baseURL);
25
+ if (path !== "/llms.txt" && !/^\/llms\/[^/]+\.txt$/i.test(path))
26
+ return;
27
+ assertMethod(event, "GET", true);
28
+ const config = runtimeConfig["nuxt-ai-ready"];
29
+ const cacheSeconds = config.llmsTxtCacheSeconds;
30
+ const cacheEnabled = !import.meta.dev && !import.meta.prerender && cacheSeconds > 0;
31
+ const files = cacheEnabled ? await getBuildLlmsTxtCached(cacheSeconds)(event) : await buildLlmsTxtFiles(event);
32
+ const content = files[path];
33
+ if (content === void 0)
34
+ throw createError({ statusCode: 404, statusMessage: "llms.txt file not found" });
24
35
  setHeader(event, "Content-Type", "text/plain; charset=utf-8");
25
- if (cacheEnabled) {
36
+ if (cacheEnabled)
26
37
  setHeader(event, "Cache-Control", publicCacheControl(cacheSeconds, 3600));
27
- }
28
38
  return content;
29
39
  });
@@ -1,4 +1,4 @@
1
- import { eventHandler, setHeader } from "#nuxtseo/h3";
1
+ import { assertMethod, eventHandler, setHeader } from "#nuxtseo/h3";
2
2
  import { defineCachedFunction, useRuntimeConfig } from "#nuxtseo/nitro";
3
3
  import { getSiteConfig } from "#site-config/server/composables";
4
4
  import { publicCacheControl } from "../../cache-control.js";
@@ -42,6 +42,7 @@ function getBuildSitemapMdCached(maxAge) {
42
42
  return cached;
43
43
  }
44
44
  export default eventHandler(async (event) => {
45
+ assertMethod(event, "GET", true);
45
46
  const fullRuntimeConfig = useRuntimeConfig(event);
46
47
  const runtimeConfig = fullRuntimeConfig["nuxt-ai-ready"];
47
48
  const cacheSeconds = runtimeConfig.llmsTxtCacheSeconds ?? 600;