nuxt-ai-ready 3.2.0 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunks/agent-skills.mjs +2 -1
- package/dist/chunks/prerender.mjs +197 -28
- package/dist/cli.mjs +259 -0
- package/dist/module.d.mts +1 -0
- package/dist/module.json +1 -1
- package/dist/module.mjs +2 -1
- package/dist/runtime/llms-txt-files.d.ts +30 -0
- package/dist/runtime/llms-txt-files.js +196 -0
- package/dist/runtime/llms-txt-format.js +5 -4
- package/dist/runtime/llms-txt-utils.d.ts +3 -2
- package/dist/runtime/llms-txt-utils.js +85 -224
- package/dist/runtime/markdown-text.d.ts +2 -0
- package/dist/runtime/markdown-text.js +3 -0
- package/dist/runtime/server/db/drizzle/queries.d.ts +2 -6
- package/dist/runtime/server/db/drizzle/queries.js +30 -10
- package/dist/runtime/server/db/queries.d.ts +5 -6
- package/dist/runtime/server/db/queries.js +40 -9
- package/dist/runtime/server/db/schema/postgres.d.ts +30 -0
- package/dist/runtime/server/db/schema/postgres.js +1 -0
- package/dist/runtime/server/db/schema/sqlite.d.ts +32 -0
- package/dist/runtime/server/db/schema/sqlite.js +1 -0
- package/dist/runtime/server/db/schema-sql.d.ts +1 -1
- package/dist/runtime/server/db/schema-sql.js +2 -1
- package/dist/runtime/server/db/seed.d.ts +8 -0
- package/dist/runtime/server/db/seed.js +16 -0
- package/dist/runtime/server/db/shared.d.ts +3 -3
- package/dist/runtime/server/db/shared.js +19 -1
- package/dist/runtime/server/middleware/markdown.js +1 -1
- package/dist/runtime/server/plugins/sitemap-seeder.js +9 -1
- package/dist/runtime/server/routes/llms-full.txt.get.js +2 -1
- package/dist/runtime/server/routes/llms.txt.get.js +21 -11
- package/dist/runtime/server/routes/sitemap.md.get.js +2 -1
- package/dist/runtime/server/utils/llms-full.js +31 -5
- package/dist/runtime/server/utils/markdown-request.js +2 -5
- package/dist/runtime/server/utils/negotiation-response.js +3 -0
- package/dist/runtime/server/utils/runCron.js +1 -1
- package/dist/runtime/server/utils/sitemap-lastmod.d.ts +4 -0
- package/dist/runtime/server/utils/sitemap-lastmod.js +19 -0
- package/dist/runtime/server/utils/sitemap-md.js +3 -2
- package/dist/runtime/server/utils/sitemap.d.ts +1 -0
- package/dist/runtime/server/utils/sitemap.js +26 -12
- package/dist/runtime/server/utils.js +50 -2
- package/dist/runtime/types.d.ts +13 -1
- package/dist/shared/{nuxt-ai-ready.DH9mSayy.mjs → nuxt-ai-ready.CKXM1jnE.mjs} +40 -12
- package/package.json +3 -3
- package/skills/nuxt-ai-ready/SKILL.md +34 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { SitemapCrawlState } from '../../utils/sitemap-crawl-state.js';
|
|
2
2
|
import type { AiReadyDatabaseEvent } from '../context.js';
|
|
3
|
-
import type { SeedRoutesOptions } from '../seed.js';
|
|
3
|
+
import type { SeedRoute, SeedRoutesOptions } from '../seed.js';
|
|
4
4
|
export interface PageInput {
|
|
5
5
|
route: string;
|
|
6
6
|
title: string;
|
|
@@ -182,9 +182,5 @@ export declare function resetSitemapErrors(event: AiReadyDatabaseEvent | undefin
|
|
|
182
182
|
* guard each run bumped `last_seen_at` on every row, which cost millions of
|
|
183
183
|
* D1 row writes a month. Keep the two layers in sync.
|
|
184
184
|
*/
|
|
185
|
-
export declare function seedRoutes(event: AiReadyDatabaseEvent | undefined, routes:
|
|
186
|
-
route: string;
|
|
187
|
-
locale?: string;
|
|
188
|
-
url?: string;
|
|
189
|
-
}>, options?: SeedRoutesOptions): Promise<number>;
|
|
185
|
+
export declare function seedRoutes(event: AiReadyDatabaseEvent | undefined, routes: SeedRoute[], options?: SeedRoutesOptions): Promise<number>;
|
|
190
186
|
export {};
|
|
@@ -3,7 +3,7 @@ import { cronRuns, info, pages, sitemaps } from "#ai-ready-virtual/db-schema.mjs
|
|
|
3
3
|
import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
4
4
|
import { parseSitemapCrawlState, serializeSitemapCrawlState } from "../../utils/sitemap-crawl-state.js";
|
|
5
5
|
import { resolveFtsTokenizer as validateFtsTokenizer } from "../schema-sql.js";
|
|
6
|
-
import { deriveLocale, SEED_REFRESH_WINDOW_MS } from "../seed.js";
|
|
6
|
+
import { deriveLocale, parseSourceLastmod, SEED_REFRESH_WINDOW_MS } from "../seed.js";
|
|
7
7
|
import { LIKE_ESCAPE, likeSubstring, maxRowsPerInsert, toFtsMatchQuery } from "../shared.js";
|
|
8
8
|
import { useDrizzle } from "./client.js";
|
|
9
9
|
import { getRawExecutor, useRawDb } from "./raw.js";
|
|
@@ -234,9 +234,11 @@ export async function deleteInfoValue(event, key) {
|
|
|
234
234
|
const client = await useDrizzle(event);
|
|
235
235
|
await client.db.delete(info).where(eq(info.id, key));
|
|
236
236
|
}
|
|
237
|
-
const
|
|
237
|
+
const LEGACY_SQLITE_SCHEMA_VERSION = "v2.3.0-drizzle";
|
|
238
|
+
const SQLITE_SCHEMA_VERSION = "v2.4.0-drizzle-source-lastmod";
|
|
238
239
|
const LEGACY_POSTGRES_INTEGER_SCHEMA_VERSION = "v2.3.0-drizzle";
|
|
239
|
-
const
|
|
240
|
+
const LEGACY_POSTGRES_SCHEMA_VERSION = "v2.3.0-drizzle-postgres-bigint";
|
|
241
|
+
const POSTGRES_SCHEMA_VERSION = "v2.4.0-drizzle-postgres-source-lastmod";
|
|
240
242
|
export async function initSchema(event) {
|
|
241
243
|
const client = await useDrizzle(event);
|
|
242
244
|
const tokenizer = resolveFtsTokenizer(event);
|
|
@@ -249,6 +251,11 @@ export async function initSchema(event) {
|
|
|
249
251
|
return;
|
|
250
252
|
if (client.dialect === "postgres" && currentVersion === LEGACY_POSTGRES_INTEGER_SCHEMA_VERSION) {
|
|
251
253
|
await migratePostgresIntegerTimestamps(client);
|
|
254
|
+
await migrateSourceLastmod(client, schemaVersion);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
if (tokenizerMatches && (client.dialect === "postgres" ? currentVersion === LEGACY_POSTGRES_SCHEMA_VERSION : currentVersion === LEGACY_SQLITE_SCHEMA_VERSION)) {
|
|
258
|
+
await migrateSourceLastmod(client, schemaVersion);
|
|
252
259
|
return;
|
|
253
260
|
}
|
|
254
261
|
if (currentVersion) {
|
|
@@ -341,10 +348,16 @@ async function migratePostgresIntegerTimestamps(client) {
|
|
|
341
348
|
{ sql: "ALTER TABLE ai_ready_sitemaps ALTER COLUMN last_crawled_at TYPE BIGINT USING last_crawled_at::BIGINT" },
|
|
342
349
|
{
|
|
343
350
|
sql: "UPDATE _ai_ready_info SET version = ? WHERE id = 'schema'",
|
|
344
|
-
params: [
|
|
351
|
+
params: [LEGACY_POSTGRES_SCHEMA_VERSION]
|
|
345
352
|
}
|
|
346
353
|
]);
|
|
347
354
|
}
|
|
355
|
+
async function migrateSourceLastmod(client, schemaVersion) {
|
|
356
|
+
await getRawExecutor(client).batch([
|
|
357
|
+
{ sql: "ALTER TABLE ai_ready_pages ADD COLUMN source_lastmod TEXT" },
|
|
358
|
+
{ sql: "UPDATE _ai_ready_info SET version = ? WHERE id = 'schema'", params: [schemaVersion] }
|
|
359
|
+
]);
|
|
360
|
+
}
|
|
348
361
|
async function createSQLiteTables(client, ftsTokenizer) {
|
|
349
362
|
const statements = [
|
|
350
363
|
sql`CREATE TABLE IF NOT EXISTS ai_ready_pages (
|
|
@@ -358,6 +371,7 @@ async function createSQLiteTables(client, ftsTokenizer) {
|
|
|
358
371
|
keywords TEXT NOT NULL DEFAULT '[]',
|
|
359
372
|
content_hash TEXT,
|
|
360
373
|
updated_at TEXT NOT NULL,
|
|
374
|
+
source_lastmod TEXT,
|
|
361
375
|
indexed_at INTEGER NOT NULL,
|
|
362
376
|
is_error INTEGER NOT NULL DEFAULT 0,
|
|
363
377
|
indexed INTEGER NOT NULL DEFAULT 0,
|
|
@@ -445,6 +459,7 @@ async function createPostgresTables(client) {
|
|
|
445
459
|
content_hash TEXT,
|
|
446
460
|
updated_at TEXT NOT NULL,
|
|
447
461
|
indexed_at BIGINT NOT NULL,
|
|
462
|
+
source_lastmod TEXT,
|
|
448
463
|
is_error INTEGER NOT NULL DEFAULT 0,
|
|
449
464
|
indexed INTEGER NOT NULL DEFAULT 0,
|
|
450
465
|
source TEXT NOT NULL DEFAULT 'prerender',
|
|
@@ -685,29 +700,34 @@ export async function seedRoutes(event, routes, options = {}) {
|
|
|
685
700
|
byRoute.set(route, {
|
|
686
701
|
route,
|
|
687
702
|
routeKey: normalizeRouteKey(route),
|
|
688
|
-
locale: deriveLocale(event, route, explicitLocale, entryUrl)
|
|
703
|
+
locale: deriveLocale(event, route, explicitLocale, entryUrl),
|
|
704
|
+
sourceLastmod: parseSourceLastmod(typeof entry === "string" ? void 0 : entry.lastmod)
|
|
689
705
|
});
|
|
690
706
|
}
|
|
691
|
-
const rowsPerInsert = maxRowsPerInsert(
|
|
707
|
+
const rowsPerInsert = maxRowsPerInsert(5);
|
|
692
708
|
const db = await useRawDb(event);
|
|
693
709
|
const entries = [...byRoute.values()];
|
|
694
710
|
const stmts = [];
|
|
695
711
|
for (let i = 0; i < entries.length; i += rowsPerInsert) {
|
|
696
712
|
const batch = entries.slice(i, i + rowsPerInsert);
|
|
697
|
-
const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', 0, 0, 0, 'runtime', ?, ?)`).join(", ");
|
|
698
|
-
const params = batch.flatMap((r) => [r.route, r.routeKey, nowMs, r.locale]);
|
|
713
|
+
const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', ?, 0, 0, 0, 'runtime', ?, ?)`).join(", ");
|
|
714
|
+
const params = batch.flatMap((r) => [r.route, r.routeKey, r.sourceLastmod, nowMs, r.locale]);
|
|
699
715
|
stmts.push({
|
|
700
716
|
sql: `
|
|
701
|
-
INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at, locale)
|
|
717
|
+
INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, source_lastmod, indexed_at, is_error, indexed, source, last_seen_at, locale)
|
|
702
718
|
VALUES ${valuesSql}
|
|
703
719
|
ON CONFLICT(route) DO UPDATE SET
|
|
704
720
|
last_seen_at = excluded.last_seen_at,
|
|
705
721
|
locale = excluded.locale,
|
|
722
|
+
source_lastmod = COALESCE(excluded.source_lastmod, ai_ready_pages.source_lastmod),
|
|
706
723
|
is_error = 0,
|
|
707
|
-
indexed = CASE WHEN ai_ready_pages.is_error = 1
|
|
724
|
+
indexed = CASE WHEN ai_ready_pages.is_error = 1
|
|
725
|
+
OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
|
|
726
|
+
THEN 0 ELSE ai_ready_pages.indexed END
|
|
708
727
|
WHERE ai_ready_pages.last_seen_at IS NULL
|
|
709
728
|
OR ai_ready_pages.last_seen_at < excluded.last_seen_at - ${refreshWindowMs}
|
|
710
729
|
OR ai_ready_pages.locale IS DISTINCT FROM excluded.locale
|
|
730
|
+
OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
|
|
711
731
|
`,
|
|
712
732
|
params
|
|
713
733
|
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { SitemapCrawlState } from '../utils/sitemap-crawl-state.js';
|
|
2
2
|
import type { AiReadyDatabaseEvent } from './context.js';
|
|
3
|
-
import type { SeedRoutesOptions } from './seed.js';
|
|
3
|
+
import type { SeedRoute, SeedRoutesOptions } from './seed.js';
|
|
4
4
|
export type { SeedRoutesOptions };
|
|
5
5
|
export { resolveSeedRefreshWindowMs, SEED_REFRESH_WINDOW_MS } from './seed.js';
|
|
6
6
|
export interface PageRow {
|
|
@@ -34,6 +34,9 @@ export interface PageEntry {
|
|
|
34
34
|
export interface PageData extends PageEntry {
|
|
35
35
|
markdown: string;
|
|
36
36
|
}
|
|
37
|
+
export type PageMetadata = Pick<PageEntry, 'route' | 'title' | 'description' | 'isError' | 'locale'>;
|
|
38
|
+
/** Read only admitted page metadata. Each query fits D1's 100 parameter limit. */
|
|
39
|
+
export declare function queryPageMetadata(event: AiReadyDatabaseEvent | undefined, routes: readonly string[]): Promise<PageMetadata[]>;
|
|
37
40
|
export interface SearchResult {
|
|
38
41
|
route: string;
|
|
39
42
|
title: string;
|
|
@@ -142,11 +145,7 @@ export declare function getPageHash(event: AiReadyDatabaseEvent | undefined, rou
|
|
|
142
145
|
/**
|
|
143
146
|
* Seed routes from sitemap (insert with indexed=0 if not exists)
|
|
144
147
|
*/
|
|
145
|
-
export declare function seedRoutes(event: AiReadyDatabaseEvent | undefined, routes:
|
|
146
|
-
route: string;
|
|
147
|
-
locale?: string;
|
|
148
|
-
url?: string;
|
|
149
|
-
}>, options?: SeedRoutesOptions): Promise<number>;
|
|
148
|
+
export declare function seedRoutes(event: AiReadyDatabaseEvent | undefined, routes: SeedRoute[], options?: SeedRoutesOptions): Promise<number>;
|
|
150
149
|
/** Remove a page after its route returns HTTP 410. */
|
|
151
150
|
export declare function prunePage(event: AiReadyDatabaseEvent | undefined, route: string): Promise<void>;
|
|
152
151
|
/**
|
|
@@ -3,7 +3,7 @@ import { useEvent, useRuntimeConfig } from "#nuxtseo/nitro";
|
|
|
3
3
|
import { logger } from "../logger.js";
|
|
4
4
|
import { checkAndHandleStale } from "../utils/checkStale.js";
|
|
5
5
|
import { parseSitemapCrawlState, serializeSitemapCrawlState } from "../utils/sitemap-crawl-state.js";
|
|
6
|
-
import { deriveLocale, SEED_REFRESH_WINDOW_MS } from "./seed.js";
|
|
6
|
+
import { deriveLocale, parseSourceLastmod, SEED_REFRESH_WINDOW_MS } from "./seed.js";
|
|
7
7
|
import { LIKE_ESCAPE, likeSubstring, maxRowsPerInsert, normalizeRoute, normalizeRouteKey, toFtsMatchQuery } from "./shared.js";
|
|
8
8
|
export { resolveSeedRefreshWindowMs, SEED_REFRESH_WINDOW_MS } from "./seed.js";
|
|
9
9
|
function getEventFromContext(providedEvent) {
|
|
@@ -80,6 +80,7 @@ async function getPrerenderDb() {
|
|
|
80
80
|
const isErrorQuery = sql.includes("is_error = 1") || params.includes(1) && sql.includes("is_error");
|
|
81
81
|
const excludeErrors = sql.includes("is_error = 0");
|
|
82
82
|
const includeMarkdown = wantsMarkdown(sql);
|
|
83
|
+
const admittedRoutes = sql.includes("WHERE route IN (") ? new Set(params) : void 0;
|
|
83
84
|
if (isErrorQuery) {
|
|
84
85
|
return [...errorRoutes].map((route) => ({
|
|
85
86
|
route,
|
|
@@ -94,8 +95,8 @@ async function getPrerenderDb() {
|
|
|
94
95
|
locale: ""
|
|
95
96
|
}));
|
|
96
97
|
}
|
|
97
|
-
const filtered =
|
|
98
|
-
|
|
98
|
+
const filtered = pages.filter((p) => (!excludeErrors || !errorRoutes.has(p.route)) && (!admittedRoutes || admittedRoutes.has(p.route)));
|
|
99
|
+
const rows = filtered.map((p) => ({
|
|
99
100
|
route: p.route,
|
|
100
101
|
title: p.title,
|
|
101
102
|
description: p.description,
|
|
@@ -107,6 +108,13 @@ async function getPrerenderDb() {
|
|
|
107
108
|
indexed: 1,
|
|
108
109
|
locale: p.locale || ""
|
|
109
110
|
}));
|
|
111
|
+
if (admittedRoutes && !excludeErrors) {
|
|
112
|
+
for (const route of errorRoutes) {
|
|
113
|
+
if (admittedRoutes.has(route))
|
|
114
|
+
rows.push({ route, title: "", description: "", headings: [], keywords: "[]", updated_at: "", is_error: 1, indexed: 0, locale: "" });
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return rows;
|
|
110
118
|
},
|
|
111
119
|
first: async (sql, params = []) => {
|
|
112
120
|
if (sql.includes("WHERE route = ?")) {
|
|
@@ -136,6 +144,24 @@ async function getPrerenderDb() {
|
|
|
136
144
|
}
|
|
137
145
|
};
|
|
138
146
|
}
|
|
147
|
+
export async function queryPageMetadata(event, routes) {
|
|
148
|
+
if (!routes.length)
|
|
149
|
+
return [];
|
|
150
|
+
const db = await getDb(event);
|
|
151
|
+
if (!db)
|
|
152
|
+
return [];
|
|
153
|
+
const unique = [...new Set(routes)];
|
|
154
|
+
const pages = [];
|
|
155
|
+
for (let i = 0; i < unique.length; i += 100) {
|
|
156
|
+
const batch = unique.slice(i, i + 100);
|
|
157
|
+
const rows = await db.all(
|
|
158
|
+
`SELECT route, title, description, is_error, locale FROM ai_ready_pages WHERE route IN (${batch.map(() => "?").join(", ")})`,
|
|
159
|
+
batch
|
|
160
|
+
);
|
|
161
|
+
pages.push(...rows.map((row) => ({ route: row.route, title: row.title, description: row.description, isError: row.is_error === 1, locale: row.locale || "" })));
|
|
162
|
+
}
|
|
163
|
+
return pages;
|
|
164
|
+
}
|
|
139
165
|
function buildWhereClause(where) {
|
|
140
166
|
const conditions = [];
|
|
141
167
|
const params = [];
|
|
@@ -383,26 +409,31 @@ export async function seedRoutes(event, routes, options = {}) {
|
|
|
383
409
|
byRoute.set(route, {
|
|
384
410
|
route,
|
|
385
411
|
routeKey: normalizeRouteKey(route),
|
|
386
|
-
locale: deriveLocale(event, route, explicitLocale, entryUrl)
|
|
412
|
+
locale: deriveLocale(event, route, explicitLocale, entryUrl),
|
|
413
|
+
sourceLastmod: parseSourceLastmod(typeof entry === "string" ? void 0 : entry.lastmod)
|
|
387
414
|
});
|
|
388
415
|
}
|
|
389
|
-
const rowsPerInsert = maxRowsPerInsert(
|
|
416
|
+
const rowsPerInsert = maxRowsPerInsert(5);
|
|
390
417
|
const stmts = [];
|
|
391
418
|
for (const batch of chunk([...byRoute.values()], rowsPerInsert)) {
|
|
392
|
-
const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', 0, 0, 0, 'runtime', ?, ?)`).join(", ");
|
|
393
|
-
const params = batch.flatMap((r) => [r.route, r.routeKey, nowMs, r.locale]);
|
|
419
|
+
const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', '', ?, 0, 0, 0, 'runtime', ?, ?)`).join(", ");
|
|
420
|
+
const params = batch.flatMap((r) => [r.route, r.routeKey, r.sourceLastmod, nowMs, r.locale]);
|
|
394
421
|
stmts.push({
|
|
395
422
|
sql: `
|
|
396
|
-
INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at, locale)
|
|
423
|
+
INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, source_lastmod, indexed_at, is_error, indexed, source, last_seen_at, locale)
|
|
397
424
|
VALUES ${valuesSql}
|
|
398
425
|
ON CONFLICT(route) DO UPDATE SET
|
|
399
426
|
last_seen_at = excluded.last_seen_at,
|
|
400
427
|
locale = excluded.locale,
|
|
428
|
+
source_lastmod = COALESCE(excluded.source_lastmod, ai_ready_pages.source_lastmod),
|
|
401
429
|
is_error = 0,
|
|
402
|
-
indexed = CASE WHEN ai_ready_pages.is_error = 1
|
|
430
|
+
indexed = CASE WHEN ai_ready_pages.is_error = 1
|
|
431
|
+
OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
|
|
432
|
+
THEN 0 ELSE ai_ready_pages.indexed END
|
|
403
433
|
WHERE ai_ready_pages.last_seen_at IS NULL
|
|
404
434
|
OR ai_ready_pages.last_seen_at < excluded.last_seen_at - ${refreshWindowMs}
|
|
405
435
|
OR ai_ready_pages.locale IS DISTINCT FROM excluded.locale
|
|
436
|
+
OR (excluded.source_lastmod IS NOT NULL AND ai_ready_pages.source_lastmod IS DISTINCT FROM excluded.source_lastmod)
|
|
406
437
|
`,
|
|
407
438
|
params
|
|
408
439
|
});
|
|
@@ -152,6 +152,21 @@ export declare const pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
|
152
152
|
identity: undefined;
|
|
153
153
|
generated: undefined;
|
|
154
154
|
}>;
|
|
155
|
+
sourceLastmod: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").PgTextBuilder<[string, ...string[]]>, {
|
|
156
|
+
name: string;
|
|
157
|
+
tableName: "ai_ready_pages";
|
|
158
|
+
dataType: "string";
|
|
159
|
+
data: string;
|
|
160
|
+
driverParam: string;
|
|
161
|
+
notNull: false;
|
|
162
|
+
hasDefault: false;
|
|
163
|
+
isPrimaryKey: false;
|
|
164
|
+
isAutoincrement: false;
|
|
165
|
+
hasRuntimeDefault: false;
|
|
166
|
+
enumValues: undefined;
|
|
167
|
+
identity: undefined;
|
|
168
|
+
generated: undefined;
|
|
169
|
+
}>;
|
|
155
170
|
indexedAt: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgBigInt53Builder>, {
|
|
156
171
|
name: string;
|
|
157
172
|
tableName: "ai_ready_pages";
|
|
@@ -721,6 +736,21 @@ export declare const schema: {
|
|
|
721
736
|
identity: undefined;
|
|
722
737
|
generated: undefined;
|
|
723
738
|
}>;
|
|
739
|
+
sourceLastmod: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").PgTextBuilder<[string, ...string[]]>, {
|
|
740
|
+
name: string;
|
|
741
|
+
tableName: "ai_ready_pages";
|
|
742
|
+
dataType: "string";
|
|
743
|
+
data: string;
|
|
744
|
+
driverParam: string;
|
|
745
|
+
notNull: false;
|
|
746
|
+
hasDefault: false;
|
|
747
|
+
isPrimaryKey: false;
|
|
748
|
+
isAutoincrement: false;
|
|
749
|
+
hasRuntimeDefault: false;
|
|
750
|
+
enumValues: undefined;
|
|
751
|
+
identity: undefined;
|
|
752
|
+
generated: undefined;
|
|
753
|
+
}>;
|
|
724
754
|
indexedAt: import("drizzle-orm/pg-core").PgBuildColumn<"ai_ready_pages", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgBigInt53Builder>, {
|
|
725
755
|
name: string;
|
|
726
756
|
tableName: "ai_ready_pages";
|
|
@@ -10,6 +10,7 @@ export const pages = pgTable("ai_ready_pages", {
|
|
|
10
10
|
keywords: text("keywords").notNull().default("[]"),
|
|
11
11
|
contentHash: text("content_hash"),
|
|
12
12
|
updatedAt: text("updated_at").notNull(),
|
|
13
|
+
sourceLastmod: text("source_lastmod"),
|
|
13
14
|
indexedAt: bigint("indexed_at", { mode: "number" }).notNull(),
|
|
14
15
|
isError: integer("is_error").notNull().default(0),
|
|
15
16
|
indexed: integer("indexed").notNull().default(0),
|
|
@@ -162,6 +162,22 @@ export declare const pages: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
162
162
|
identity: undefined;
|
|
163
163
|
generated: undefined;
|
|
164
164
|
}, {}>;
|
|
165
|
+
sourceLastmod: import("drizzle-orm/sqlite-core").SQLiteColumn<{
|
|
166
|
+
name: string;
|
|
167
|
+
tableName: "ai_ready_pages";
|
|
168
|
+
dataType: "string";
|
|
169
|
+
data: string;
|
|
170
|
+
driverParam: string;
|
|
171
|
+
notNull: false;
|
|
172
|
+
hasDefault: false;
|
|
173
|
+
isPrimaryKey: false;
|
|
174
|
+
isAutoincrement: false;
|
|
175
|
+
hasRuntimeDefault: false;
|
|
176
|
+
enumValues: [string, ...string[]];
|
|
177
|
+
baseColumn: never;
|
|
178
|
+
identity: undefined;
|
|
179
|
+
generated: undefined;
|
|
180
|
+
}, {}>;
|
|
165
181
|
indexedAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
|
|
166
182
|
name: string;
|
|
167
183
|
tableName: "ai_ready_pages";
|
|
@@ -767,6 +783,22 @@ export declare const schema: {
|
|
|
767
783
|
identity: undefined;
|
|
768
784
|
generated: undefined;
|
|
769
785
|
}, {}>;
|
|
786
|
+
sourceLastmod: import("drizzle-orm/sqlite-core").SQLiteColumn<{
|
|
787
|
+
name: string;
|
|
788
|
+
tableName: "ai_ready_pages";
|
|
789
|
+
dataType: "string";
|
|
790
|
+
data: string;
|
|
791
|
+
driverParam: string;
|
|
792
|
+
notNull: false;
|
|
793
|
+
hasDefault: false;
|
|
794
|
+
isPrimaryKey: false;
|
|
795
|
+
isAutoincrement: false;
|
|
796
|
+
hasRuntimeDefault: false;
|
|
797
|
+
enumValues: [string, ...string[]];
|
|
798
|
+
baseColumn: never;
|
|
799
|
+
identity: undefined;
|
|
800
|
+
generated: undefined;
|
|
801
|
+
}, {}>;
|
|
770
802
|
indexedAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
|
|
771
803
|
name: string;
|
|
772
804
|
tableName: "ai_ready_pages";
|
|
@@ -10,6 +10,7 @@ export const pages = sqliteTable("ai_ready_pages", {
|
|
|
10
10
|
keywords: text("keywords").notNull().default("[]"),
|
|
11
11
|
contentHash: text("content_hash"),
|
|
12
12
|
updatedAt: text("updated_at").notNull(),
|
|
13
|
+
sourceLastmod: text("source_lastmod"),
|
|
13
14
|
indexedAt: integer("indexed_at").notNull(),
|
|
14
15
|
isError: integer("is_error").notNull().default(0),
|
|
15
16
|
indexed: integer("indexed").notNull().default(0),
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* SQLite schema definitions for build-time prerendering
|
|
3
3
|
* Used by shared.ts during nuxi generate/build
|
|
4
4
|
*/
|
|
5
|
-
export declare const SCHEMA_VERSION = "v2.
|
|
5
|
+
export declare const SCHEMA_VERSION = "v2.4.0";
|
|
6
6
|
/**
|
|
7
7
|
* Allowed FTS5 tokenizer values. The configured value gets interpolated into the
|
|
8
8
|
* `CREATE VIRTUAL TABLE … tokenize='…'` DDL, so we restrict it to a known-safe
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export const SCHEMA_VERSION = "v2.
|
|
1
|
+
export const SCHEMA_VERSION = "v2.4.0";
|
|
2
2
|
export const SUPPORTED_FTS_TOKENIZERS = [
|
|
3
3
|
"unicode61 remove_diacritics 2",
|
|
4
4
|
"trigram"
|
|
@@ -24,6 +24,7 @@ export function buildSchemaSql(opts = {}) {
|
|
|
24
24
|
keywords TEXT NOT NULL DEFAULT '[]',
|
|
25
25
|
content_hash TEXT,
|
|
26
26
|
updated_at TEXT NOT NULL,
|
|
27
|
+
source_lastmod TEXT,
|
|
27
28
|
indexed_at INTEGER NOT NULL,
|
|
28
29
|
is_error INTEGER NOT NULL DEFAULT 0,
|
|
29
30
|
indexed INTEGER NOT NULL DEFAULT 0,
|
|
@@ -30,3 +30,11 @@ export interface SeedRoutesOptions {
|
|
|
30
30
|
*/
|
|
31
31
|
refreshWindowMs?: number;
|
|
32
32
|
}
|
|
33
|
+
export type SeedRoute = string | {
|
|
34
|
+
route: string;
|
|
35
|
+
locale?: string;
|
|
36
|
+
url?: string;
|
|
37
|
+
lastmod?: string | Date;
|
|
38
|
+
};
|
|
39
|
+
/** Parse source dates once, so equivalent W3C date representations compare equal. */
|
|
40
|
+
export declare function parseSourceLastmod(input: string | Date | undefined): string | null;
|
|
@@ -39,3 +39,19 @@ export function resolveSeedRefreshWindowMs(pruneTtlSeconds) {
|
|
|
39
39
|
return SEED_REFRESH_WINDOW_MS;
|
|
40
40
|
return Math.min(SEED_REFRESH_WINDOW_MS, Math.floor(pruneTtlSeconds * 1e3 / 2));
|
|
41
41
|
}
|
|
42
|
+
export function parseSourceLastmod(input) {
|
|
43
|
+
if (input === void 0)
|
|
44
|
+
return null;
|
|
45
|
+
if (typeof input === "string" && (input.length > 64 || !/^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2}))?$/.test(input)))
|
|
46
|
+
return null;
|
|
47
|
+
const date = new Date(input);
|
|
48
|
+
if (!Number.isFinite(date.getTime()))
|
|
49
|
+
return null;
|
|
50
|
+
if (typeof input === "string") {
|
|
51
|
+
const calendarDate = input.slice(0, 10);
|
|
52
|
+
const parsedCalendarDate = new Date(calendarDate);
|
|
53
|
+
if (!Number.isFinite(parsedCalendarDate.getTime()) || parsedCalendarDate.toISOString().slice(0, 10) !== calendarDate)
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
return date.toISOString();
|
|
57
|
+
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Compute content hash for change detection (first 16 chars of SHA-256).
|
|
3
|
-
* Hash the page body
|
|
4
|
-
* (`canonical_url`, `last_updated`) that differ between
|
|
5
|
-
*
|
|
3
|
+
* Hash the page body and image metadata. Other frontmatter holds fields
|
|
4
|
+
* (`canonical_url`, `last_updated`) that can differ between conversion paths.
|
|
5
|
+
* Those fields stay excluded so unchanged pages keep the same hash.
|
|
6
6
|
*/
|
|
7
7
|
export declare function computeContentHash(markdown: string): Promise<string>;
|
|
8
8
|
export interface DatabaseAdapter {
|
|
@@ -10,7 +10,25 @@ function stripPageFrontmatter(markdown) {
|
|
|
10
10
|
if (document.errors.length || !isMap(document.contents))
|
|
11
11
|
return markdown;
|
|
12
12
|
const isPageFrontmatter = document.contents.items.some((item) => isScalar(item.key) && PAGE_FRONTMATTER_KEYS.has(String(item.key.value)));
|
|
13
|
-
|
|
13
|
+
if (!isPageFrontmatter)
|
|
14
|
+
return markdown;
|
|
15
|
+
const body = markdown.slice(match[0].length);
|
|
16
|
+
const meta = document.get("meta", true);
|
|
17
|
+
const images = [];
|
|
18
|
+
if (isMap(meta)) {
|
|
19
|
+
for (const { key, value } of meta.items) {
|
|
20
|
+
if (!isScalar(key) || typeof key.value !== "string" || !isScalar(value))
|
|
21
|
+
continue;
|
|
22
|
+
if (key.value !== "og:image" && !key.value.startsWith("og:image:") && key.value !== "twitter:image" && !key.value.startsWith("twitter:image:"))
|
|
23
|
+
continue;
|
|
24
|
+
if (typeof value.value === "string" || typeof value.value === "number" && Number.isFinite(value.value))
|
|
25
|
+
images.push([key.value, value.value]);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
if (!images.length)
|
|
29
|
+
return body;
|
|
30
|
+
images.sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
|
|
31
|
+
return JSON.stringify([images, body.trim()]);
|
|
14
32
|
}
|
|
15
33
|
export async function computeContentHash(markdown) {
|
|
16
34
|
const encoder = new TextEncoder();
|
|
@@ -20,7 +20,7 @@ function notFoundMarkdown(ctx, canonicalUrl, build) {
|
|
|
20
20
|
``,
|
|
21
21
|
`- [Sitemap](${resolveUrl("/sitemap.xml")})`,
|
|
22
22
|
`- [llms.txt](${resolveUrl("/llms.txt")})`,
|
|
23
|
-
`- [llms-full.txt](${resolveUrl("/llms-full.txt")})
|
|
23
|
+
...config.llmsFullTxt === false ? [] : [`- [llms-full.txt](${resolveUrl("/llms-full.txt")})`],
|
|
24
24
|
``
|
|
25
25
|
].join("\n");
|
|
26
26
|
const i18n = config.i18n;
|
|
@@ -2,8 +2,10 @@ import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
|
2
2
|
import { trackDrizzleWork } from "../db/drizzle/client.js";
|
|
3
3
|
import { getPageLastmods, getSitemapLastCrawledAt, markSitemapSeeded, resolveSeedRefreshWindowMs, seedRoutes } from "../db/queries.js";
|
|
4
4
|
import { logger } from "../logger.js";
|
|
5
|
+
import { markIndexedSitemapLastmods } from "../utils/sitemap-lastmod.js";
|
|
5
6
|
import { mapSitemapRoutes } from "../utils/sitemap-routes.js";
|
|
6
7
|
const DIAGNOSTICS_KEY = "_aiReadySitemapDiagnostics";
|
|
8
|
+
const INDEXED_LASTMODS_KEY = "_aiReadyIndexedSitemapLastmods";
|
|
7
9
|
function isDebug(_event) {
|
|
8
10
|
return !!useRuntimeConfig()["nuxt-ai-ready"]?.debug;
|
|
9
11
|
}
|
|
@@ -66,6 +68,7 @@ export default function sitemapSeederPlugin(nitroApp) {
|
|
|
66
68
|
const routeToUrl = mapSitemapRoutes(urls);
|
|
67
69
|
if (routeToUrl.size === 0)
|
|
68
70
|
return;
|
|
71
|
+
const routes = [...routeToUrl.entries()].map(([route, url]) => ({ route, url: url.loc, lastmod: url.lastmod }));
|
|
69
72
|
logger.debug(`[sitemap-seeder] Processing ${routeToUrl.size} routes from ${sitemapName}`);
|
|
70
73
|
const record = isDebug(event) ? (m) => recordDiagnostic(event, m) : void 0;
|
|
71
74
|
const readStart = Date.now();
|
|
@@ -75,6 +78,9 @@ export default function sitemapSeederPlugin(nitroApp) {
|
|
|
75
78
|
const lastmod = lastmods.get(route);
|
|
76
79
|
if (lastmod && !url.lastmod) {
|
|
77
80
|
url.lastmod = lastmod;
|
|
81
|
+
const context = event.context;
|
|
82
|
+
const enrichedUrls = context[INDEXED_LASTMODS_KEY] ??= /* @__PURE__ */ new Set();
|
|
83
|
+
enrichedUrls.add(url.loc);
|
|
78
84
|
enriched++;
|
|
79
85
|
}
|
|
80
86
|
}
|
|
@@ -92,7 +98,6 @@ export default function sitemapSeederPlugin(nitroApp) {
|
|
|
92
98
|
if (Date.now() - lastSeed < SEED_INTERVAL_MS)
|
|
93
99
|
return;
|
|
94
100
|
lastSeedAt.set(sitemapName, Date.now());
|
|
95
|
-
const routes = [...routeToUrl.entries()].map(([route, url]) => ({ route, url: url.loc }));
|
|
96
101
|
const urlCount = urls.length;
|
|
97
102
|
const seed = async () => {
|
|
98
103
|
const seedStart = Date.now();
|
|
@@ -120,6 +125,9 @@ export default function sitemapSeederPlugin(nitroApp) {
|
|
|
120
125
|
}
|
|
121
126
|
});
|
|
122
127
|
nitroApp.hooks.hook("sitemap:output", (ctx) => {
|
|
128
|
+
const enrichedUrls = ctx.event.context[INDEXED_LASTMODS_KEY];
|
|
129
|
+
if (enrichedUrls?.size)
|
|
130
|
+
ctx.sitemap = markIndexedSitemapLastmods(ctx.sitemap, enrichedUrls);
|
|
123
131
|
const diagnostics = ctx.event.context[DIAGNOSTICS_KEY];
|
|
124
132
|
if (!diagnostics?.length)
|
|
125
133
|
return;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { eventHandler, sendIterable, setHeader, setResponseHeader } from "#nuxtseo/h3";
|
|
1
|
+
import { assertMethod, eventHandler, sendIterable, setHeader, setResponseHeader } from "#nuxtseo/h3";
|
|
2
2
|
import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
3
3
|
import { getSiteConfig } from "#site-config/server/composables";
|
|
4
4
|
import { withSiteUrl } from "#site-config/server/composables/utils";
|
|
@@ -7,6 +7,7 @@ import { toDeployedRoute } from "../../route-path.js";
|
|
|
7
7
|
import { countPages, streamPages } from "../db/queries.js";
|
|
8
8
|
import { buildLlmsFullTxtHeader, formatPageForLlmsFullTxt } from "../utils/llms-full.js";
|
|
9
9
|
export default eventHandler(async (event) => {
|
|
10
|
+
assertMethod(event, "GET", true);
|
|
10
11
|
if (import.meta.prerender) {
|
|
11
12
|
setHeader(event, "Content-Type", "text/plain; charset=utf-8");
|
|
12
13
|
return "# llms-full.txt\n\nThis file is generated during prerender.";
|
|
@@ -1,29 +1,39 @@
|
|
|
1
|
-
import { eventHandler, setHeader } from "#nuxtseo/h3";
|
|
1
|
+
import { assertMethod, createError, eventHandler, getRequestURL, setHeader } from "#nuxtseo/h3";
|
|
2
2
|
import { defineCachedFunction, useRuntimeConfig } from "#nuxtseo/nitro";
|
|
3
3
|
import { publicCacheControl } from "../../cache-control.js";
|
|
4
|
-
import {
|
|
4
|
+
import { buildLlmsTxtFiles } from "../../llms-txt-utils.js";
|
|
5
|
+
import { toLogicalRoute } from "../../route-path.js";
|
|
5
6
|
const cachedBuilders = /* @__PURE__ */ new Map();
|
|
6
7
|
function getBuildLlmsTxtCached(maxAge) {
|
|
7
8
|
let cached = cachedBuilders.get(maxAge);
|
|
8
9
|
if (!cached) {
|
|
9
|
-
cached = defineCachedFunction(
|
|
10
|
-
name: "llms-txt",
|
|
10
|
+
cached = defineCachedFunction(buildLlmsTxtFiles, {
|
|
11
|
+
name: "llms-txt-files",
|
|
11
12
|
group: "ai-ready",
|
|
12
13
|
maxAge,
|
|
13
|
-
swr: true
|
|
14
|
+
swr: true,
|
|
15
|
+
// Share one discovery snapshot between root and child files, separated by origin.
|
|
16
|
+
getKey: (event) => getRequestURL(event, { xForwardedHost: true, xForwardedProto: true }).origin
|
|
14
17
|
});
|
|
15
18
|
cachedBuilders.set(maxAge, cached);
|
|
16
19
|
}
|
|
17
20
|
return cached;
|
|
18
21
|
}
|
|
19
22
|
export default eventHandler(async (event) => {
|
|
20
|
-
const runtimeConfig = useRuntimeConfig(event)
|
|
21
|
-
const
|
|
22
|
-
|
|
23
|
-
|
|
23
|
+
const runtimeConfig = useRuntimeConfig(event);
|
|
24
|
+
const path = toLogicalRoute(event.path.split("?")[0], runtimeConfig.app.baseURL);
|
|
25
|
+
if (path !== "/llms.txt" && !/^\/llms\/[^/]+\.txt$/i.test(path))
|
|
26
|
+
return;
|
|
27
|
+
assertMethod(event, "GET", true);
|
|
28
|
+
const config = runtimeConfig["nuxt-ai-ready"];
|
|
29
|
+
const cacheSeconds = config.llmsTxtCacheSeconds;
|
|
30
|
+
const cacheEnabled = !import.meta.dev && !import.meta.prerender && cacheSeconds > 0;
|
|
31
|
+
const files = cacheEnabled ? await getBuildLlmsTxtCached(cacheSeconds)(event) : await buildLlmsTxtFiles(event);
|
|
32
|
+
const content = files[path];
|
|
33
|
+
if (content === void 0)
|
|
34
|
+
throw createError({ statusCode: 404, statusMessage: "llms.txt file not found" });
|
|
24
35
|
setHeader(event, "Content-Type", "text/plain; charset=utf-8");
|
|
25
|
-
if (cacheEnabled)
|
|
36
|
+
if (cacheEnabled)
|
|
26
37
|
setHeader(event, "Cache-Control", publicCacheControl(cacheSeconds, 3600));
|
|
27
|
-
}
|
|
28
38
|
return content;
|
|
29
39
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { eventHandler, setHeader } from "#nuxtseo/h3";
|
|
1
|
+
import { assertMethod, eventHandler, setHeader } from "#nuxtseo/h3";
|
|
2
2
|
import { defineCachedFunction, useRuntimeConfig } from "#nuxtseo/nitro";
|
|
3
3
|
import { getSiteConfig } from "#site-config/server/composables";
|
|
4
4
|
import { publicCacheControl } from "../../cache-control.js";
|
|
@@ -42,6 +42,7 @@ function getBuildSitemapMdCached(maxAge) {
|
|
|
42
42
|
return cached;
|
|
43
43
|
}
|
|
44
44
|
export default eventHandler(async (event) => {
|
|
45
|
+
assertMethod(event, "GET", true);
|
|
45
46
|
const fullRuntimeConfig = useRuntimeConfig(event);
|
|
46
47
|
const runtimeConfig = fullRuntimeConfig["nuxt-ai-ready"];
|
|
47
48
|
const cacheSeconds = runtimeConfig.llmsTxtCacheSeconds ?? 600;
|