nuxt-ai-ready 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/module.json +1 -1
- package/dist/module.mjs +3 -2
- package/dist/runtime/llms-txt-utils.js +10 -8
- package/dist/runtime/server/db/queries.d.ts +47 -0
- package/dist/runtime/server/db/queries.js +84 -4
- package/dist/runtime/server/db/schema-sql.d.ts +1 -1
- package/dist/runtime/server/db/schema-sql.js +19 -2
- package/dist/runtime/server/db/shared.d.ts +25 -3
- package/dist/runtime/server/db/shared.js +49 -7
- package/dist/runtime/server/middleware/markdown.js +13 -1
- package/dist/runtime/server/routes/__ai-ready-debug.get.d.ts +25 -0
- package/dist/runtime/server/routes/__ai-ready-debug.get.js +49 -3
- package/dist/runtime/server/utils/runCron.d.ts +1 -0
- package/dist/runtime/server/utils/runCron.js +20 -0
- package/package.json +1 -1
package/dist/module.json
CHANGED
package/dist/module.mjs
CHANGED
|
@@ -271,8 +271,9 @@ function setupPrerenderHandler(dbPath, siteInfo, llmsTxtConfig) {
|
|
|
271
271
|
const publicDataDir = join(nitro.options.output.publicDir, "__ai-ready");
|
|
272
272
|
await mkdir(publicDataDir, { recursive: true });
|
|
273
273
|
if (state.db) {
|
|
274
|
-
const
|
|
275
|
-
const
|
|
274
|
+
const allPages = await queryAllPages(state.db, { includeErrors: true, excludeMarkdown: true });
|
|
275
|
+
const pages = allPages.filter((p) => !p.isError);
|
|
276
|
+
const errorRoutesList = allPages.filter((p) => p.isError).map((p) => p.route);
|
|
276
277
|
const jsonContent = JSON.stringify({
|
|
277
278
|
pages: pages.map((p) => ({
|
|
278
279
|
route: p.route,
|
|
@@ -127,20 +127,22 @@ Canonical Origin: ${siteConfig.url}`);
|
|
|
127
127
|
parts.push(normalizedContent);
|
|
128
128
|
parts.push("");
|
|
129
129
|
}
|
|
130
|
-
const pages = await
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
const prerendered = [];
|
|
130
|
+
const [pages, errorPages, urls] = await Promise.all([
|
|
131
|
+
queryPages(event),
|
|
132
|
+
queryPages(event, { where: { hasError: true } }),
|
|
133
|
+
fetchSitemapUrls(event)
|
|
134
|
+
]);
|
|
136
135
|
const seenPaths = /* @__PURE__ */ new Set();
|
|
136
|
+
const errorSet = new Set(errorPages.map((p) => p.route));
|
|
137
|
+
const prerendered = [];
|
|
137
138
|
for (const page of pages) {
|
|
138
|
-
prerendered.push({ pathname: page.route, title: page.title, description: page.description });
|
|
139
139
|
seenPaths.add(page.route);
|
|
140
|
+
prerendered.push({ pathname: page.route, title: page.title, description: page.description });
|
|
140
141
|
}
|
|
142
|
+
const devModeHint = import.meta.dev && prerendered.length === 0 ? " (dev mode - run `nuxi generate` for page titles)" : "";
|
|
141
143
|
const other = [];
|
|
142
144
|
for (const url of urls) {
|
|
143
|
-
const pathname = url.loc.startsWith("
|
|
145
|
+
const pathname = url.loc.startsWith("/") ? url.loc : new URL(url.loc).pathname;
|
|
144
146
|
if (!seenPaths.has(pathname) && !errorSet.has(pathname)) {
|
|
145
147
|
other.push({ pathname });
|
|
146
148
|
seenPaths.add(pathname);
|
|
@@ -24,6 +24,7 @@ export interface PageEntry {
|
|
|
24
24
|
headings: string;
|
|
25
25
|
keywords: string[];
|
|
26
26
|
updatedAt: string;
|
|
27
|
+
isError: boolean;
|
|
27
28
|
}
|
|
28
29
|
export interface PageData extends PageEntry {
|
|
29
30
|
markdown: string;
|
|
@@ -162,3 +163,49 @@ export interface IndexNowStats {
|
|
|
162
163
|
* Get IndexNow stats
|
|
163
164
|
*/
|
|
164
165
|
export declare function getIndexNowStats(event: H3Event | undefined): Promise<IndexNowStats>;
|
|
166
|
+
export interface CronRunRow {
|
|
167
|
+
id: number;
|
|
168
|
+
started_at: number;
|
|
169
|
+
finished_at: number | null;
|
|
170
|
+
duration_ms: number | null;
|
|
171
|
+
pages_indexed: number;
|
|
172
|
+
pages_remaining: number;
|
|
173
|
+
indexnow_submitted: number;
|
|
174
|
+
indexnow_remaining: number;
|
|
175
|
+
errors: string;
|
|
176
|
+
status: 'running' | 'success' | 'partial' | 'error';
|
|
177
|
+
}
|
|
178
|
+
export interface CronRun {
|
|
179
|
+
id: number;
|
|
180
|
+
startedAt: number;
|
|
181
|
+
finishedAt: number | null;
|
|
182
|
+
durationMs: number | null;
|
|
183
|
+
pagesIndexed: number;
|
|
184
|
+
pagesRemaining: number;
|
|
185
|
+
indexNowSubmitted: number;
|
|
186
|
+
indexNowRemaining: number;
|
|
187
|
+
errors: string[];
|
|
188
|
+
status: 'running' | 'success' | 'partial' | 'error';
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Start a cron run and return its ID
|
|
192
|
+
*/
|
|
193
|
+
export declare function startCronRun(event: H3Event | undefined): Promise<number | null>;
|
|
194
|
+
/**
|
|
195
|
+
* Complete a cron run with results
|
|
196
|
+
*/
|
|
197
|
+
export declare function completeCronRun(event: H3Event | undefined, runId: number, result: {
|
|
198
|
+
pagesIndexed: number;
|
|
199
|
+
pagesRemaining: number;
|
|
200
|
+
indexNowSubmitted: number;
|
|
201
|
+
indexNowRemaining: number;
|
|
202
|
+
errors: string[];
|
|
203
|
+
}): Promise<void>;
|
|
204
|
+
/**
|
|
205
|
+
* Get recent cron runs
|
|
206
|
+
*/
|
|
207
|
+
export declare function getRecentCronRuns(event: H3Event | undefined, limit?: number): Promise<CronRun[]>;
|
|
208
|
+
/**
|
|
209
|
+
* Clean up old cron runs (keep last N)
|
|
210
|
+
*/
|
|
211
|
+
export declare function cleanupOldCronRuns(event: H3Event | undefined, keepCount?: number): Promise<number>;
|
|
@@ -29,13 +29,18 @@ async function getPrerenderDb() {
|
|
|
29
29
|
const data = await m.readPageDataFromFilesystem();
|
|
30
30
|
const pages = data.pages || [];
|
|
31
31
|
const errorRoutes = new Set(data.errorRoutes || []);
|
|
32
|
+
const wantsMarkdown = (sql) => sql.includes("SELECT *") || sql.toLowerCase().includes("markdown");
|
|
32
33
|
return {
|
|
33
34
|
all: async (sql, params = []) => {
|
|
34
35
|
const isErrorQuery = sql.includes("is_error = 1") || params.includes(1) && sql.includes("is_error");
|
|
35
36
|
const excludeErrors = sql.includes("is_error = 0");
|
|
37
|
+
const includeMarkdown = wantsMarkdown(sql);
|
|
36
38
|
if (isErrorQuery) {
|
|
37
39
|
return pages.filter((p) => errorRoutes.has(p.route)).map((p) => ({
|
|
38
|
-
|
|
40
|
+
route: p.route,
|
|
41
|
+
title: p.title,
|
|
42
|
+
description: p.description,
|
|
43
|
+
...includeMarkdown ? { markdown: p.markdown } : {},
|
|
39
44
|
headings: p.headings,
|
|
40
45
|
keywords: JSON.stringify(p.keywords),
|
|
41
46
|
updated_at: p.updatedAt,
|
|
@@ -48,7 +53,7 @@ async function getPrerenderDb() {
|
|
|
48
53
|
route: p.route,
|
|
49
54
|
title: p.title,
|
|
50
55
|
description: p.description,
|
|
51
|
-
markdown: p.markdown,
|
|
56
|
+
...includeMarkdown ? { markdown: p.markdown } : {},
|
|
52
57
|
headings: p.headings,
|
|
53
58
|
keywords: JSON.stringify(p.keywords),
|
|
54
59
|
updated_at: p.updatedAt,
|
|
@@ -62,11 +67,12 @@ async function getPrerenderDb() {
|
|
|
62
67
|
const page = pages.find((p) => p.route === route);
|
|
63
68
|
if (!page)
|
|
64
69
|
return void 0;
|
|
70
|
+
const includeMarkdown = wantsMarkdown(sql);
|
|
65
71
|
return {
|
|
66
72
|
route: page.route,
|
|
67
73
|
title: page.title,
|
|
68
74
|
description: page.description,
|
|
69
|
-
markdown: page.markdown,
|
|
75
|
+
...includeMarkdown ? { markdown: page.markdown } : {},
|
|
70
76
|
headings: page.headings,
|
|
71
77
|
keywords: JSON.stringify(page.keywords),
|
|
72
78
|
updated_at: page.updatedAt,
|
|
@@ -110,7 +116,8 @@ function rowToEntry(row) {
|
|
|
110
116
|
description: row.description,
|
|
111
117
|
headings: row.headings,
|
|
112
118
|
keywords: JSON.parse(row.keywords || "[]"),
|
|
113
|
-
updatedAt: row.updated_at
|
|
119
|
+
updatedAt: row.updated_at,
|
|
120
|
+
isError: row.is_error === 1
|
|
114
121
|
};
|
|
115
122
|
}
|
|
116
123
|
function rowToData(row) {
|
|
@@ -374,3 +381,76 @@ export async function getIndexNowStats(event) {
|
|
|
374
381
|
lastError: stats.indexnow_last_error || null
|
|
375
382
|
};
|
|
376
383
|
}
|
|
384
|
+
function rowToCronRun(row) {
|
|
385
|
+
return {
|
|
386
|
+
id: row.id,
|
|
387
|
+
startedAt: row.started_at,
|
|
388
|
+
finishedAt: row.finished_at,
|
|
389
|
+
durationMs: row.duration_ms,
|
|
390
|
+
pagesIndexed: row.pages_indexed,
|
|
391
|
+
pagesRemaining: row.pages_remaining,
|
|
392
|
+
indexNowSubmitted: row.indexnow_submitted,
|
|
393
|
+
indexNowRemaining: row.indexnow_remaining,
|
|
394
|
+
errors: JSON.parse(row.errors || "[]"),
|
|
395
|
+
status: row.status
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
export async function startCronRun(event) {
|
|
399
|
+
const db = await getDb(event);
|
|
400
|
+
if (!db)
|
|
401
|
+
return null;
|
|
402
|
+
const now = Date.now();
|
|
403
|
+
await db.exec(
|
|
404
|
+
"INSERT INTO ai_ready_cron_runs (started_at, status) VALUES (?, ?)",
|
|
405
|
+
[now, "running"]
|
|
406
|
+
);
|
|
407
|
+
const row = await db.first("SELECT last_insert_rowid() as id");
|
|
408
|
+
return row?.id || null;
|
|
409
|
+
}
|
|
410
|
+
export async function completeCronRun(event, runId, result) {
|
|
411
|
+
const db = await getDb(event);
|
|
412
|
+
if (!db)
|
|
413
|
+
return;
|
|
414
|
+
const now = Date.now();
|
|
415
|
+
const row = await db.first("SELECT started_at FROM ai_ready_cron_runs WHERE id = ?", [runId]);
|
|
416
|
+
const durationMs = row ? now - row.started_at : null;
|
|
417
|
+
const status = result.errors.length > 0 ? result.pagesIndexed > 0 ? "partial" : "error" : "success";
|
|
418
|
+
await db.exec(`
|
|
419
|
+
UPDATE ai_ready_cron_runs SET
|
|
420
|
+
finished_at = ?,
|
|
421
|
+
duration_ms = ?,
|
|
422
|
+
pages_indexed = ?,
|
|
423
|
+
pages_remaining = ?,
|
|
424
|
+
indexnow_submitted = ?,
|
|
425
|
+
indexnow_remaining = ?,
|
|
426
|
+
errors = ?,
|
|
427
|
+
status = ?
|
|
428
|
+
WHERE id = ?
|
|
429
|
+
`, [now, durationMs, result.pagesIndexed, result.pagesRemaining, result.indexNowSubmitted, result.indexNowRemaining, JSON.stringify(result.errors), status, runId]);
|
|
430
|
+
}
|
|
431
|
+
export async function getRecentCronRuns(event, limit = 10) {
|
|
432
|
+
const db = await getDb(event);
|
|
433
|
+
if (!db)
|
|
434
|
+
return [];
|
|
435
|
+
const rows = await db.all(
|
|
436
|
+
"SELECT * FROM ai_ready_cron_runs ORDER BY started_at DESC LIMIT ?",
|
|
437
|
+
[limit]
|
|
438
|
+
);
|
|
439
|
+
return rows.map(rowToCronRun);
|
|
440
|
+
}
|
|
441
|
+
export async function cleanupOldCronRuns(event, keepCount = 50) {
|
|
442
|
+
const db = await getDb(event);
|
|
443
|
+
if (!db)
|
|
444
|
+
return 0;
|
|
445
|
+
const countRow = await db.first("SELECT COUNT(*) as count FROM ai_ready_cron_runs");
|
|
446
|
+
const total = countRow?.count || 0;
|
|
447
|
+
if (total <= keepCount)
|
|
448
|
+
return 0;
|
|
449
|
+
const deleteCount = total - keepCount;
|
|
450
|
+
await db.exec(`
|
|
451
|
+
DELETE FROM ai_ready_cron_runs WHERE id IN (
|
|
452
|
+
SELECT id FROM ai_ready_cron_runs ORDER BY started_at ASC LIMIT ?
|
|
453
|
+
)
|
|
454
|
+
`, [deleteCount]);
|
|
455
|
+
return deleteCount;
|
|
456
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export const SCHEMA_VERSION = "v1.
|
|
1
|
+
export const SCHEMA_VERSION = "v1.7.0";
|
|
2
2
|
const PAGES_TABLE_SQL = `
|
|
3
3
|
CREATE TABLE IF NOT EXISTS ai_ready_pages (
|
|
4
4
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -54,9 +54,24 @@ CREATE TABLE IF NOT EXISTS _ai_ready_info (
|
|
|
54
54
|
checksum TEXT,
|
|
55
55
|
ready INTEGER DEFAULT 0
|
|
56
56
|
)`;
|
|
57
|
+
const CRON_RUNS_TABLE_SQL = `
|
|
58
|
+
CREATE TABLE IF NOT EXISTS ai_ready_cron_runs (
|
|
59
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
60
|
+
started_at INTEGER NOT NULL,
|
|
61
|
+
finished_at INTEGER,
|
|
62
|
+
duration_ms INTEGER,
|
|
63
|
+
pages_indexed INTEGER DEFAULT 0,
|
|
64
|
+
pages_remaining INTEGER DEFAULT 0,
|
|
65
|
+
indexnow_submitted INTEGER DEFAULT 0,
|
|
66
|
+
indexnow_remaining INTEGER DEFAULT 0,
|
|
67
|
+
errors TEXT DEFAULT '[]',
|
|
68
|
+
status TEXT DEFAULT 'running'
|
|
69
|
+
)`;
|
|
70
|
+
const CRON_RUNS_INDEX_SQL = "CREATE INDEX IF NOT EXISTS idx_ai_ready_cron_runs_started ON ai_ready_cron_runs(started_at DESC)";
|
|
57
71
|
export const DROP_TABLES_SQL = [
|
|
58
72
|
"DROP TABLE IF EXISTS ai_ready_pages_fts",
|
|
59
73
|
"DROP TABLE IF EXISTS ai_ready_pages",
|
|
74
|
+
"DROP TABLE IF EXISTS ai_ready_cron_runs",
|
|
60
75
|
"DROP TABLE IF EXISTS _ai_ready_info",
|
|
61
76
|
// Legacy unprefixed tables (migration from v1.0.0)
|
|
62
77
|
"DROP TABLE IF EXISTS pages_fts",
|
|
@@ -67,5 +82,7 @@ export const ALL_SCHEMA_SQL = [
|
|
|
67
82
|
...PAGES_INDEXES_SQL,
|
|
68
83
|
FTS_TABLE_SQL,
|
|
69
84
|
...FTS_TRIGGERS_SQL,
|
|
70
|
-
INFO_TABLE_SQL
|
|
85
|
+
INFO_TABLE_SQL,
|
|
86
|
+
CRON_RUNS_TABLE_SQL,
|
|
87
|
+
CRON_RUNS_INDEX_SQL
|
|
71
88
|
];
|
|
@@ -52,16 +52,37 @@ export interface PageOutput {
|
|
|
52
52
|
updatedAt: string;
|
|
53
53
|
isError: boolean;
|
|
54
54
|
}
|
|
55
|
+
export interface PageMetaOutput {
|
|
56
|
+
route: string;
|
|
57
|
+
title: string;
|
|
58
|
+
description: string;
|
|
59
|
+
headings: string;
|
|
60
|
+
keywords: string[];
|
|
61
|
+
contentHash?: string;
|
|
62
|
+
updatedAt: string;
|
|
63
|
+
isError: boolean;
|
|
64
|
+
}
|
|
55
65
|
/**
|
|
56
66
|
* Insert or update a page
|
|
57
67
|
*/
|
|
58
68
|
export declare function insertPage(db: DatabaseAdapter, page: PageInput): Promise<void>;
|
|
69
|
+
export interface QueryAllPagesOptions {
|
|
70
|
+
includeErrors?: boolean;
|
|
71
|
+
excludeMarkdown?: boolean;
|
|
72
|
+
}
|
|
59
73
|
/**
|
|
60
74
|
* Query all pages from database
|
|
75
|
+
* @param db - Database adapter
|
|
76
|
+
* @param options - Query options
|
|
77
|
+
* @param options.excludeMarkdown - If true, omit markdown field to reduce memory usage
|
|
61
78
|
*/
|
|
62
|
-
export declare function queryAllPages(db: DatabaseAdapter, options?: {
|
|
63
|
-
|
|
79
|
+
export declare function queryAllPages(db: DatabaseAdapter, options?: QueryAllPagesOptions & {
|
|
80
|
+
excludeMarkdown: true;
|
|
81
|
+
}): Promise<PageMetaOutput[]>;
|
|
82
|
+
export declare function queryAllPages(db: DatabaseAdapter, options?: QueryAllPagesOptions & {
|
|
83
|
+
excludeMarkdown?: false;
|
|
64
84
|
}): Promise<PageOutput[]>;
|
|
85
|
+
export declare function queryAllPages(db: DatabaseAdapter, options?: QueryAllPagesOptions): Promise<PageOutput[] | PageMetaOutput[]>;
|
|
65
86
|
export interface DumpRow {
|
|
66
87
|
route: string;
|
|
67
88
|
route_key: string;
|
|
@@ -79,7 +100,8 @@ export interface DumpRow {
|
|
|
79
100
|
last_seen_at: number | null;
|
|
80
101
|
}
|
|
81
102
|
/**
|
|
82
|
-
* Export database as compressed dump (base64 gzip)
|
|
103
|
+
* Export database as compressed dump (base64 gzip) using batched streaming
|
|
104
|
+
* Processes rows in batches to avoid loading entire database into memory
|
|
83
105
|
*/
|
|
84
106
|
export declare function exportDbDump(db: DatabaseAdapter): Promise<string>;
|
|
85
107
|
/**
|
|
@@ -86,12 +86,13 @@ export async function insertPage(db, page) {
|
|
|
86
86
|
}
|
|
87
87
|
export async function queryAllPages(db, options) {
|
|
88
88
|
const where = options?.includeErrors ? "" : "WHERE is_error = 0";
|
|
89
|
-
const
|
|
89
|
+
const fields = options?.excludeMarkdown ? "route, title, description, headings, keywords, content_hash, updated_at, is_error" : "route, title, description, markdown, headings, keywords, content_hash, updated_at, is_error";
|
|
90
|
+
const rows = await db.all(`SELECT ${fields} FROM ai_ready_pages ${where}`);
|
|
90
91
|
return rows.map((row) => ({
|
|
91
92
|
route: row.route,
|
|
92
93
|
title: row.title,
|
|
93
94
|
description: row.description,
|
|
94
|
-
markdown: row.markdown,
|
|
95
|
+
...options?.excludeMarkdown ? {} : { markdown: row.markdown },
|
|
95
96
|
headings: row.headings,
|
|
96
97
|
keywords: JSON.parse(row.keywords || "[]"),
|
|
97
98
|
contentHash: row.content_hash || void 0,
|
|
@@ -99,12 +100,53 @@ export async function queryAllPages(db, options) {
|
|
|
99
100
|
isError: row.is_error === 1
|
|
100
101
|
}));
|
|
101
102
|
}
|
|
103
|
+
const DUMP_BATCH_SIZE = 500;
|
|
102
104
|
export async function exportDbDump(db) {
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
105
|
+
const encoder = new TextEncoder();
|
|
106
|
+
const chunks = [];
|
|
107
|
+
const compressionStream = new CompressionStream("gzip");
|
|
108
|
+
const writer = compressionStream.writable.getWriter();
|
|
109
|
+
const reader = compressionStream.readable.getReader();
|
|
110
|
+
const readPromise = (async () => {
|
|
111
|
+
while (true) {
|
|
112
|
+
const { done, value } = await reader.read();
|
|
113
|
+
if (done)
|
|
114
|
+
break;
|
|
115
|
+
chunks.push(value);
|
|
116
|
+
}
|
|
117
|
+
})();
|
|
118
|
+
await writer.write(encoder.encode("["));
|
|
119
|
+
let offset = 0;
|
|
120
|
+
let first = true;
|
|
121
|
+
while (true) {
|
|
122
|
+
const rows = await db.all(`
|
|
123
|
+
SELECT route, route_key, title, description, markdown, headings, keywords, content_hash, updated_at, indexed_at, is_error, indexed, source, last_seen_at
|
|
124
|
+
FROM ai_ready_pages
|
|
125
|
+
ORDER BY route
|
|
126
|
+
LIMIT ${DUMP_BATCH_SIZE} OFFSET ${offset}
|
|
127
|
+
`);
|
|
128
|
+
if (rows.length === 0)
|
|
129
|
+
break;
|
|
130
|
+
for (const row of rows) {
|
|
131
|
+
const prefix = first ? "" : ",";
|
|
132
|
+
first = false;
|
|
133
|
+
await writer.write(encoder.encode(prefix + JSON.stringify(row)));
|
|
134
|
+
}
|
|
135
|
+
if (rows.length < DUMP_BATCH_SIZE)
|
|
136
|
+
break;
|
|
137
|
+
offset += DUMP_BATCH_SIZE;
|
|
138
|
+
}
|
|
139
|
+
await writer.write(encoder.encode("]"));
|
|
140
|
+
await writer.close();
|
|
141
|
+
await readPromise;
|
|
142
|
+
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
|
143
|
+
const result = new Uint8Array(totalLength);
|
|
144
|
+
let pos = 0;
|
|
145
|
+
for (const chunk of chunks) {
|
|
146
|
+
result.set(chunk, pos);
|
|
147
|
+
pos += chunk.length;
|
|
148
|
+
}
|
|
149
|
+
return Buffer.from(result).toString("base64");
|
|
108
150
|
}
|
|
109
151
|
export async function importDbDump(db, rows) {
|
|
110
152
|
for (const row of rows) {
|
|
@@ -13,7 +13,8 @@ export default defineEventHandler(async (event) => {
|
|
|
13
13
|
const { path, isExplicit } = renderInfo;
|
|
14
14
|
const config = useRuntimeConfig(event)["nuxt-ai-ready"];
|
|
15
15
|
const response = await event.fetch(path, {
|
|
16
|
-
headers: { [INTERNAL_HEADER]: "1" }
|
|
16
|
+
headers: { [INTERNAL_HEADER]: "1" },
|
|
17
|
+
redirect: "manual"
|
|
17
18
|
}).catch((e) => {
|
|
18
19
|
logger.error(`Failed to fetch HTML for ${path}`, e);
|
|
19
20
|
return null;
|
|
@@ -28,6 +29,17 @@ export default defineEventHandler(async (event) => {
|
|
|
28
29
|
}
|
|
29
30
|
return;
|
|
30
31
|
}
|
|
32
|
+
if (response.status >= 300 && response.status < 400) {
|
|
33
|
+
const location = response.headers.get("location");
|
|
34
|
+
if (location) {
|
|
35
|
+
const redirectTarget = location.endsWith("/") ? `${location.slice(0, -1)}.md` : `${location}.md`;
|
|
36
|
+
setHeader(event, "location", redirectTarget);
|
|
37
|
+
return createError({
|
|
38
|
+
statusCode: response.status,
|
|
39
|
+
statusMessage: response.statusText
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
}
|
|
31
43
|
if (!response.ok) {
|
|
32
44
|
if (isExplicit) {
|
|
33
45
|
return createError({
|
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
interface CronRunInfo {
|
|
2
|
+
id: number;
|
|
3
|
+
startedAt: string;
|
|
4
|
+
finishedAt: string | null;
|
|
5
|
+
durationMs: number | null;
|
|
6
|
+
status: string;
|
|
7
|
+
pagesIndexed: number;
|
|
8
|
+
pagesRemaining: number;
|
|
9
|
+
indexNowSubmitted: number;
|
|
10
|
+
indexNowRemaining: number;
|
|
11
|
+
errors: string[];
|
|
12
|
+
}
|
|
1
13
|
interface DebugInfo {
|
|
2
14
|
version: string;
|
|
3
15
|
environment: {
|
|
@@ -10,6 +22,19 @@ interface DebugInfo {
|
|
|
10
22
|
cacheMaxAgeSeconds: number;
|
|
11
23
|
mdreamOptions: unknown;
|
|
12
24
|
};
|
|
25
|
+
runtimeSync?: {
|
|
26
|
+
total: number;
|
|
27
|
+
indexed: number;
|
|
28
|
+
pending: number;
|
|
29
|
+
sitemapSeededAt: string | null;
|
|
30
|
+
};
|
|
31
|
+
indexNow?: {
|
|
32
|
+
pending: number;
|
|
33
|
+
totalSubmitted: number;
|
|
34
|
+
lastSubmittedAt: string | null;
|
|
35
|
+
lastError: string | null;
|
|
36
|
+
};
|
|
37
|
+
cronRuns?: CronRunInfo[];
|
|
13
38
|
pageData: {
|
|
14
39
|
source: string;
|
|
15
40
|
pageCount: number;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createError, eventHandler, setHeader } from "h3";
|
|
2
2
|
import { useRuntimeConfig } from "nitropack/runtime";
|
|
3
|
-
import { queryPages } from "../db/queries.js";
|
|
3
|
+
import { countPages, countPagesNeedingIndexNowSync, getIndexNowStats, getRecentCronRuns, getSitemapSeededAt, queryPages } from "../db/queries.js";
|
|
4
4
|
export default eventHandler(async (event) => {
|
|
5
5
|
const runtimeConfig = useRuntimeConfig(event)["nuxt-ai-ready"];
|
|
6
6
|
if (!runtimeConfig.debug) {
|
|
@@ -19,8 +19,10 @@ export default eventHandler(async (event) => {
|
|
|
19
19
|
} else {
|
|
20
20
|
mode = "production";
|
|
21
21
|
}
|
|
22
|
-
const pages = await
|
|
23
|
-
|
|
22
|
+
const [pages, errorRoutes] = await Promise.all([
|
|
23
|
+
queryPages(event),
|
|
24
|
+
queryPages(event, { where: { hasError: true } })
|
|
25
|
+
]);
|
|
24
26
|
let source;
|
|
25
27
|
if (isDev) {
|
|
26
28
|
source = "empty (dev mode returns empty array)";
|
|
@@ -90,6 +92,47 @@ export default eventHandler(async (event) => {
|
|
|
90
92
|
issues.push("Prerender mode but no pages found");
|
|
91
93
|
suggestions.push("Check if pages.db exists in .data/ai-ready/");
|
|
92
94
|
}
|
|
95
|
+
let runtimeSyncInfo;
|
|
96
|
+
let indexNowInfo;
|
|
97
|
+
let cronRunsInfo;
|
|
98
|
+
if (!isDev && !isPrerender) {
|
|
99
|
+
const [total, pending, sitemapSeededAt, cronRuns] = await Promise.all([
|
|
100
|
+
countPages(event),
|
|
101
|
+
countPages(event, { where: { pending: true } }),
|
|
102
|
+
getSitemapSeededAt(event),
|
|
103
|
+
getRecentCronRuns(event, 20)
|
|
104
|
+
]);
|
|
105
|
+
runtimeSyncInfo = {
|
|
106
|
+
total,
|
|
107
|
+
indexed: total - pending,
|
|
108
|
+
pending,
|
|
109
|
+
sitemapSeededAt: sitemapSeededAt ? new Date(sitemapSeededAt).toISOString() : null
|
|
110
|
+
};
|
|
111
|
+
cronRunsInfo = cronRuns.map((run) => ({
|
|
112
|
+
id: run.id,
|
|
113
|
+
startedAt: new Date(run.startedAt).toISOString(),
|
|
114
|
+
finishedAt: run.finishedAt ? new Date(run.finishedAt).toISOString() : null,
|
|
115
|
+
durationMs: run.durationMs,
|
|
116
|
+
status: run.status,
|
|
117
|
+
pagesIndexed: run.pagesIndexed,
|
|
118
|
+
pagesRemaining: run.pagesRemaining,
|
|
119
|
+
indexNowSubmitted: run.indexNowSubmitted,
|
|
120
|
+
indexNowRemaining: run.indexNowRemaining,
|
|
121
|
+
errors: run.errors
|
|
122
|
+
}));
|
|
123
|
+
if (runtimeConfig.indexNowKey) {
|
|
124
|
+
const [indexNowPending, indexNowStats] = await Promise.all([
|
|
125
|
+
countPagesNeedingIndexNowSync(event),
|
|
126
|
+
getIndexNowStats(event)
|
|
127
|
+
]);
|
|
128
|
+
indexNowInfo = {
|
|
129
|
+
pending: indexNowPending,
|
|
130
|
+
totalSubmitted: indexNowStats.totalSubmitted,
|
|
131
|
+
lastSubmittedAt: indexNowStats.lastSubmittedAt ? new Date(indexNowStats.lastSubmittedAt).toISOString() : null,
|
|
132
|
+
lastError: indexNowStats.lastError
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
}
|
|
93
136
|
const debugInfo = {
|
|
94
137
|
version: runtimeConfig.version || "unknown",
|
|
95
138
|
environment: {
|
|
@@ -102,6 +145,9 @@ export default eventHandler(async (event) => {
|
|
|
102
145
|
cacheMaxAgeSeconds: runtimeConfig.cacheMaxAgeSeconds,
|
|
103
146
|
mdreamOptions: runtimeConfig.mdreamOptions
|
|
104
147
|
},
|
|
148
|
+
runtimeSync: runtimeSyncInfo,
|
|
149
|
+
indexNow: indexNowInfo,
|
|
150
|
+
cronRuns: cronRunsInfo,
|
|
105
151
|
pageData: {
|
|
106
152
|
source,
|
|
107
153
|
pageCount: pages.length,
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { useRuntimeConfig } from "nitropack/runtime";
|
|
2
|
+
import { cleanupOldCronRuns, completeCronRun, startCronRun } from "../db/queries.js";
|
|
2
3
|
import { batchIndexPages } from "./batchIndex.js";
|
|
3
4
|
import { syncToIndexNow } from "./indexnow.js";
|
|
4
5
|
export async function runCron(event, options) {
|
|
5
6
|
const config = useRuntimeConfig()["nuxt-ai-ready"];
|
|
6
7
|
const results = {};
|
|
8
|
+
const allErrors = [];
|
|
9
|
+
const runId = await startCronRun(event);
|
|
10
|
+
results.runId = runId;
|
|
7
11
|
if (config.runtimeSync.enabled) {
|
|
8
12
|
const limit = options?.batchSize ?? config.runtimeSync.batchSize;
|
|
9
13
|
const indexResult = await batchIndexPages(event, {
|
|
@@ -16,6 +20,9 @@ export async function runCron(event, options) {
|
|
|
16
20
|
errors: indexResult.errors.length > 0 ? indexResult.errors : void 0,
|
|
17
21
|
complete: indexResult.complete
|
|
18
22
|
};
|
|
23
|
+
if (indexResult.errors.length > 0) {
|
|
24
|
+
allErrors.push(...indexResult.errors);
|
|
25
|
+
}
|
|
19
26
|
}
|
|
20
27
|
if (config.indexNowKey) {
|
|
21
28
|
const indexNowResult = await syncToIndexNow(event, 100).catch((err) => {
|
|
@@ -27,6 +34,19 @@ export async function runCron(event, options) {
|
|
|
27
34
|
remaining: indexNowResult.remaining,
|
|
28
35
|
error: indexNowResult.error
|
|
29
36
|
};
|
|
37
|
+
if (indexNowResult.error) {
|
|
38
|
+
allErrors.push(`IndexNow: ${indexNowResult.error}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
if (runId) {
|
|
42
|
+
await completeCronRun(event, runId, {
|
|
43
|
+
pagesIndexed: results.index?.indexed || 0,
|
|
44
|
+
pagesRemaining: results.index?.remaining || 0,
|
|
45
|
+
indexNowSubmitted: results.indexNow?.submitted || 0,
|
|
46
|
+
indexNowRemaining: results.indexNow?.remaining || 0,
|
|
47
|
+
errors: allErrors
|
|
48
|
+
});
|
|
49
|
+
await cleanupOldCronRuns(event, 50);
|
|
30
50
|
}
|
|
31
51
|
return results;
|
|
32
52
|
}
|