nuxt-ai-ready 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/module.d.mts +12 -1
- package/dist/module.json +1 -1
- package/dist/module.mjs +226 -352
- package/dist/runtime/index.d.ts +5 -7
- package/dist/runtime/index.js +14 -3
- package/dist/runtime/llms-txt-format.d.ts +8 -0
- package/dist/runtime/llms-txt-format.js +32 -0
- package/dist/runtime/llms-txt-utils.d.ts +2 -5
- package/dist/runtime/llms-txt-utils.js +31 -88
- package/dist/runtime/server/db/index.d.ts +3 -7
- package/dist/runtime/server/db/index.js +14 -31
- package/dist/runtime/server/db/queries.d.ts +106 -26
- package/dist/runtime/server/db/queries.js +330 -33
- package/dist/runtime/server/db/schema-sql.d.ts +3 -0
- package/dist/runtime/server/db/schema-sql.js +71 -0
- package/dist/runtime/server/db/shared.d.ts +88 -0
- package/dist/runtime/server/db/shared.js +117 -0
- package/dist/runtime/server/mcp/resources/pages.js +11 -3
- package/dist/runtime/server/mcp/tools/list-pages.js +33 -7
- package/dist/runtime/server/mcp/tools/search-pages.js +3 -3
- package/dist/runtime/server/middleware/markdown.js +2 -3
- package/dist/runtime/server/middleware/markdown.prerender.js +6 -5
- package/dist/runtime/server/plugins/db-restore.js +10 -8
- package/dist/runtime/server/plugins/sitemap-seeder.js +44 -0
- package/dist/runtime/server/routes/__ai-ready/indexnow.post.d.ts +2 -0
- package/dist/runtime/server/routes/__ai-ready/indexnow.post.js +18 -0
- package/dist/runtime/server/routes/__ai-ready/poll.post.d.ts +8 -0
- package/dist/runtime/server/routes/__ai-ready/poll.post.js +25 -0
- package/dist/runtime/server/routes/__ai-ready/prune.post.d.ts +14 -0
- package/dist/runtime/server/routes/__ai-ready/prune.post.js +21 -0
- package/dist/runtime/server/routes/__ai-ready/status.get.d.ts +2 -0
- package/dist/runtime/server/routes/__ai-ready/status.get.js +28 -0
- package/dist/runtime/server/routes/__ai-ready-debug.get.js +13 -14
- package/dist/runtime/server/routes/indexnow-key.get.d.ts +6 -0
- package/dist/runtime/server/routes/indexnow-key.get.js +10 -0
- package/dist/runtime/server/routes/llms-full.txt.get.d.ts +1 -1
- package/dist/runtime/server/routes/llms-full.txt.get.js +34 -3
- package/dist/runtime/server/tasks/ai-ready-index.d.ts +11 -0
- package/dist/runtime/server/tasks/ai-ready-index.js +39 -0
- package/dist/runtime/server/utils/batchIndex.d.ts +26 -0
- package/dist/runtime/server/utils/batchIndex.js +53 -0
- package/dist/runtime/server/utils/indexPage.d.ts +8 -2
- package/dist/runtime/server/utils/indexPage.js +38 -23
- package/dist/runtime/server/utils/indexnow.d.ts +27 -0
- package/dist/runtime/server/utils/indexnow.js +66 -0
- package/dist/runtime/server/utils/keywords.d.ts +0 -4
- package/dist/runtime/server/utils/keywords.js +0 -3
- package/dist/runtime/server/utils/llms-full.d.ts +11 -0
- package/dist/runtime/server/utils/llms-full.js +39 -0
- package/dist/runtime/server/utils/sitemap.js +3 -3
- package/dist/runtime/server/utils.d.ts +12 -10
- package/dist/runtime/server/utils.js +63 -94
- package/dist/runtime/types.d.ts +64 -40
- package/package.json +28 -29
- package/dist/runtime/mcp.d.ts +0 -32
- package/dist/runtime/mcp.js +0 -5
- package/dist/runtime/server/db/dump.d.ts +0 -29
- package/dist/runtime/server/db/dump.js +0 -29
- package/dist/runtime/server/db/schema.d.ts +0 -8
- package/dist/runtime/server/db/schema.js +0 -124
- package/dist/runtime/server/plugins/page-indexer.js +0 -68
- package/dist/runtime/server/tsconfig.json +0 -3
- package/dist/runtime/server/utils/pageData.d.ts +0 -28
- package/dist/runtime/server/utils/pageData.js +0 -43
- package/mcp.d.ts +0 -4
- /package/dist/runtime/{nuxt → app}/plugins/md-hints.prerender.d.ts +0 -0
- /package/dist/runtime/{nuxt → app}/plugins/md-hints.prerender.js +0 -0
- /package/dist/runtime/server/plugins/{page-indexer.d.ts → sitemap-seeder.d.ts} +0 -0
|
@@ -1,5 +1,107 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
import { useEvent } from "nitropack/runtime";
|
|
2
|
+
import { useDatabase } from "./index.js";
|
|
3
|
+
import { normalizeRouteKey } from "./shared.js";
|
|
4
|
+
function getEventFromContext(providedEvent) {
|
|
5
|
+
if (providedEvent)
|
|
6
|
+
return providedEvent;
|
|
7
|
+
try {
|
|
8
|
+
return useEvent();
|
|
9
|
+
} catch {
|
|
10
|
+
return void 0;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
let devWarningShown = false;
|
|
14
|
+
async function getDb(event) {
|
|
15
|
+
if (import.meta.dev) {
|
|
16
|
+
if (!devWarningShown) {
|
|
17
|
+
console.warn("[nuxt-ai-ready] Page data unavailable in dev. Run `nuxi generate` for full metadata.");
|
|
18
|
+
devWarningShown = true;
|
|
19
|
+
}
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
if (import.meta.prerender) {
|
|
23
|
+
return getPrerenderDb();
|
|
24
|
+
}
|
|
25
|
+
return useDatabase(getEventFromContext(event));
|
|
26
|
+
}
|
|
27
|
+
async function getPrerenderDb() {
|
|
28
|
+
const m = await import("#ai-ready-virtual/read-page-data.mjs");
|
|
29
|
+
const data = await m.readPageDataFromFilesystem();
|
|
30
|
+
const pages = data.pages || [];
|
|
31
|
+
const errorRoutes = new Set(data.errorRoutes || []);
|
|
32
|
+
return {
|
|
33
|
+
all: async (sql, params = []) => {
|
|
34
|
+
const isErrorQuery = sql.includes("is_error = 1") || params.includes(1) && sql.includes("is_error");
|
|
35
|
+
const excludeErrors = sql.includes("is_error = 0");
|
|
36
|
+
if (isErrorQuery) {
|
|
37
|
+
return pages.filter((p) => errorRoutes.has(p.route)).map((p) => ({
|
|
38
|
+
...p,
|
|
39
|
+
headings: p.headings,
|
|
40
|
+
keywords: JSON.stringify(p.keywords),
|
|
41
|
+
updated_at: p.updatedAt,
|
|
42
|
+
is_error: 1,
|
|
43
|
+
indexed: 0
|
|
44
|
+
}));
|
|
45
|
+
}
|
|
46
|
+
const filtered = excludeErrors ? pages.filter((p) => !errorRoutes.has(p.route)) : pages;
|
|
47
|
+
return filtered.map((p) => ({
|
|
48
|
+
route: p.route,
|
|
49
|
+
title: p.title,
|
|
50
|
+
description: p.description,
|
|
51
|
+
markdown: p.markdown,
|
|
52
|
+
headings: p.headings,
|
|
53
|
+
keywords: JSON.stringify(p.keywords),
|
|
54
|
+
updated_at: p.updatedAt,
|
|
55
|
+
is_error: errorRoutes.has(p.route) ? 1 : 0,
|
|
56
|
+
indexed: 1
|
|
57
|
+
}));
|
|
58
|
+
},
|
|
59
|
+
first: async (sql, params = []) => {
|
|
60
|
+
if (sql.includes("WHERE route = ?")) {
|
|
61
|
+
const route = params[0];
|
|
62
|
+
const page = pages.find((p) => p.route === route);
|
|
63
|
+
if (!page)
|
|
64
|
+
return void 0;
|
|
65
|
+
return {
|
|
66
|
+
route: page.route,
|
|
67
|
+
title: page.title,
|
|
68
|
+
description: page.description,
|
|
69
|
+
markdown: page.markdown,
|
|
70
|
+
headings: page.headings,
|
|
71
|
+
keywords: JSON.stringify(page.keywords),
|
|
72
|
+
updated_at: page.updatedAt,
|
|
73
|
+
is_error: errorRoutes.has(page.route) ? 1 : 0,
|
|
74
|
+
indexed: 1
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
return void 0;
|
|
78
|
+
},
|
|
79
|
+
exec: async () => {
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
function buildWhereClause(where) {
|
|
84
|
+
const conditions = [];
|
|
85
|
+
const params = [];
|
|
86
|
+
if (where?.pending !== void 0) {
|
|
87
|
+
conditions.push("indexed = ?");
|
|
88
|
+
params.push(where.pending ? 0 : 1);
|
|
89
|
+
}
|
|
90
|
+
if (where?.hasError !== void 0) {
|
|
91
|
+
conditions.push("is_error = ?");
|
|
92
|
+
params.push(where.hasError ? 1 : 0);
|
|
93
|
+
}
|
|
94
|
+
if (where?.source) {
|
|
95
|
+
conditions.push("source = ?");
|
|
96
|
+
params.push(where.source);
|
|
97
|
+
}
|
|
98
|
+
if (where?.hasError === void 0) {
|
|
99
|
+
conditions.push("is_error = 0");
|
|
100
|
+
}
|
|
101
|
+
return {
|
|
102
|
+
sql: conditions.length ? `WHERE ${conditions.join(" AND ")}` : "",
|
|
103
|
+
params
|
|
104
|
+
};
|
|
3
105
|
}
|
|
4
106
|
function rowToEntry(row) {
|
|
5
107
|
return {
|
|
@@ -11,22 +113,78 @@ function rowToEntry(row) {
|
|
|
11
113
|
updatedAt: row.updated_at
|
|
12
114
|
};
|
|
13
115
|
}
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
116
|
+
function rowToData(row) {
|
|
117
|
+
return {
|
|
118
|
+
...rowToEntry(row),
|
|
119
|
+
markdown: row.markdown
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
export async function queryPages(event, options = {}) {
|
|
123
|
+
const { route, includeMarkdown, where, limit, offset } = options;
|
|
124
|
+
const db = await getDb(event);
|
|
125
|
+
if (!db)
|
|
126
|
+
return route ? void 0 : [];
|
|
127
|
+
if (route) {
|
|
128
|
+
const row = await db.first("SELECT * FROM ai_ready_pages WHERE route = ?", [route]);
|
|
129
|
+
if (!row)
|
|
130
|
+
return void 0;
|
|
131
|
+
return includeMarkdown ? rowToData(row) : rowToEntry(row);
|
|
132
|
+
}
|
|
133
|
+
const { sql: whereClause, params } = buildWhereClause(where);
|
|
134
|
+
let sql = `SELECT * FROM ai_ready_pages ${whereClause}`;
|
|
135
|
+
if (limit) {
|
|
136
|
+
sql += ` LIMIT ?`;
|
|
137
|
+
params.push(limit);
|
|
138
|
+
if (offset) {
|
|
139
|
+
sql += ` OFFSET ?`;
|
|
140
|
+
params.push(offset);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
const rows = await db.all(sql, params);
|
|
144
|
+
return includeMarkdown ? rows.map(rowToData) : rows.map(rowToEntry);
|
|
17
145
|
}
|
|
18
|
-
export async function
|
|
19
|
-
const
|
|
20
|
-
|
|
146
|
+
export async function* streamPages(event, options = {}) {
|
|
147
|
+
const db = await getDb(event);
|
|
148
|
+
if (!db)
|
|
149
|
+
return;
|
|
150
|
+
const batchSize = options.batchSize || 50;
|
|
151
|
+
let offset = 0;
|
|
152
|
+
while (true) {
|
|
153
|
+
const rows = await db.all(
|
|
154
|
+
`SELECT * FROM ai_ready_pages WHERE is_error = 0 ORDER BY route LIMIT ? OFFSET ?`,
|
|
155
|
+
[batchSize, offset]
|
|
156
|
+
);
|
|
157
|
+
if (rows.length === 0)
|
|
158
|
+
break;
|
|
159
|
+
for (const row of rows) {
|
|
160
|
+
yield rowToData(row);
|
|
161
|
+
}
|
|
162
|
+
if (rows.length < batchSize)
|
|
163
|
+
break;
|
|
164
|
+
offset += batchSize;
|
|
165
|
+
}
|
|
21
166
|
}
|
|
22
|
-
export async function
|
|
23
|
-
const
|
|
24
|
-
|
|
167
|
+
export async function countPages(event, options = {}) {
|
|
168
|
+
const db = await getDb(event);
|
|
169
|
+
if (!db)
|
|
170
|
+
return 0;
|
|
171
|
+
const { sql: whereClause, params } = buildWhereClause(options.where);
|
|
172
|
+
const row = await db.first(
|
|
173
|
+
`SELECT COUNT(*) as count FROM ai_ready_pages ${whereClause}`,
|
|
174
|
+
params
|
|
175
|
+
);
|
|
176
|
+
return row?.count || 0;
|
|
25
177
|
}
|
|
26
|
-
export async function searchPages(
|
|
27
|
-
|
|
178
|
+
export async function searchPages(event, query, options = {}) {
|
|
179
|
+
if (import.meta.dev || import.meta.prerender)
|
|
180
|
+
return [];
|
|
181
|
+
const db = await getDb(event);
|
|
182
|
+
if (!db)
|
|
183
|
+
return [];
|
|
184
|
+
const { limit = 10 } = options;
|
|
28
185
|
const sanitized = query.replace(/[*:^"()]/g, " ").trim();
|
|
29
|
-
if (!sanitized)
|
|
186
|
+
if (!sanitized)
|
|
187
|
+
return [];
|
|
30
188
|
const terms = sanitized.split(/\s+/).map((t) => `${t}*`).join(" ");
|
|
31
189
|
return db.all(`
|
|
32
190
|
SELECT p.route, p.title, p.description, bm25(ai_ready_pages_fts, 5.0, 3.0, 1.0, 0.5, 2.0, 2.0) as score
|
|
@@ -37,43 +195,182 @@ export async function searchPages(db, query, opts = {}) {
|
|
|
37
195
|
LIMIT ?
|
|
38
196
|
`, [terms, limit]);
|
|
39
197
|
}
|
|
40
|
-
export async function upsertPage(
|
|
198
|
+
export async function upsertPage(event, page) {
|
|
199
|
+
const db = await getDb(event);
|
|
200
|
+
if (!db)
|
|
201
|
+
return;
|
|
41
202
|
const routeKey = normalizeRouteKey(page.route);
|
|
42
203
|
const keywordsJson = JSON.stringify(page.keywords);
|
|
43
204
|
const indexedAt = Date.now();
|
|
205
|
+
const source = page.source || "runtime";
|
|
206
|
+
const lastSeenAt = source === "runtime" ? indexedAt : null;
|
|
44
207
|
await db.exec(`
|
|
45
|
-
INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error)
|
|
46
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
208
|
+
INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, content_hash, updated_at, indexed_at, is_error, indexed, source, last_seen_at)
|
|
209
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
|
47
210
|
ON CONFLICT(route) DO UPDATE SET
|
|
48
211
|
title = excluded.title,
|
|
49
212
|
description = excluded.description,
|
|
50
213
|
markdown = excluded.markdown,
|
|
51
214
|
headings = excluded.headings,
|
|
52
215
|
keywords = excluded.keywords,
|
|
216
|
+
content_hash = excluded.content_hash,
|
|
53
217
|
updated_at = excluded.updated_at,
|
|
54
218
|
indexed_at = excluded.indexed_at,
|
|
55
|
-
is_error = excluded.is_error
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
return rows.map((r) => r.route);
|
|
219
|
+
is_error = excluded.is_error,
|
|
220
|
+
indexed = 1,
|
|
221
|
+
source = excluded.source,
|
|
222
|
+
last_seen_at = excluded.last_seen_at
|
|
223
|
+
`, [page.route, routeKey, page.title, page.description, page.markdown, page.headings, keywordsJson, page.contentHash || null, page.updatedAt, indexedAt, page.isError ? 1 : 0, source, lastSeenAt]);
|
|
61
224
|
}
|
|
62
|
-
export async function isPageFresh(
|
|
63
|
-
if (ttlSeconds <= 0)
|
|
225
|
+
export async function isPageFresh(event, route, ttlSeconds) {
|
|
226
|
+
if (ttlSeconds <= 0)
|
|
227
|
+
return false;
|
|
228
|
+
const db = await getDb(event);
|
|
229
|
+
if (!db)
|
|
230
|
+
return false;
|
|
64
231
|
const row = await db.first("SELECT indexed_at FROM ai_ready_pages WHERE route = ?", [route]);
|
|
65
|
-
if (!row)
|
|
232
|
+
if (!row)
|
|
233
|
+
return false;
|
|
66
234
|
const age = (Date.now() - row.indexed_at) / 1e3;
|
|
67
235
|
return age < ttlSeconds;
|
|
68
236
|
}
|
|
69
|
-
export async function
|
|
70
|
-
|
|
237
|
+
export async function getPageHash(event, route) {
|
|
238
|
+
const db = await getDb(event);
|
|
239
|
+
if (!db)
|
|
240
|
+
return null;
|
|
241
|
+
const row = await db.first("SELECT content_hash FROM ai_ready_pages WHERE route = ?", [route]);
|
|
242
|
+
return row?.content_hash || null;
|
|
71
243
|
}
|
|
72
|
-
export async function
|
|
73
|
-
const
|
|
244
|
+
export async function seedRoutes(event, routes) {
|
|
245
|
+
const db = await getDb(event);
|
|
246
|
+
if (!db)
|
|
247
|
+
return 0;
|
|
248
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
249
|
+
const nowMs = Date.now();
|
|
250
|
+
for (const route of routes) {
|
|
251
|
+
const routeKey = normalizeRouteKey(route);
|
|
252
|
+
await db.exec(`
|
|
253
|
+
INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at)
|
|
254
|
+
VALUES (?, ?, '', '', '', '[]', '[]', ?, 0, 0, 0, 'runtime', ?)
|
|
255
|
+
ON CONFLICT(route) DO UPDATE SET last_seen_at = excluded.last_seen_at
|
|
256
|
+
`, [route, routeKey, now, nowMs]);
|
|
257
|
+
}
|
|
258
|
+
return routes.length;
|
|
259
|
+
}
|
|
260
|
+
export async function getSitemapSeededAt(event) {
|
|
261
|
+
const db = await getDb(event);
|
|
262
|
+
if (!db)
|
|
263
|
+
return void 0;
|
|
264
|
+
const row = await db.first("SELECT value FROM _ai_ready_info WHERE id = ?", ["sitemap_seeded_at"]);
|
|
265
|
+
return row ? Number.parseInt(row.value, 10) : void 0;
|
|
266
|
+
}
|
|
267
|
+
export async function setSitemapSeededAt(event, timestamp) {
|
|
268
|
+
const db = await getDb(event);
|
|
269
|
+
if (!db)
|
|
270
|
+
return;
|
|
271
|
+
await db.exec("INSERT OR REPLACE INTO _ai_ready_info (id, value) VALUES (?, ?)", ["sitemap_seeded_at", String(timestamp)]);
|
|
272
|
+
}
|
|
273
|
+
export async function pruneStaleRoutes(event, staleThresholdSeconds) {
|
|
274
|
+
const db = await getDb(event);
|
|
275
|
+
if (!db)
|
|
276
|
+
return 0;
|
|
277
|
+
const threshold = Date.now() - staleThresholdSeconds * 1e3;
|
|
278
|
+
const countRow = await db.first(
|
|
279
|
+
"SELECT COUNT(*) as count FROM ai_ready_pages WHERE source = ? AND last_seen_at < ?",
|
|
280
|
+
["runtime", threshold]
|
|
281
|
+
);
|
|
282
|
+
const count = countRow?.count || 0;
|
|
283
|
+
if (count > 0) {
|
|
284
|
+
await db.exec("DELETE FROM ai_ready_pages WHERE source = ? AND last_seen_at < ?", ["runtime", threshold]);
|
|
285
|
+
}
|
|
286
|
+
return count;
|
|
287
|
+
}
|
|
288
|
+
export async function getStaleRoutes(event, staleThresholdSeconds) {
|
|
289
|
+
const db = await getDb(event);
|
|
290
|
+
if (!db)
|
|
291
|
+
return [];
|
|
292
|
+
const threshold = Date.now() - staleThresholdSeconds * 1e3;
|
|
293
|
+
const rows = await db.all(
|
|
294
|
+
"SELECT route FROM ai_ready_pages WHERE source = ? AND last_seen_at < ?",
|
|
295
|
+
["runtime", threshold]
|
|
296
|
+
);
|
|
297
|
+
return rows.map((r) => r.route);
|
|
298
|
+
}
|
|
299
|
+
export async function getPagesNeedingIndexNowSync(event, limit = 100) {
|
|
300
|
+
const db = await getDb(event);
|
|
301
|
+
if (!db)
|
|
302
|
+
return [];
|
|
303
|
+
return db.all(`
|
|
304
|
+
SELECT route FROM ai_ready_pages
|
|
305
|
+
WHERE indexed = 1
|
|
306
|
+
AND is_error = 0
|
|
307
|
+
AND (indexnow_synced_at IS NULL OR indexnow_synced_at < indexed_at)
|
|
308
|
+
LIMIT ?
|
|
309
|
+
`, [limit]);
|
|
310
|
+
}
|
|
311
|
+
export async function countPagesNeedingIndexNowSync(event) {
|
|
312
|
+
const db = await getDb(event);
|
|
313
|
+
if (!db)
|
|
314
|
+
return 0;
|
|
315
|
+
const row = await db.first(`
|
|
316
|
+
SELECT COUNT(*) as count FROM ai_ready_pages
|
|
317
|
+
WHERE indexed = 1
|
|
318
|
+
AND is_error = 0
|
|
319
|
+
AND (indexnow_synced_at IS NULL OR indexnow_synced_at < indexed_at)
|
|
320
|
+
`);
|
|
74
321
|
return row?.count || 0;
|
|
75
322
|
}
|
|
76
|
-
export async function
|
|
77
|
-
const
|
|
78
|
-
|
|
323
|
+
export async function markIndexNowSynced(event, routes) {
|
|
324
|
+
const db = await getDb(event);
|
|
325
|
+
if (!db || routes.length === 0)
|
|
326
|
+
return;
|
|
327
|
+
const now = Date.now();
|
|
328
|
+
const placeholders = routes.map(() => "?").join(",");
|
|
329
|
+
await db.exec(
|
|
330
|
+
`UPDATE ai_ready_pages SET indexnow_synced_at = ? WHERE route IN (${placeholders})`,
|
|
331
|
+
[now, ...routes]
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
export async function updateIndexNowStats(event, submitted, error) {
|
|
335
|
+
const db = await getDb(event);
|
|
336
|
+
if (!db)
|
|
337
|
+
return;
|
|
338
|
+
const now = Date.now();
|
|
339
|
+
if (error) {
|
|
340
|
+
await db.exec(
|
|
341
|
+
"INSERT OR REPLACE INTO _ai_ready_info (id, value) VALUES (?, ?)",
|
|
342
|
+
["indexnow_last_error", error]
|
|
343
|
+
);
|
|
344
|
+
} else {
|
|
345
|
+
await db.exec(`
|
|
346
|
+
INSERT INTO _ai_ready_info (id, value) VALUES (?, ?)
|
|
347
|
+
ON CONFLICT(id) DO UPDATE SET value = CAST((CAST(value AS INTEGER) + ?) AS TEXT)
|
|
348
|
+
`, ["indexnow_total_submitted", String(submitted), submitted]);
|
|
349
|
+
await db.exec(
|
|
350
|
+
"INSERT OR REPLACE INTO _ai_ready_info (id, value) VALUES (?, ?)",
|
|
351
|
+
["indexnow_last_submitted_at", String(now)]
|
|
352
|
+
);
|
|
353
|
+
await db.exec(
|
|
354
|
+
"DELETE FROM _ai_ready_info WHERE id = ?",
|
|
355
|
+
["indexnow_last_error"]
|
|
356
|
+
);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
export async function getIndexNowStats(event) {
|
|
360
|
+
const db = await getDb(event);
|
|
361
|
+
if (!db)
|
|
362
|
+
return { totalSubmitted: 0, lastSubmittedAt: null, lastError: null };
|
|
363
|
+
const rows = await db.all(
|
|
364
|
+
"SELECT id, value FROM _ai_ready_info WHERE id LIKE ?",
|
|
365
|
+
["indexnow_%"]
|
|
366
|
+
);
|
|
367
|
+
const stats = {};
|
|
368
|
+
for (const row of rows) {
|
|
369
|
+
stats[row.id] = row.value;
|
|
370
|
+
}
|
|
371
|
+
return {
|
|
372
|
+
totalSubmitted: Number.parseInt(stats.indexnow_total_submitted || "0", 10) || 0,
|
|
373
|
+
lastSubmittedAt: stats.indexnow_last_submitted_at ? Number.parseInt(stats.indexnow_last_submitted_at, 10) : null,
|
|
374
|
+
lastError: stats.indexnow_last_error || null
|
|
375
|
+
};
|
|
79
376
|
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
export const SCHEMA_VERSION = "v1.6.0";
|
|
2
|
+
const PAGES_TABLE_SQL = `
|
|
3
|
+
CREATE TABLE IF NOT EXISTS ai_ready_pages (
|
|
4
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
5
|
+
route TEXT UNIQUE NOT NULL,
|
|
6
|
+
route_key TEXT UNIQUE NOT NULL,
|
|
7
|
+
title TEXT NOT NULL DEFAULT '',
|
|
8
|
+
description TEXT NOT NULL DEFAULT '',
|
|
9
|
+
markdown TEXT NOT NULL DEFAULT '',
|
|
10
|
+
headings TEXT NOT NULL DEFAULT '[]',
|
|
11
|
+
keywords TEXT NOT NULL DEFAULT '[]',
|
|
12
|
+
content_hash TEXT,
|
|
13
|
+
updated_at TEXT NOT NULL,
|
|
14
|
+
indexed_at INTEGER NOT NULL,
|
|
15
|
+
is_error INTEGER NOT NULL DEFAULT 0,
|
|
16
|
+
indexed INTEGER NOT NULL DEFAULT 0,
|
|
17
|
+
source TEXT NOT NULL DEFAULT 'prerender',
|
|
18
|
+
last_seen_at INTEGER,
|
|
19
|
+
indexnow_synced_at INTEGER
|
|
20
|
+
)`;
|
|
21
|
+
const PAGES_INDEXES_SQL = [
|
|
22
|
+
"CREATE INDEX IF NOT EXISTS idx_ai_ready_pages_route ON ai_ready_pages(route)",
|
|
23
|
+
"CREATE INDEX IF NOT EXISTS idx_ai_ready_pages_is_error ON ai_ready_pages(is_error)",
|
|
24
|
+
"CREATE INDEX IF NOT EXISTS idx_ai_ready_pages_indexed ON ai_ready_pages(indexed)",
|
|
25
|
+
"CREATE INDEX IF NOT EXISTS idx_ai_ready_pages_source ON ai_ready_pages(source)",
|
|
26
|
+
"CREATE INDEX IF NOT EXISTS idx_ai_ready_pages_last_seen ON ai_ready_pages(last_seen_at)"
|
|
27
|
+
];
|
|
28
|
+
const FTS_TABLE_SQL = `
|
|
29
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS ai_ready_pages_fts USING fts5(
|
|
30
|
+
route, title, description, markdown, headings, keywords,
|
|
31
|
+
content=ai_ready_pages, content_rowid=id
|
|
32
|
+
)`;
|
|
33
|
+
const FTS_TRIGGERS_SQL = [
|
|
34
|
+
`CREATE TRIGGER IF NOT EXISTS ai_ready_pages_ai AFTER INSERT ON ai_ready_pages BEGIN
|
|
35
|
+
INSERT INTO ai_ready_pages_fts(rowid, route, title, description, markdown, headings, keywords)
|
|
36
|
+
VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);
|
|
37
|
+
END`,
|
|
38
|
+
`CREATE TRIGGER IF NOT EXISTS ai_ready_pages_ad AFTER DELETE ON ai_ready_pages BEGIN
|
|
39
|
+
INSERT INTO ai_ready_pages_fts(ai_ready_pages_fts, rowid, route, title, description, markdown, headings, keywords)
|
|
40
|
+
VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);
|
|
41
|
+
END`,
|
|
42
|
+
`CREATE TRIGGER IF NOT EXISTS ai_ready_pages_au AFTER UPDATE ON ai_ready_pages BEGIN
|
|
43
|
+
INSERT INTO ai_ready_pages_fts(ai_ready_pages_fts, rowid, route, title, description, markdown, headings, keywords)
|
|
44
|
+
VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);
|
|
45
|
+
INSERT INTO ai_ready_pages_fts(rowid, route, title, description, markdown, headings, keywords)
|
|
46
|
+
VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);
|
|
47
|
+
END`
|
|
48
|
+
];
|
|
49
|
+
const INFO_TABLE_SQL = `
|
|
50
|
+
CREATE TABLE IF NOT EXISTS _ai_ready_info (
|
|
51
|
+
id TEXT PRIMARY KEY,
|
|
52
|
+
value TEXT,
|
|
53
|
+
version TEXT,
|
|
54
|
+
checksum TEXT,
|
|
55
|
+
ready INTEGER DEFAULT 0
|
|
56
|
+
)`;
|
|
57
|
+
export const DROP_TABLES_SQL = [
|
|
58
|
+
"DROP TABLE IF EXISTS ai_ready_pages_fts",
|
|
59
|
+
"DROP TABLE IF EXISTS ai_ready_pages",
|
|
60
|
+
"DROP TABLE IF EXISTS _ai_ready_info",
|
|
61
|
+
// Legacy unprefixed tables (migration from v1.0.0)
|
|
62
|
+
"DROP TABLE IF EXISTS pages_fts",
|
|
63
|
+
"DROP TABLE IF EXISTS pages"
|
|
64
|
+
];
|
|
65
|
+
export const ALL_SCHEMA_SQL = [
|
|
66
|
+
PAGES_TABLE_SQL,
|
|
67
|
+
...PAGES_INDEXES_SQL,
|
|
68
|
+
FTS_TABLE_SQL,
|
|
69
|
+
...FTS_TRIGGERS_SQL,
|
|
70
|
+
INFO_TABLE_SQL
|
|
71
|
+
];
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import type { Connector } from 'db0';
|
|
2
|
+
/**
|
|
3
|
+
* Compute content hash for change detection (first 16 chars of SHA-256)
|
|
4
|
+
*/
|
|
5
|
+
export declare function computeContentHash(markdown: string): Promise<string>;
|
|
6
|
+
export interface DatabaseAdapter {
|
|
7
|
+
all: <T>(sql: string, params?: unknown[]) => Promise<T[]>;
|
|
8
|
+
first: <T>(sql: string, params?: unknown[]) => Promise<T | undefined>;
|
|
9
|
+
exec: (sql: string, params?: unknown[]) => Promise<void>;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Create a DatabaseAdapter from a db0 Connector
|
|
13
|
+
*/
|
|
14
|
+
export declare function createAdapter(connector: Connector): DatabaseAdapter;
|
|
15
|
+
/**
|
|
16
|
+
* Initialize database schema with version checking
|
|
17
|
+
*/
|
|
18
|
+
export declare function initSchema(db: DatabaseAdapter): Promise<void>;
|
|
19
|
+
/**
|
|
20
|
+
* Normalize route to storage key format
|
|
21
|
+
* e.g., '/about/team' -> 'about:team', '/' -> 'index'
|
|
22
|
+
*/
|
|
23
|
+
export declare function normalizeRouteKey(route: string): string;
|
|
24
|
+
/**
|
|
25
|
+
* Compress data to base64 gzip
|
|
26
|
+
*/
|
|
27
|
+
export declare function compressToBase64(data: unknown): Promise<string>;
|
|
28
|
+
/**
|
|
29
|
+
* Decompress from base64 gzip
|
|
30
|
+
*/
|
|
31
|
+
export declare function decompressFromBase64<T>(base64: string): Promise<T>;
|
|
32
|
+
export interface PageInput {
|
|
33
|
+
route: string;
|
|
34
|
+
title: string;
|
|
35
|
+
description: string;
|
|
36
|
+
markdown: string;
|
|
37
|
+
headings: string;
|
|
38
|
+
keywords: string[];
|
|
39
|
+
contentHash?: string;
|
|
40
|
+
updatedAt: string;
|
|
41
|
+
isError?: boolean;
|
|
42
|
+
source?: 'prerender' | 'runtime';
|
|
43
|
+
}
|
|
44
|
+
export interface PageOutput {
|
|
45
|
+
route: string;
|
|
46
|
+
title: string;
|
|
47
|
+
description: string;
|
|
48
|
+
markdown: string;
|
|
49
|
+
headings: string;
|
|
50
|
+
keywords: string[];
|
|
51
|
+
contentHash?: string;
|
|
52
|
+
updatedAt: string;
|
|
53
|
+
isError: boolean;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Insert or update a page
|
|
57
|
+
*/
|
|
58
|
+
export declare function insertPage(db: DatabaseAdapter, page: PageInput): Promise<void>;
|
|
59
|
+
/**
|
|
60
|
+
* Query all pages from database
|
|
61
|
+
*/
|
|
62
|
+
export declare function queryAllPages(db: DatabaseAdapter, options?: {
|
|
63
|
+
includeErrors?: boolean;
|
|
64
|
+
}): Promise<PageOutput[]>;
|
|
65
|
+
export interface DumpRow {
|
|
66
|
+
route: string;
|
|
67
|
+
route_key: string;
|
|
68
|
+
title: string;
|
|
69
|
+
description: string;
|
|
70
|
+
markdown: string;
|
|
71
|
+
headings: string;
|
|
72
|
+
keywords: string;
|
|
73
|
+
content_hash: string | null;
|
|
74
|
+
updated_at: string;
|
|
75
|
+
indexed_at: number;
|
|
76
|
+
is_error: number;
|
|
77
|
+
indexed: number;
|
|
78
|
+
source: string;
|
|
79
|
+
last_seen_at: number | null;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Export database as compressed dump (base64 gzip)
|
|
83
|
+
*/
|
|
84
|
+
export declare function exportDbDump(db: DatabaseAdapter): Promise<string>;
|
|
85
|
+
/**
|
|
86
|
+
* Import dump into database
|
|
87
|
+
*/
|
|
88
|
+
export declare function importDbDump(db: DatabaseAdapter, rows: DumpRow[]): Promise<void>;
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { subtle } from "uncrypto";
|
|
2
|
+
import { ALL_SCHEMA_SQL, DROP_TABLES_SQL, SCHEMA_VERSION } from "./schema-sql.js";
|
|
3
|
+
export async function computeContentHash(markdown) {
|
|
4
|
+
const encoder = new TextEncoder();
|
|
5
|
+
const data = encoder.encode(markdown);
|
|
6
|
+
const hashBuffer = await subtle.digest("SHA-256", data);
|
|
7
|
+
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
|
8
|
+
return hashArray.map((b) => b.toString(16).padStart(2, "0")).join("").slice(0, 16);
|
|
9
|
+
}
|
|
10
|
+
export function createAdapter(connector) {
|
|
11
|
+
return {
|
|
12
|
+
all: async (sql, params = []) => {
|
|
13
|
+
const result = await connector.prepare(sql).all(...params);
|
|
14
|
+
return result || [];
|
|
15
|
+
},
|
|
16
|
+
first: async (sql, params = []) => {
|
|
17
|
+
return connector.prepare(sql).get(...params);
|
|
18
|
+
},
|
|
19
|
+
exec: async (sql, params = []) => {
|
|
20
|
+
await connector.prepare(sql).run(...params);
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
export async function initSchema(db) {
|
|
25
|
+
const needsRebuild = await checkSchemaVersion(db);
|
|
26
|
+
if (needsRebuild) {
|
|
27
|
+
for (const sql of DROP_TABLES_SQL) {
|
|
28
|
+
await db.exec(sql);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
for (const sql of ALL_SCHEMA_SQL) {
|
|
32
|
+
await db.exec(sql);
|
|
33
|
+
}
|
|
34
|
+
await db.exec(
|
|
35
|
+
"INSERT OR REPLACE INTO _ai_ready_info (id, version) VALUES (?, ?)",
|
|
36
|
+
["schema", SCHEMA_VERSION]
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
async function checkSchemaVersion(db) {
|
|
40
|
+
const info = await db.first(
|
|
41
|
+
"SELECT version FROM _ai_ready_info WHERE id = ?",
|
|
42
|
+
["schema"]
|
|
43
|
+
).catch(() => null);
|
|
44
|
+
return !info || info.version !== SCHEMA_VERSION;
|
|
45
|
+
}
|
|
46
|
+
export function normalizeRouteKey(route) {
|
|
47
|
+
return route.replace(/^\//, "").replace(/\//g, ":") || "index";
|
|
48
|
+
}
|
|
49
|
+
export async function compressToBase64(data) {
|
|
50
|
+
const json = JSON.stringify(data);
|
|
51
|
+
const encoder = new TextEncoder();
|
|
52
|
+
const stream = new Blob([encoder.encode(json)]).stream();
|
|
53
|
+
const compressed = stream.pipeThrough(new CompressionStream("gzip"));
|
|
54
|
+
const buffer = await new Response(compressed).arrayBuffer();
|
|
55
|
+
return Buffer.from(buffer).toString("base64");
|
|
56
|
+
}
|
|
57
|
+
export async function decompressFromBase64(base64) {
|
|
58
|
+
const buffer = Buffer.from(base64, "base64");
|
|
59
|
+
const stream = new Blob([buffer]).stream();
|
|
60
|
+
const decompressed = stream.pipeThrough(new DecompressionStream("gzip"));
|
|
61
|
+
const text = await new Response(decompressed).text();
|
|
62
|
+
return JSON.parse(text);
|
|
63
|
+
}
|
|
64
|
+
export async function insertPage(db, page) {
|
|
65
|
+
const now = Date.now();
|
|
66
|
+
const source = page.source || "prerender";
|
|
67
|
+
await db.exec(`
|
|
68
|
+
INSERT OR REPLACE INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, content_hash, updated_at, indexed_at, is_error, indexed, source, last_seen_at)
|
|
69
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
70
|
+
`, [
|
|
71
|
+
page.route,
|
|
72
|
+
normalizeRouteKey(page.route),
|
|
73
|
+
page.title,
|
|
74
|
+
page.description,
|
|
75
|
+
page.markdown,
|
|
76
|
+
page.headings,
|
|
77
|
+
JSON.stringify(page.keywords),
|
|
78
|
+
page.contentHash || null,
|
|
79
|
+
page.updatedAt,
|
|
80
|
+
now,
|
|
81
|
+
page.isError ? 1 : 0,
|
|
82
|
+
page.isError ? 0 : 1,
|
|
83
|
+
source,
|
|
84
|
+
now
|
|
85
|
+
]);
|
|
86
|
+
}
|
|
87
|
+
export async function queryAllPages(db, options) {
|
|
88
|
+
const where = options?.includeErrors ? "" : "WHERE is_error = 0";
|
|
89
|
+
const rows = await db.all(`SELECT route, title, description, markdown, headings, keywords, content_hash, updated_at, is_error FROM ai_ready_pages ${where}`);
|
|
90
|
+
return rows.map((row) => ({
|
|
91
|
+
route: row.route,
|
|
92
|
+
title: row.title,
|
|
93
|
+
description: row.description,
|
|
94
|
+
markdown: row.markdown,
|
|
95
|
+
headings: row.headings,
|
|
96
|
+
keywords: JSON.parse(row.keywords || "[]"),
|
|
97
|
+
contentHash: row.content_hash || void 0,
|
|
98
|
+
updatedAt: row.updated_at,
|
|
99
|
+
isError: row.is_error === 1
|
|
100
|
+
}));
|
|
101
|
+
}
|
|
102
|
+
export async function exportDbDump(db) {
|
|
103
|
+
const rows = await db.all(`
|
|
104
|
+
SELECT route, route_key, title, description, markdown, headings, keywords, content_hash, updated_at, indexed_at, is_error, indexed, source, last_seen_at
|
|
105
|
+
FROM ai_ready_pages
|
|
106
|
+
`);
|
|
107
|
+
return compressToBase64(rows);
|
|
108
|
+
}
|
|
109
|
+
export async function importDbDump(db, rows) {
|
|
110
|
+
for (const row of rows) {
|
|
111
|
+
const source = row.source || "prerender";
|
|
112
|
+
await db.exec(`
|
|
113
|
+
INSERT OR REPLACE INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, content_hash, updated_at, indexed_at, is_error, indexed, source, last_seen_at)
|
|
114
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, NULL)
|
|
115
|
+
`, [row.route, row.route_key, row.title, row.description, row.markdown, row.headings, row.keywords, row.content_hash || null, row.updated_at, row.indexed_at, row.is_error, source]);
|
|
116
|
+
}
|
|
117
|
+
}
|