nuxt-ai-ready 2.1.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunks/agent-skills.mjs +4 -1
- package/dist/chunks/prerender.mjs +10 -3
- package/dist/cli.mjs +67 -11
- package/dist/module.d.mts +2 -0
- package/dist/module.json +1 -1
- package/dist/module.mjs +4 -1
- package/dist/runtime/app/plugins/md-alternate.server.js +6 -2
- package/dist/runtime/llms-txt-utils.js +2 -2
- package/dist/runtime/markdown-path.d.ts +2 -0
- package/dist/runtime/markdown-path.js +4 -0
- package/dist/runtime/server/db/drizzle/index.d.ts +1 -1
- package/dist/runtime/server/db/drizzle/index.js +0 -1
- package/dist/runtime/server/db/drizzle/queries.d.ts +0 -4
- package/dist/runtime/server/db/drizzle/queries.js +22 -16
- package/dist/runtime/server/db/drizzle/raw.js +1 -4
- package/dist/runtime/server/db/queries.d.ts +22 -21
- package/dist/runtime/server/db/queries.js +89 -66
- package/dist/runtime/server/db/shared.d.ts +16 -0
- package/dist/runtime/server/db/shared.js +12 -0
- package/dist/runtime/server/middleware/markdown.js +10 -5
- package/dist/runtime/server/middleware/markdown.prerender.js +4 -1
- package/dist/runtime/server/plugins/sitemap-seeder.js +5 -2
- package/dist/runtime/server/routes/__ai-ready/cron.get.js +10 -2
- package/dist/runtime/server/routes/__ai-ready/poll.post.js +30 -14
- package/dist/runtime/server/routes/__ai-ready/reindex.post.d.ts +2 -0
- package/dist/runtime/server/routes/__ai-ready/reindex.post.js +23 -0
- package/dist/runtime/server/routes/api-catalog.js +11 -2
- package/dist/runtime/server/routes/llms.txt.get.js +17 -12
- package/dist/runtime/server/routes/sitemap.md.get.d.ts +2 -0
- package/dist/runtime/server/routes/sitemap.md.get.js +51 -0
- package/dist/runtime/server/tasks/ai-ready-cron.js +3 -0
- package/dist/runtime/server/utils/i18n.d.ts +4 -0
- package/dist/runtime/server/utils/i18n.js +13 -0
- package/dist/runtime/server/utils/indexPage.js +7 -1
- package/dist/runtime/server/utils/link-header.d.ts +2 -0
- package/dist/runtime/server/utils/link-header.js +6 -0
- package/dist/runtime/server/utils/markdown-request.js +3 -1
- package/dist/runtime/server/utils/runCron.d.ts +24 -0
- package/dist/runtime/server/utils/runCron.js +21 -4
- package/dist/runtime/server/utils/sitemap-md.d.ts +13 -0
- package/dist/runtime/server/utils/sitemap-md.js +65 -0
- package/dist/runtime/types.d.ts +16 -3
- package/dist/shared/{nuxt-ai-ready.BTQAkSYt.mjs → nuxt-ai-ready.BVWXwmUS.mjs} +140 -36
- package/package.json +17 -17
|
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import { realpath, readFile } from 'node:fs/promises';
|
|
3
3
|
import { resolve, isAbsolute, relative, sep } from 'node:path';
|
|
4
4
|
import { parseDocument } from 'yaml';
|
|
5
|
-
import { A as AGENT_SKILLS_SCHEMA } from '../shared/nuxt-ai-ready.
|
|
5
|
+
import { A as AGENT_SKILLS_SCHEMA } from '../shared/nuxt-ai-ready.BVWXwmUS.mjs';
|
|
6
6
|
import 'node:module';
|
|
7
7
|
import '@nuxt/kit';
|
|
8
8
|
import 'defu';
|
|
@@ -10,9 +10,12 @@ import 'nuxt-site-config/kit';
|
|
|
10
10
|
import 'nuxtseo-shared/kit';
|
|
11
11
|
import 'pkg-types';
|
|
12
12
|
import 'node:url';
|
|
13
|
+
import '../../dist/runtime/markdown-path.js';
|
|
14
|
+
import '../../dist/runtime/server/utils/sitemap-md.js';
|
|
13
15
|
import '../../dist/runtime/server/utils/discovery-response.js';
|
|
14
16
|
import 'pathe';
|
|
15
17
|
import 'nuxtseo-shared/i18n';
|
|
18
|
+
import '../../dist/runtime/route-path.js';
|
|
16
19
|
|
|
17
20
|
const namePattern = /^(?!.*--)[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$/;
|
|
18
21
|
const digestPattern = /^sha256:[a-f0-9]{64}$/;
|
|
@@ -5,11 +5,12 @@ import { colorize } from 'consola/utils';
|
|
|
5
5
|
import { resolveLocaleFromRoute } from 'nuxtseo-shared/i18n-runtime';
|
|
6
6
|
import { collectSitemap } from 'sitemapd/parse';
|
|
7
7
|
import { withLeadingSlash, joinURL, withBase } from 'ufo';
|
|
8
|
-
import { l as logger, s as supportsNativeNodeSqlite, M as MARKDOWN_LINK_AVAILABILITY_FILE } from '../shared/nuxt-ai-ready.
|
|
8
|
+
import { l as logger, s as supportsNativeNodeSqlite, M as MARKDOWN_LINK_AVAILABILITY_FILE } from '../shared/nuxt-ai-ready.BVWXwmUS.mjs';
|
|
9
9
|
import { normalizePagePath, toMarkdownPath } from '../../dist/runtime/markdown-path.js';
|
|
10
|
-
import {
|
|
10
|
+
import { toDeployedRoute, toLogicalRoute } from '../../dist/runtime/route-path.js';
|
|
11
11
|
import { initSchema, computeContentHash, insertPage, queryAllPages, exportDbDump } from '../../dist/runtime/server/db/shared.js';
|
|
12
12
|
import { buildLlmsFullTxtHeader, formatPageForLlmsFullTxt } from '../../dist/runtime/server/utils/llms-full.js';
|
|
13
|
+
import { SITEMAP_MD_ROUTE, appendSitemapSection } from '../../dist/runtime/server/utils/sitemap-md.js';
|
|
13
14
|
import 'node:crypto';
|
|
14
15
|
import 'node:module';
|
|
15
16
|
import 'defu';
|
|
@@ -320,6 +321,8 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras
|
|
|
320
321
|
logger.debug(`Detected error page: ${pageRoute2}`);
|
|
321
322
|
return;
|
|
322
323
|
}
|
|
324
|
+
if (route.fileName === SITEMAP_MD_ROUTE)
|
|
325
|
+
return;
|
|
323
326
|
if (!route.fileName?.endsWith(".md"))
|
|
324
327
|
return;
|
|
325
328
|
let pageRoute = route.route.replace(RE_MD_EXT, "");
|
|
@@ -331,7 +334,7 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras
|
|
|
331
334
|
await initPromise;
|
|
332
335
|
const parsed = JSON.parse(route.contents || "{}");
|
|
333
336
|
await processMarkdownRoute(state, nuxt, pageRoute, parsed);
|
|
334
|
-
route.contents = parsed.markdown;
|
|
337
|
+
route.contents = options.sitemapMd === false ? parsed.markdown : appendSitemapSection(parsed.markdown, toDeployedRoute(SITEMAP_MD_ROUTE, nitro.options.baseURL));
|
|
335
338
|
route.contentType = "text/markdown; charset=utf-8";
|
|
336
339
|
state.totalProcessingTime += Date.now() - pageStartTime;
|
|
337
340
|
});
|
|
@@ -406,6 +409,10 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, extras
|
|
|
406
409
|
const llmsStats = await prerenderRoute(nitro, "/llms.txt");
|
|
407
410
|
const llmsFullStats = await stat(state.llmsFullTxtPath);
|
|
408
411
|
nitro._prerenderedRoutes.push({ route: "/llms-full.txt", fileName: "/llms-full.txt" });
|
|
412
|
+
if (options.sitemapMd !== false) {
|
|
413
|
+
const sitemapMdStats = await prerenderRoute(nitro, SITEMAP_MD_ROUTE);
|
|
414
|
+
logger.debug(`Wrote sitemap.md (${(sitemapMdStats.size / 1024).toFixed(1)}kb)`);
|
|
415
|
+
}
|
|
409
416
|
const kb = (b) => (b / 1024).toFixed(1);
|
|
410
417
|
const totalKb = kb(llmsStats.size + llmsFullStats.size);
|
|
411
418
|
const dim = (s) => colorize("dim", s);
|
package/dist/cli.mjs
CHANGED
|
@@ -26,11 +26,22 @@ async function requireSecret(cwd, hint = "") {
|
|
|
26
26
|
}
|
|
27
27
|
return secret;
|
|
28
28
|
}
|
|
29
|
+
function describeHttpError(status, body) {
|
|
30
|
+
const error = body;
|
|
31
|
+
const message = typeof error?.message === "string" && error.message ? error.message : typeof error?.statusMessage === "string" && error.statusMessage ? error.statusMessage : typeof error?.error === "string" && error.error ? error.error : null;
|
|
32
|
+
return message ? `${status}: ${message}` : `request failed with status ${status}`;
|
|
33
|
+
}
|
|
29
34
|
async function fetchJson(url, init, errorLabel = "Failed") {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
return null;
|
|
35
|
+
const r = await fetch(url, init).catch((err) => {
|
|
36
|
+
throw new Error(`${errorLabel}: ${err instanceof Error ? err.message : String(err)}`);
|
|
33
37
|
});
|
|
38
|
+
if (!r.ok) {
|
|
39
|
+
const body = await r.json().catch(() => {
|
|
40
|
+
return null;
|
|
41
|
+
});
|
|
42
|
+
throw new Error(`${errorLabel}: ${describeHttpError(r.status, body)}`);
|
|
43
|
+
}
|
|
44
|
+
return r.json();
|
|
34
45
|
}
|
|
35
46
|
const main = defineCommand({
|
|
36
47
|
meta: {
|
|
@@ -65,8 +76,6 @@ const main = defineCommand({
|
|
|
65
76
|
const url = `${args.url}/__ai-ready/status`;
|
|
66
77
|
consola.info(`Fetching status from ${args.url}...`);
|
|
67
78
|
const res = await fetchJson(url, { headers: authHeaders(secret) }, "Failed to connect");
|
|
68
|
-
if (!res)
|
|
69
|
-
return;
|
|
70
79
|
consola.box("AI Ready Status");
|
|
71
80
|
consola.info(`Total pages: ${colors.cyan(res.total?.toString() || "0")}`);
|
|
72
81
|
consola.info(`Indexed: ${colors.green(res.indexed?.toString() || "0")}`);
|
|
@@ -151,8 +160,6 @@ const main = defineCommand({
|
|
|
151
160
|
const url = `${args.url}/__ai-ready/poll?${params}`;
|
|
152
161
|
consola.info(`Triggering poll at ${args.url}...`);
|
|
153
162
|
const res = await fetchJson(url, { method: "POST", headers: authHeaders(secret) });
|
|
154
|
-
if (!res)
|
|
155
|
-
return;
|
|
156
163
|
consola.success(`Indexed: ${colors.green(res.indexed?.toString() || "0")} pages`);
|
|
157
164
|
consola.info(`Remaining: ${colors.yellow(res.remaining?.toString() || "0")}`);
|
|
158
165
|
if (res.errors?.length) {
|
|
@@ -199,8 +206,6 @@ const main = defineCommand({
|
|
|
199
206
|
const url = `${args.url}/__ai-ready/restore?${params}`;
|
|
200
207
|
consola.info(`Restoring database at ${args.url}...`);
|
|
201
208
|
const res = await fetchJson(url, { method: "POST", headers: authHeaders(secret) });
|
|
202
|
-
if (!res)
|
|
203
|
-
return;
|
|
204
209
|
consola.success(`Restored: ${colors.green(res.restored?.toString() || "0")} pages`);
|
|
205
210
|
if (res.cleared) {
|
|
206
211
|
consola.info(`Cleared: ${colors.yellow(res.cleared?.toString() || "0")} existing pages`);
|
|
@@ -252,8 +257,6 @@ const main = defineCommand({
|
|
|
252
257
|
method: "POST",
|
|
253
258
|
headers: secret ? authHeaders(secret) : void 0
|
|
254
259
|
});
|
|
255
|
-
if (!res)
|
|
256
|
-
return;
|
|
257
260
|
if (args.dry) {
|
|
258
261
|
consola.info(`Would prune: ${colors.yellow(res.count?.toString() || "0")} routes`);
|
|
259
262
|
if (res.routes?.length) {
|
|
@@ -268,6 +271,59 @@ const main = defineCommand({
|
|
|
268
271
|
consola.success(`Pruned: ${colors.green(res.pruned?.toString() || "0")} routes`);
|
|
269
272
|
}
|
|
270
273
|
}
|
|
274
|
+
}),
|
|
275
|
+
reindex: () => defineCommand({
|
|
276
|
+
meta: {
|
|
277
|
+
name: "reindex",
|
|
278
|
+
description: "Reindex a single route"
|
|
279
|
+
},
|
|
280
|
+
args: {
|
|
281
|
+
route: {
|
|
282
|
+
type: "positional",
|
|
283
|
+
description: "Route to reindex (e.g. /about)",
|
|
284
|
+
required: true
|
|
285
|
+
},
|
|
286
|
+
url: {
|
|
287
|
+
type: "string",
|
|
288
|
+
alias: "u",
|
|
289
|
+
description: "Site URL (default: http://localhost:3000)",
|
|
290
|
+
default: "http://localhost:3000"
|
|
291
|
+
},
|
|
292
|
+
force: {
|
|
293
|
+
type: "boolean",
|
|
294
|
+
description: "Index even when the page is still fresh",
|
|
295
|
+
default: true,
|
|
296
|
+
negativeDescription: "Skip indexing when the page is still fresh"
|
|
297
|
+
},
|
|
298
|
+
cwd: {
|
|
299
|
+
type: "string",
|
|
300
|
+
description: "Working directory",
|
|
301
|
+
default: "."
|
|
302
|
+
}
|
|
303
|
+
},
|
|
304
|
+
async run({ args }) {
|
|
305
|
+
const cwd = resolve(args.cwd || ".");
|
|
306
|
+
const secret = await requireSecret(cwd);
|
|
307
|
+
if (!secret)
|
|
308
|
+
return;
|
|
309
|
+
const params = new URLSearchParams();
|
|
310
|
+
params.set("route", args.route);
|
|
311
|
+
if (!args.force)
|
|
312
|
+
params.set("force", "false");
|
|
313
|
+
const url = `${args.url}/__ai-ready/reindex?${params}`;
|
|
314
|
+
consola.info(`Reindexing ${args.route} at ${args.url}...`);
|
|
315
|
+
const res = await fetchJson(url, { method: "POST", headers: authHeaders(secret) });
|
|
316
|
+
if (res.indexed) {
|
|
317
|
+
consola.success(`Indexed: ${colors.green(res.route)}`);
|
|
318
|
+
if (res.contentChanged !== void 0) {
|
|
319
|
+
consola.info(`Content changed: ${res.contentChanged ? colors.green("yes") : colors.yellow("no")}`);
|
|
320
|
+
}
|
|
321
|
+
} else if (res.skipped) {
|
|
322
|
+
consola.info(`Skipped: ${colors.yellow(res.route)} (still fresh)`);
|
|
323
|
+
} else {
|
|
324
|
+
throw new Error(`Failed to reindex ${res.route ?? args.route}${res.error ? `: ${res.error}` : ""}`);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
271
327
|
})
|
|
272
328
|
}
|
|
273
329
|
});
|
package/dist/module.d.mts
CHANGED
|
@@ -85,6 +85,8 @@ interface ModulePublicRuntimeConfig {
|
|
|
85
85
|
debugCron: boolean;
|
|
86
86
|
contentNegotiation: ContentNegotiationPolicy;
|
|
87
87
|
version: string;
|
|
88
|
+
sitemapMd: boolean;
|
|
89
|
+
describedby: boolean;
|
|
88
90
|
mdreamOptions: ModuleOptions['mdreamOptions'];
|
|
89
91
|
markdownCacheHeaders: Required<NonNullable<ModuleOptions['markdownCacheHeaders']>>;
|
|
90
92
|
database: ResolvedDatabase;
|
package/dist/module.json
CHANGED
package/dist/module.mjs
CHANGED
|
@@ -7,8 +7,11 @@ import 'defu';
|
|
|
7
7
|
import 'nuxt-site-config/kit';
|
|
8
8
|
import 'nuxtseo-shared/kit';
|
|
9
9
|
import 'pkg-types';
|
|
10
|
-
export { m as default } from './shared/nuxt-ai-ready.
|
|
10
|
+
export { m as default } from './shared/nuxt-ai-ready.BVWXwmUS.mjs';
|
|
11
|
+
import '../dist/runtime/markdown-path.js';
|
|
12
|
+
import '../dist/runtime/server/utils/sitemap-md.js';
|
|
11
13
|
import '../dist/runtime/server/utils/discovery-response.js';
|
|
12
14
|
import 'node:url';
|
|
13
15
|
import 'pathe';
|
|
14
16
|
import 'nuxtseo-shared/i18n';
|
|
17
|
+
import '../dist/runtime/route-path.js';
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { defineNuxtPlugin, useHead, useRequestURL } from "nuxt/app";
|
|
1
|
+
import { defineNuxtPlugin, useHead, useRequestURL, useRuntimeConfig } from "nuxt/app";
|
|
2
|
+
import { joinURL } from "ufo";
|
|
2
3
|
import { toMarkdownPath } from "../../markdown-path.js";
|
|
3
4
|
export default defineNuxtPlugin({
|
|
4
5
|
setup() {
|
|
@@ -7,9 +8,12 @@ export default defineNuxtPlugin({
|
|
|
7
8
|
const lastSegment = path.split("/").pop() || "";
|
|
8
9
|
if (lastSegment.includes("."))
|
|
9
10
|
return;
|
|
11
|
+
const runtimeConfig = useRuntimeConfig();
|
|
12
|
+
const describedby = runtimeConfig["nuxt-ai-ready"]?.describedby !== false;
|
|
10
13
|
useHead({
|
|
11
14
|
link: [
|
|
12
|
-
{ rel: "alternate", type: "text/markdown", href: toMarkdownPath(path) }
|
|
15
|
+
{ rel: "alternate", type: "text/markdown", href: toMarkdownPath(path) },
|
|
16
|
+
...describedby ? [{ rel: "describedby", href: joinURL(runtimeConfig.app.baseURL, "llms.txt") }] : []
|
|
13
17
|
]
|
|
14
18
|
});
|
|
15
19
|
}
|
|
@@ -4,7 +4,7 @@ import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
|
4
4
|
import { getSiteConfig } from "#site-config/server/composables/getSiteConfig";
|
|
5
5
|
import { withSiteTrailingSlash, withSiteUrl } from "#site-config/server/composables/utils";
|
|
6
6
|
import { formatLlmsTxtPageLink, normalizeLlmsTxtConfig } from "./llms-txt-format.js";
|
|
7
|
-
import { toMarkdownPath } from "./markdown-path.js";
|
|
7
|
+
import { isReservedPath, toMarkdownPath } from "./markdown-path.js";
|
|
8
8
|
import { normalizePersistedRoute, toDeployedRoute, toLogicalRoute } from "./route-path.js";
|
|
9
9
|
import { queryPages } from "./server/db/queries.js";
|
|
10
10
|
import { logger } from "./server/logger.js";
|
|
@@ -12,7 +12,7 @@ import { fetchSitemapUrls } from "./server/utils/sitemap.js";
|
|
|
12
12
|
export { normalizeLlmsTxtConfig };
|
|
13
13
|
function hasRuntimeMarkdownHandler(pathname) {
|
|
14
14
|
const lastSegment = pathname.split("/").pop() || "";
|
|
15
|
-
return !
|
|
15
|
+
return !isReservedPath(pathname) && !lastSegment.includes(".");
|
|
16
16
|
}
|
|
17
17
|
async function findMarkdownLinkAvailability() {
|
|
18
18
|
if (!import.meta.prerender) {
|
|
@@ -1,2 +1,4 @@
|
|
|
1
|
+
/** True for the `/api` namespace and Nitro's `/_*` and `/@*` internal paths. */
|
|
2
|
+
export declare function isReservedPath(path: string): boolean;
|
|
1
3
|
export declare function normalizePagePath(path: string): string;
|
|
2
4
|
export declare function toMarkdownPath(path: string): string;
|
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
const RE_TRAILING_SLASHES = /\/+$/;
|
|
2
|
+
const RE_RESERVED_PATH = /^\/(?:api(?:\/|$)|_|@)/;
|
|
3
|
+
export function isReservedPath(path) {
|
|
4
|
+
return RE_RESERVED_PATH.test(path);
|
|
5
|
+
}
|
|
2
6
|
export function normalizePagePath(path) {
|
|
3
7
|
return path.replace(RE_TRAILING_SLASHES, "") || "/";
|
|
4
8
|
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
export * from '#ai-ready-virtual/db-schema.mjs';
|
|
5
5
|
export { closeDrizzle, finishDrizzleResponse, trackDrizzleWork, useDrizzle } from './client.js';
|
|
6
6
|
export type { DatabaseDialect, DrizzleDatabase } from './client.js';
|
|
7
|
-
export { completeCronRun, countPages, deleteInfoValue,
|
|
7
|
+
export { completeCronRun, countPages, deleteInfoValue, getAllPages, getContentHashes, getInfoValue, getNextSitemapToCrawl, getPageByRoute, getPageLastmods, getPendingPages, getRecentCronRuns, getSitemapStatus, initSchema, markPageIndexed, markRoutesPending, markSitemapCrawled, markSitemapCrawlPartial, markSitemapError, markSitemapSeeded, resetSitemapErrors, searchPages, seedRoutes, setInfoValue, startCronRun, syncSitemaps, upsertPage, } from './queries.js';
|
|
8
8
|
export type { CronRunOutput, PageInput, PageMetaOutput, PageOutput, SitemapOutput, } from './queries.js';
|
|
9
9
|
export { useRawDb } from './raw.js';
|
|
10
10
|
export type { RawExecutor } from './raw.js';
|
|
@@ -65,10 +65,6 @@ export declare function countPages(event: H3Event | undefined, options?: {
|
|
|
65
65
|
indexed?: boolean;
|
|
66
66
|
errors?: boolean;
|
|
67
67
|
}): Promise<number>;
|
|
68
|
-
/**
|
|
69
|
-
* Delete a page by route
|
|
70
|
-
*/
|
|
71
|
-
export declare function deletePage(event: H3Event | undefined, route: string): Promise<void>;
|
|
72
68
|
/**
|
|
73
69
|
* Get pending pages (not yet indexed)
|
|
74
70
|
*/
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { and, count, desc, eq, gt, isNotNull, isNull,
|
|
1
|
+
import { and, count, desc, eq, gt, isNotNull, isNull, lt, or, sql } from "drizzle-orm";
|
|
2
2
|
import { cronRuns, info, pages, sitemaps } from "#ai-ready-virtual/db-schema.mjs";
|
|
3
3
|
import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
4
4
|
import { parseSitemapCrawlState, serializeSitemapCrawlState } from "../../utils/sitemap-crawl-state.js";
|
|
5
5
|
import { resolveFtsTokenizer as validateFtsTokenizer } from "../schema-sql.js";
|
|
6
|
+
import { LIKE_ESCAPE, likeSubstring, maxRowsPerInsert } from "../shared.js";
|
|
6
7
|
import { useDrizzle } from "./client.js";
|
|
7
8
|
import { getRawExecutor, useRawDb } from "./raw.js";
|
|
8
9
|
function resolveFtsTokenizer(event) {
|
|
@@ -127,7 +128,8 @@ export async function getPageLastmods(event) {
|
|
|
127
128
|
export async function searchPages(event, query, options) {
|
|
128
129
|
const client = await useDrizzle(event);
|
|
129
130
|
const limitNum = options?.limit || 20;
|
|
130
|
-
const searchTerm =
|
|
131
|
+
const searchTerm = likeSubstring(query);
|
|
132
|
+
const matches = (col) => sql`${col} LIKE ${searchTerm} ESCAPE ${LIKE_ESCAPE}`;
|
|
131
133
|
if (client.dialect === "sqlite") {
|
|
132
134
|
try {
|
|
133
135
|
const ftsResult = await client.db.all(sql`
|
|
@@ -155,10 +157,10 @@ export async function searchPages(event, query, options) {
|
|
|
155
157
|
and(
|
|
156
158
|
eq(pages.isError, 0),
|
|
157
159
|
or(
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
160
|
+
matches(pages.title),
|
|
161
|
+
matches(pages.description),
|
|
162
|
+
matches(pages.markdown),
|
|
163
|
+
matches(pages.headings)
|
|
162
164
|
)
|
|
163
165
|
)
|
|
164
166
|
).limit(limitNum);
|
|
@@ -172,6 +174,8 @@ export async function countPages(event, options) {
|
|
|
172
174
|
}
|
|
173
175
|
if (options?.errors !== void 0) {
|
|
174
176
|
conditions.push(eq(pages.isError, options.errors ? 1 : 0));
|
|
177
|
+
} else {
|
|
178
|
+
conditions.push(eq(pages.isError, 0));
|
|
175
179
|
}
|
|
176
180
|
let query = client.db.select({ count: count() }).from(pages);
|
|
177
181
|
if (conditions.length > 0) {
|
|
@@ -180,13 +184,12 @@ export async function countPages(event, options) {
|
|
|
180
184
|
const result = await query;
|
|
181
185
|
return Number(result[0]?.count || 0);
|
|
182
186
|
}
|
|
183
|
-
export async function deletePage(event, route) {
|
|
184
|
-
const client = await useDrizzle(event);
|
|
185
|
-
await client.db.delete(pages).where(eq(pages.route, route));
|
|
186
|
-
}
|
|
187
187
|
export async function getPendingPages(event, limit = 10) {
|
|
188
188
|
const client = await useDrizzle(event);
|
|
189
|
-
return client.db.select({ route: pages.route }).from(pages).where(
|
|
189
|
+
return client.db.select({ route: pages.route }).from(pages).where(and(
|
|
190
|
+
eq(pages.indexed, 0),
|
|
191
|
+
eq(pages.isError, 0)
|
|
192
|
+
)).orderBy(pages.route).limit(limit);
|
|
190
193
|
}
|
|
191
194
|
export async function markPageIndexed(event, route) {
|
|
192
195
|
const client = await useDrizzle(event);
|
|
@@ -202,7 +205,7 @@ export async function markRoutesPending(event, routes) {
|
|
|
202
205
|
const batch = routes.slice(i, i + D1_MAX_IN_ROUTES);
|
|
203
206
|
const placeholders = batch.map(() => "?").join(",");
|
|
204
207
|
stmts.push({
|
|
205
|
-
sql: `UPDATE ai_ready_pages SET indexed = 0 WHERE route IN (${placeholders})`,
|
|
208
|
+
sql: `UPDATE ai_ready_pages SET indexed = 0, is_error = 0 WHERE route IN (${placeholders})`,
|
|
206
209
|
params: batch
|
|
207
210
|
});
|
|
208
211
|
}
|
|
@@ -675,21 +678,24 @@ export async function seedRoutes(event, routes) {
|
|
|
675
678
|
const route = normalizeRoute(raw);
|
|
676
679
|
byRoute.set(route, normalizeRouteKey(route));
|
|
677
680
|
}
|
|
678
|
-
const
|
|
681
|
+
const rowsPerInsert = maxRowsPerInsert(4);
|
|
679
682
|
const db = await useRawDb(event);
|
|
680
683
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
681
684
|
const nowMs = Date.now();
|
|
682
685
|
const entries = [...byRoute.entries()];
|
|
683
686
|
const stmts = [];
|
|
684
|
-
for (let i = 0; i < entries.length; i +=
|
|
685
|
-
const batch = entries.slice(i, i +
|
|
687
|
+
for (let i = 0; i < entries.length; i += rowsPerInsert) {
|
|
688
|
+
const batch = entries.slice(i, i + rowsPerInsert);
|
|
686
689
|
const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', ?, 0, 0, 0, 'runtime', ?)`).join(", ");
|
|
687
690
|
const params = batch.flatMap(([route, routeKey]) => [route, routeKey, now, nowMs]);
|
|
688
691
|
stmts.push({
|
|
689
692
|
sql: `
|
|
690
693
|
INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at)
|
|
691
694
|
VALUES ${valuesSql}
|
|
692
|
-
ON CONFLICT(route) DO UPDATE SET
|
|
695
|
+
ON CONFLICT(route) DO UPDATE SET
|
|
696
|
+
last_seen_at = excluded.last_seen_at,
|
|
697
|
+
is_error = 0,
|
|
698
|
+
indexed = CASE WHEN ai_ready_pages.is_error = 1 THEN 0 ELSE ai_ready_pages.indexed END
|
|
693
699
|
`,
|
|
694
700
|
params
|
|
695
701
|
});
|
|
@@ -136,10 +136,7 @@ export function getRawExecutor(client) {
|
|
|
136
136
|
const sqlFn = driver;
|
|
137
137
|
for (let i = 0; i < queries.length; i += MAX_BATCH_STATEMENTS) {
|
|
138
138
|
const chunk = queries.slice(i, i + MAX_BATCH_STATEMENTS);
|
|
139
|
-
|
|
140
|
-
return sqlFn.query(toPostgresQuery(q.sql), q.params || []);
|
|
141
|
-
});
|
|
142
|
-
await sqlFn.transaction(pgQueries);
|
|
139
|
+
await sqlFn.transaction((tx) => chunk.map((q) => tx.query(toPostgresQuery(q.sql), q.params || [])));
|
|
143
140
|
}
|
|
144
141
|
break;
|
|
145
142
|
}
|
|
@@ -140,15 +140,8 @@ export declare function getPageHash(event: H3Event | undefined, route: string):
|
|
|
140
140
|
export declare function seedRoutes(event: H3Event | undefined, routes: Array<string | {
|
|
141
141
|
route: string;
|
|
142
142
|
locale?: string;
|
|
143
|
+
url?: string;
|
|
143
144
|
}>): Promise<number>;
|
|
144
|
-
/**
|
|
145
|
-
* Get sitemap seeded timestamp from _ai_ready_info
|
|
146
|
-
*/
|
|
147
|
-
export declare function getSitemapSeededAt(event: H3Event | undefined): Promise<number | undefined>;
|
|
148
|
-
/**
|
|
149
|
-
* Set sitemap seeded timestamp
|
|
150
|
-
*/
|
|
151
|
-
export declare function setSitemapSeededAt(event: H3Event | undefined, timestamp: number): Promise<void>;
|
|
152
145
|
/**
|
|
153
146
|
* Prune routes not seen in sitemap for longer than threshold
|
|
154
147
|
* Only prunes routes with source='runtime' (never prerendered pages)
|
|
@@ -194,10 +187,6 @@ export declare function completeCronRun(event: H3Event | undefined, runId: numbe
|
|
|
194
187
|
* Get recent cron runs
|
|
195
188
|
*/
|
|
196
189
|
export declare function getRecentCronRuns(event: H3Event | undefined, limit?: number): Promise<CronRun[]>;
|
|
197
|
-
/**
|
|
198
|
-
* Clean up old cron runs (keep last N)
|
|
199
|
-
*/
|
|
200
|
-
export declare function cleanupOldCronRuns(event: H3Event | undefined, keepCount?: number): Promise<number>;
|
|
201
190
|
/**
|
|
202
191
|
* Clean up cron runs older than specified age
|
|
203
192
|
*/
|
|
@@ -214,15 +203,27 @@ export interface CronFastPathStatus {
|
|
|
214
203
|
* Reduces 6+ sequential DB calls to 1
|
|
215
204
|
*/
|
|
216
205
|
export declare function getCronFastPathStatus(event: H3Event | undefined, sitemapIntervalMinutes?: number): Promise<CronFastPathStatus | null>;
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
206
|
+
export type CronLockAcquire = {
|
|
207
|
+
_tag: 'acquired';
|
|
208
|
+
token: string;
|
|
209
|
+
} | {
|
|
210
|
+
_tag: 'held';
|
|
211
|
+
};
|
|
212
|
+
/**
|
|
213
|
+
* Try to acquire cron lock. Returns the ownership token on success, `held`
|
|
214
|
+
* when a live lock belongs to another run.
|
|
215
|
+
*
|
|
216
|
+
* The value carries a random token plus expiry in one row, so the
|
|
217
|
+
* compare-expired-then-set upsert is atomic and ownership is unique even for
|
|
218
|
+
* two isolates that start in the same millisecond. A legacy value that
|
|
219
|
+
* predates tokens parses as no expiry, so it is treated as expired.
|
|
220
|
+
*/
|
|
221
|
+
export declare function tryAcquireCronLock(event: H3Event | undefined): Promise<CronLockAcquire>;
|
|
222
|
+
/**
|
|
223
|
+
* Release cron lock. Only deletes the row when its token is ours, so a run
|
|
224
|
+
* that outlived the lock TTL cannot delete the new owner's lock.
|
|
225
|
+
*/
|
|
226
|
+
export declare function releaseCronLock(event: H3Event | undefined, token: string): Promise<void>;
|
|
226
227
|
export interface CronLockStatus {
|
|
227
228
|
held: boolean;
|
|
228
229
|
since: number | null;
|