nuxt-ai-ready 2.3.2 → 2.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunks/agent-skills.mjs +1 -1
- package/dist/chunks/prerender.mjs +1 -1
- package/dist/module.json +1 -1
- package/dist/module.mjs +1 -1
- package/dist/runtime/app/plugins/md-alternate.server.js +4 -4
- package/dist/runtime/llms-txt-utils.js +2 -2
- package/dist/runtime/markdown-path.d.ts +8 -1
- package/dist/runtime/markdown-path.js +9 -1
- package/dist/runtime/server/db/drizzle/queries.d.ts +11 -1
- package/dist/runtime/server/db/drizzle/queries.js +24 -11
- package/dist/runtime/server/db/queries.d.ts +10 -4
- package/dist/runtime/server/db/queries.js +12 -38
- package/dist/runtime/server/db/seed.d.ts +32 -0
- package/dist/runtime/server/db/seed.js +40 -0
- package/dist/runtime/server/plugins/sitemap-seeder.js +3 -2
- package/dist/runtime/server/routes/__ai-ready/prune.post.js +4 -3
- package/dist/runtime/server/utils/i18n.d.ts +1 -5
- package/dist/runtime/server/utils/i18n.js +1 -13
- package/dist/runtime/server/utils/negotiation-response.js +2 -2
- package/dist/runtime/server/utils/runCron.js +5 -4
- package/dist/shared/{nuxt-ai-ready.wTtq73UR.mjs → nuxt-ai-ready.B5l-paa5.mjs} +3 -2
- package/package.json +2 -2
|
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import { realpath, readFile, readdir } from 'node:fs/promises';
|
|
3
3
|
import { resolve, isAbsolute, join, relative, sep } from 'node:path';
|
|
4
4
|
import { parseDocument } from 'yaml';
|
|
5
|
-
import { A as AGENT_SKILLS_SCHEMA } from '../shared/nuxt-ai-ready.
|
|
5
|
+
import { A as AGENT_SKILLS_SCHEMA } from '../shared/nuxt-ai-ready.B5l-paa5.mjs';
|
|
6
6
|
import 'node:module';
|
|
7
7
|
import '@nuxt/kit';
|
|
8
8
|
import 'defu';
|
|
@@ -5,7 +5,7 @@ import { colorize } from 'consola/utils';
|
|
|
5
5
|
import { resolveLocaleFromRoute } from 'nuxtseo-shared/i18n-runtime';
|
|
6
6
|
import { collectSitemap } from 'sitemapd/parse';
|
|
7
7
|
import { withLeadingSlash, joinURL, withBase } from 'ufo';
|
|
8
|
-
import { l as logger, s as supportsNativeNodeSqlite, M as MARKDOWN_LINK_AVAILABILITY_FILE } from '../shared/nuxt-ai-ready.
|
|
8
|
+
import { l as logger, s as supportsNativeNodeSqlite, M as MARKDOWN_LINK_AVAILABILITY_FILE } from '../shared/nuxt-ai-ready.B5l-paa5.mjs';
|
|
9
9
|
import { normalizePagePath, toMarkdownPath } from '../../dist/runtime/markdown-path.js';
|
|
10
10
|
import { toDeployedRoute, toLogicalRoute } from '../../dist/runtime/route-path.js';
|
|
11
11
|
import { initSchema, computeContentHash, insertPage, queryAllPages, exportDbDump } from '../../dist/runtime/server/db/shared.js';
|
package/dist/module.json
CHANGED
package/dist/module.mjs
CHANGED
|
@@ -7,7 +7,7 @@ import 'defu';
|
|
|
7
7
|
import 'nuxt-site-config/kit';
|
|
8
8
|
import 'nuxtseo-shared/kit';
|
|
9
9
|
import 'pkg-types';
|
|
10
|
-
export { m as default } from './shared/nuxt-ai-ready.
|
|
10
|
+
export { m as default } from './shared/nuxt-ai-ready.B5l-paa5.mjs';
|
|
11
11
|
import '../dist/runtime/markdown-path.js';
|
|
12
12
|
import '../dist/runtime/server/utils/sitemap-md.js';
|
|
13
13
|
import '../dist/runtime/server/utils/discovery-response.js';
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { defineNuxtPlugin, useHead, useRequestURL, useRuntimeConfig } from "nuxt/app";
|
|
2
2
|
import { joinURL } from "ufo";
|
|
3
|
-
import {
|
|
3
|
+
import { markdownAlternatePath } from "../../markdown-path.js";
|
|
4
4
|
export default defineNuxtPlugin({
|
|
5
5
|
setup() {
|
|
6
6
|
const url = useRequestURL();
|
|
7
7
|
const path = url.pathname;
|
|
8
|
-
const
|
|
9
|
-
if (
|
|
8
|
+
const markdownPath = markdownAlternatePath(path);
|
|
9
|
+
if (!markdownPath)
|
|
10
10
|
return;
|
|
11
11
|
const runtimeConfig = useRuntimeConfig();
|
|
12
12
|
const describedby = runtimeConfig["nuxt-ai-ready"]?.describedby !== false;
|
|
13
13
|
useHead({
|
|
14
14
|
link: [
|
|
15
|
-
{ rel: "alternate", type: "text/markdown", href:
|
|
15
|
+
{ rel: "alternate", type: "text/markdown", href: markdownPath },
|
|
16
16
|
...describedby ? [{ rel: "describedby", href: joinURL(runtimeConfig.app.baseURL, "llms.txt") }] : []
|
|
17
17
|
]
|
|
18
18
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { decodePath } from "ufo";
|
|
2
|
-
import {
|
|
2
|
+
import { getRequestHost } from "#nuxtseo/h3";
|
|
3
3
|
import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
4
4
|
import { getSiteConfig } from "#site-config/server/composables/getSiteConfig";
|
|
5
5
|
import { withSiteTrailingSlash, withSiteUrl } from "#site-config/server/composables/utils";
|
|
@@ -123,7 +123,7 @@ export async function buildLlmsTxt(event) {
|
|
|
123
123
|
const llmsTxtConfig = aiReadyConfig.llmsTxt;
|
|
124
124
|
const i18n = aiReadyConfig.i18n;
|
|
125
125
|
const i18nRuntime = i18n ? { _tag: "enabled", config: i18n, ...await import("./llms-txt-i18n.js") } : { _tag: "disabled" };
|
|
126
|
-
const i18nContext = { host:
|
|
126
|
+
const i18nContext = { host: getRequestHost(event, { xForwardedHost: true }) };
|
|
127
127
|
const baseURL = runtimeConfig.app.baseURL;
|
|
128
128
|
const resolvePath = (path) => withSiteTrailingSlash(event, toDeployedRoute(path, baseURL));
|
|
129
129
|
const resolveUrl = (path) => withSiteUrl(event, toDeployedRoute(path, baseURL));
|
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
/** True for the `/api` namespace
|
|
1
|
+
/** True for the `/api` namespace, Nitro's `/_*` paths, and Vite's `/@*` internals. */
|
|
2
2
|
export declare function isReservedPath(path: string): boolean;
|
|
3
3
|
export declare function normalizePagePath(path: string): string;
|
|
4
4
|
export declare function toMarkdownPath(path: string): string;
|
|
5
|
+
/**
|
|
6
|
+
* The `.md` sibling a page may advertise, or null when it has none.
|
|
7
|
+
*
|
|
8
|
+
* Advertising and serving read the same predicate here, so a page cannot link
|
|
9
|
+
* a representation the markdown handler declines.
|
|
10
|
+
*/
|
|
11
|
+
export declare function markdownAlternatePath(path: string): string | null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const RE_TRAILING_SLASHES = /\/+$/;
|
|
2
|
-
const RE_RESERVED_PATH = /^\/(?:api(?:\/|$)|_|@)/;
|
|
2
|
+
const RE_RESERVED_PATH = /^\/(?:api(?:\/|$)|_|@(?:id|fs|vite|react-refresh)(?:\/|$))/;
|
|
3
3
|
export function isReservedPath(path) {
|
|
4
4
|
return RE_RESERVED_PATH.test(path);
|
|
5
5
|
}
|
|
@@ -12,3 +12,11 @@ export function toMarkdownPath(path) {
|
|
|
12
12
|
return "/index.md";
|
|
13
13
|
return `${normalizedPath}.md`;
|
|
14
14
|
}
|
|
15
|
+
export function markdownAlternatePath(path) {
|
|
16
|
+
if (isReservedPath(path))
|
|
17
|
+
return null;
|
|
18
|
+
const lastSegment = normalizePagePath(path).split("/").pop() || "";
|
|
19
|
+
if (lastSegment.includes("."))
|
|
20
|
+
return null;
|
|
21
|
+
return toMarkdownPath(path);
|
|
22
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { H3Event } from '#nuxtseo/h3';
|
|
2
2
|
import type { SitemapCrawlState } from '../../utils/sitemap-crawl-state.js';
|
|
3
|
+
import type { SeedRoutesOptions } from '../seed.js';
|
|
3
4
|
export interface PageInput {
|
|
4
5
|
route: string;
|
|
5
6
|
title: string;
|
|
@@ -175,6 +176,15 @@ export declare function getSitemapStatus(event: H3Event | undefined): Promise<Si
|
|
|
175
176
|
export declare function resetSitemapErrors(event: H3Event | undefined): Promise<number>;
|
|
176
177
|
/**
|
|
177
178
|
* Seed routes from sitemap
|
|
179
|
+
*
|
|
180
|
+
* Mirrors `seedRoutes` in `../queries` (the raw layer), including the guarded
|
|
181
|
+
* DO UPDATE: an unchanged row is not rewritten on every crawl. Without the
|
|
182
|
+
* guard each run bumped `last_seen_at` on every row, which cost millions of
|
|
183
|
+
* D1 row writes a month. Keep the two layers in sync.
|
|
178
184
|
*/
|
|
179
|
-
export declare function seedRoutes(event: H3Event | undefined, routes: string
|
|
185
|
+
export declare function seedRoutes(event: H3Event | undefined, routes: Array<string | {
|
|
186
|
+
route: string;
|
|
187
|
+
locale?: string;
|
|
188
|
+
url?: string;
|
|
189
|
+
}>, options?: SeedRoutesOptions): Promise<number>;
|
|
180
190
|
export {};
|
|
@@ -3,6 +3,7 @@ import { cronRuns, info, pages, sitemaps } from "#ai-ready-virtual/db-schema.mjs
|
|
|
3
3
|
import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
4
4
|
import { parseSitemapCrawlState, serializeSitemapCrawlState } from "../../utils/sitemap-crawl-state.js";
|
|
5
5
|
import { resolveFtsTokenizer as validateFtsTokenizer } from "../schema-sql.js";
|
|
6
|
+
import { deriveLocale, SEED_REFRESH_WINDOW_MS } from "../seed.js";
|
|
6
7
|
import { LIKE_ESCAPE, likeSubstring, maxRowsPerInsert } from "../shared.js";
|
|
7
8
|
import { useDrizzle } from "./client.js";
|
|
8
9
|
import { getRawExecutor, useRawDb } from "./raw.js";
|
|
@@ -670,32 +671,44 @@ export async function resetSitemapErrors(event) {
|
|
|
670
671
|
}
|
|
671
672
|
return errorCount;
|
|
672
673
|
}
|
|
673
|
-
export async function seedRoutes(event, routes) {
|
|
674
|
+
export async function seedRoutes(event, routes, options = {}) {
|
|
674
675
|
if (routes.length === 0)
|
|
675
676
|
return 0;
|
|
677
|
+
const refreshWindowMs = Math.max(0, Math.trunc(Number.isFinite(options.refreshWindowMs) ? options.refreshWindowMs : SEED_REFRESH_WINDOW_MS));
|
|
678
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
679
|
+
const nowMs = Date.now();
|
|
676
680
|
const byRoute = /* @__PURE__ */ new Map();
|
|
677
|
-
for (const
|
|
678
|
-
const route = normalizeRoute(
|
|
679
|
-
|
|
681
|
+
for (const entry of routes) {
|
|
682
|
+
const route = normalizeRoute(typeof entry === "string" ? entry : entry.route);
|
|
683
|
+
const explicitLocale = typeof entry === "string" ? void 0 : entry.locale;
|
|
684
|
+
const entryUrl = typeof entry === "string" ? void 0 : entry.url;
|
|
685
|
+
byRoute.set(route, {
|
|
686
|
+
route,
|
|
687
|
+
routeKey: normalizeRouteKey(route),
|
|
688
|
+
locale: deriveLocale(event, route, explicitLocale, entryUrl)
|
|
689
|
+
});
|
|
680
690
|
}
|
|
681
|
-
const rowsPerInsert = maxRowsPerInsert(
|
|
691
|
+
const rowsPerInsert = maxRowsPerInsert(5);
|
|
682
692
|
const db = await useRawDb(event);
|
|
683
|
-
const
|
|
684
|
-
const nowMs = Date.now();
|
|
685
|
-
const entries = [...byRoute.entries()];
|
|
693
|
+
const entries = [...byRoute.values()];
|
|
686
694
|
const stmts = [];
|
|
687
695
|
for (let i = 0; i < entries.length; i += rowsPerInsert) {
|
|
688
696
|
const batch = entries.slice(i, i + rowsPerInsert);
|
|
689
|
-
const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', ?, 0, 0, 0, 'runtime', ?)`).join(", ");
|
|
690
|
-
const params = batch.flatMap((
|
|
697
|
+
const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', ?, 0, 0, 0, 'runtime', ?, ?)`).join(", ");
|
|
698
|
+
const params = batch.flatMap((r) => [r.route, r.routeKey, now, nowMs, r.locale]);
|
|
691
699
|
stmts.push({
|
|
692
700
|
sql: `
|
|
693
|
-
INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at)
|
|
701
|
+
INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at, locale)
|
|
694
702
|
VALUES ${valuesSql}
|
|
695
703
|
ON CONFLICT(route) DO UPDATE SET
|
|
696
704
|
last_seen_at = excluded.last_seen_at,
|
|
705
|
+
locale = excluded.locale,
|
|
697
706
|
is_error = 0,
|
|
698
707
|
indexed = CASE WHEN ai_ready_pages.is_error = 1 THEN 0 ELSE ai_ready_pages.indexed END
|
|
708
|
+
WHERE ai_ready_pages.last_seen_at IS NULL
|
|
709
|
+
OR ai_ready_pages.last_seen_at < excluded.last_seen_at - ${refreshWindowMs}
|
|
710
|
+
OR ai_ready_pages.is_error = 1
|
|
711
|
+
OR ai_ready_pages.locale IS DISTINCT FROM excluded.locale
|
|
699
712
|
`,
|
|
700
713
|
params
|
|
701
714
|
});
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type { H3Event } from '#nuxtseo/h3';
|
|
2
2
|
import type { SitemapCrawlState } from '../utils/sitemap-crawl-state.js';
|
|
3
|
+
import type { SeedRoutesOptions } from './seed.js';
|
|
4
|
+
export type { SeedRoutesOptions };
|
|
5
|
+
export { resolveSeedRefreshWindowMs, SEED_REFRESH_WINDOW_MS } from './seed.js';
|
|
3
6
|
export interface PageRow {
|
|
4
7
|
id: number;
|
|
5
8
|
route: string;
|
|
@@ -141,16 +144,20 @@ export declare function seedRoutes(event: H3Event | undefined, routes: Array<str
|
|
|
141
144
|
route: string;
|
|
142
145
|
locale?: string;
|
|
143
146
|
url?: string;
|
|
144
|
-
}
|
|
147
|
+
}>, options?: SeedRoutesOptions): Promise<number>;
|
|
145
148
|
/**
|
|
146
149
|
* Prune routes not seen in sitemap for longer than threshold
|
|
147
150
|
* Only prunes routes with source='runtime' (never prerendered pages)
|
|
151
|
+
*
|
|
152
|
+
* `seedRoutes` lets `last_seen_at` lag a sighting by up to `refreshWindowMs`,
|
|
153
|
+
* so the threshold moves back by that window. A live route is then never
|
|
154
|
+
* pruned. Pass the window the seeder used; the default is the largest one.
|
|
148
155
|
*/
|
|
149
|
-
export declare function pruneStaleRoutes(event: H3Event | undefined, staleThresholdSeconds: number, protectedSince?: number): Promise<number>;
|
|
156
|
+
export declare function pruneStaleRoutes(event: H3Event | undefined, staleThresholdSeconds: number, protectedSince?: number, refreshWindowMs?: number): Promise<number>;
|
|
150
157
|
/**
|
|
151
158
|
* Get stale routes that would be pruned (for preview)
|
|
152
159
|
*/
|
|
153
|
-
export declare function getStaleRoutes(event: H3Event | undefined, staleThresholdSeconds: number): Promise<string[]>;
|
|
160
|
+
export declare function getStaleRoutes(event: H3Event | undefined, staleThresholdSeconds: number, refreshWindowMs?: number): Promise<string[]>;
|
|
154
161
|
export interface CronRunRow {
|
|
155
162
|
id: number;
|
|
156
163
|
started_at: DatabaseNumber;
|
|
@@ -305,4 +312,3 @@ export declare function getRecentlyIndexedPages(event: H3Event | undefined, limi
|
|
|
305
312
|
* Count pages indexed in a time window
|
|
306
313
|
*/
|
|
307
314
|
export declare function countRecentlyIndexed(event: H3Event | undefined, sinceMs: number): Promise<number>;
|
|
308
|
-
export {};
|
|
@@ -1,42 +1,11 @@
|
|
|
1
1
|
import { randomUUID } from "uncrypto";
|
|
2
|
-
import { getRequestURL } from "#nuxtseo/h3";
|
|
3
2
|
import { useEvent, useRuntimeConfig } from "#nuxtseo/nitro";
|
|
4
|
-
import { createUniversalContext } from "../utils/context.js";
|
|
5
|
-
import { hostMatchesLocaleDomain, resolveLocaleFromRoute } from "../utils/i18n.js";
|
|
6
3
|
import { parseSitemapCrawlState, serializeSitemapCrawlState } from "../utils/sitemap-crawl-state.js";
|
|
7
4
|
import { initSchema } from "./drizzle/queries.js";
|
|
8
5
|
import { useRawDb } from "./drizzle/raw.js";
|
|
6
|
+
import { deriveLocale, SEED_REFRESH_WINDOW_MS } from "./seed.js";
|
|
9
7
|
import { LIKE_ESCAPE, likeSubstring, maxRowsPerInsert, normalizeRoute, normalizeRouteKey } from "./shared.js";
|
|
10
|
-
|
|
11
|
-
if (!url)
|
|
12
|
-
return void 0;
|
|
13
|
-
try {
|
|
14
|
-
return new URL(url).host || void 0;
|
|
15
|
-
} catch {
|
|
16
|
-
return void 0;
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
function deriveLocale(event, route, explicit, pageUrl) {
|
|
20
|
-
if (explicit !== void 0)
|
|
21
|
-
return explicit;
|
|
22
|
-
const cfg = useRuntimeConfig(event);
|
|
23
|
-
const i18n = cfg["nuxt-ai-ready"]?.i18n;
|
|
24
|
-
if (!i18n)
|
|
25
|
-
return "";
|
|
26
|
-
const entryHost = hostFromUrl(pageUrl);
|
|
27
|
-
if (entryHost)
|
|
28
|
-
return resolveLocaleFromRoute(route, i18n, { host: entryHost }).locale;
|
|
29
|
-
let requestHost;
|
|
30
|
-
if (event) {
|
|
31
|
-
try {
|
|
32
|
-
requestHost = getRequestURL(event).host;
|
|
33
|
-
} catch {
|
|
34
|
-
requestHost = void 0;
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
const host = requestHost && hostMatchesLocaleDomain(requestHost, i18n) ? requestHost : hostFromUrl(createUniversalContext(event).siteUrl);
|
|
38
|
-
return resolveLocaleFromRoute(route, i18n, host ? { host } : void 0).locale;
|
|
39
|
-
}
|
|
8
|
+
export { resolveSeedRefreshWindowMs, SEED_REFRESH_WINDOW_MS } from "./seed.js";
|
|
40
9
|
function getEventFromContext(providedEvent) {
|
|
41
10
|
if (providedEvent)
|
|
42
11
|
return providedEvent;
|
|
@@ -373,10 +342,11 @@ function chunk(items, size) {
|
|
|
373
342
|
result.push(items.slice(i, i + size));
|
|
374
343
|
return result;
|
|
375
344
|
}
|
|
376
|
-
export async function seedRoutes(event, routes) {
|
|
345
|
+
export async function seedRoutes(event, routes, options = {}) {
|
|
377
346
|
const db = await getDb(event);
|
|
378
347
|
if (!db || routes.length === 0)
|
|
379
348
|
return 0;
|
|
349
|
+
const refreshWindowMs = Math.max(0, Math.trunc(Number.isFinite(options.refreshWindowMs) ? options.refreshWindowMs : SEED_REFRESH_WINDOW_MS));
|
|
380
350
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
381
351
|
const nowMs = Date.now();
|
|
382
352
|
const byRoute = /* @__PURE__ */ new Map();
|
|
@@ -404,6 +374,10 @@ export async function seedRoutes(event, routes) {
|
|
|
404
374
|
locale = excluded.locale,
|
|
405
375
|
is_error = 0,
|
|
406
376
|
indexed = CASE WHEN ai_ready_pages.is_error = 1 THEN 0 ELSE ai_ready_pages.indexed END
|
|
377
|
+
WHERE ai_ready_pages.last_seen_at IS NULL
|
|
378
|
+
OR ai_ready_pages.last_seen_at < excluded.last_seen_at - ${refreshWindowMs}
|
|
379
|
+
OR ai_ready_pages.is_error = 1
|
|
380
|
+
OR ai_ready_pages.locale IS DISTINCT FROM excluded.locale
|
|
407
381
|
`,
|
|
408
382
|
params
|
|
409
383
|
});
|
|
@@ -411,12 +385,12 @@ export async function seedRoutes(event, routes) {
|
|
|
411
385
|
await db.batch(stmts);
|
|
412
386
|
return byRoute.size;
|
|
413
387
|
}
|
|
414
|
-
export async function pruneStaleRoutes(event, staleThresholdSeconds, protectedSince) {
|
|
388
|
+
export async function pruneStaleRoutes(event, staleThresholdSeconds, protectedSince, refreshWindowMs = SEED_REFRESH_WINDOW_MS) {
|
|
415
389
|
const db = await getDb(event);
|
|
416
390
|
if (!db)
|
|
417
391
|
return 0;
|
|
418
392
|
const staleThreshold = Date.now() - staleThresholdSeconds * 1e3;
|
|
419
|
-
const threshold = protectedSince === void 0 ? staleThreshold : Math.min(staleThreshold, protectedSince);
|
|
393
|
+
const threshold = (protectedSince === void 0 ? staleThreshold : Math.min(staleThreshold, protectedSince)) - refreshWindowMs;
|
|
420
394
|
const countRow = await db.first(
|
|
421
395
|
"SELECT COUNT(*) as count FROM ai_ready_pages WHERE source = ? AND last_seen_at < ?",
|
|
422
396
|
["runtime", threshold]
|
|
@@ -427,11 +401,11 @@ export async function pruneStaleRoutes(event, staleThresholdSeconds, protectedSi
|
|
|
427
401
|
}
|
|
428
402
|
return count;
|
|
429
403
|
}
|
|
430
|
-
export async function getStaleRoutes(event, staleThresholdSeconds) {
|
|
404
|
+
export async function getStaleRoutes(event, staleThresholdSeconds, refreshWindowMs = SEED_REFRESH_WINDOW_MS) {
|
|
431
405
|
const db = await getDb(event);
|
|
432
406
|
if (!db)
|
|
433
407
|
return [];
|
|
434
|
-
const threshold = Date.now() - staleThresholdSeconds * 1e3;
|
|
408
|
+
const threshold = Date.now() - staleThresholdSeconds * 1e3 - refreshWindowMs;
|
|
435
409
|
const rows = await db.all(
|
|
436
410
|
"SELECT route FROM ai_ready_pages WHERE source = ? AND last_seen_at < ?",
|
|
437
411
|
["runtime", threshold]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { H3Event } from '#nuxtseo/h3';
|
|
2
|
+
/**
|
|
3
|
+
* Resolve a route's locale, deferring to the explicit value when supplied.
|
|
4
|
+
* Falls back to the runtime i18n config (set when @nuxtjs/i18n is detected at
|
|
5
|
+
* build time). Returns '' when no i18n is configured, matching the schema's
|
|
6
|
+
* default for non-i18n sites.
|
|
7
|
+
*
|
|
8
|
+
* The host comes from the page's own URL, in order of trust:
|
|
9
|
+
* 1. the sitemap entry URL, when the caller has one
|
|
10
|
+
* 2. the request host, but only when it is itself a configured locale domain:
|
|
11
|
+
* cron and poll requests can arrive on any domain (e.g. a workers.dev
|
|
12
|
+
* preview host), which would otherwise decide the locale of every indexed
|
|
13
|
+
* page on multi-domain i18n sites
|
|
14
|
+
* 3. the site config host
|
|
15
|
+
*/
|
|
16
|
+
export declare function deriveLocale(event: H3Event | undefined, route: string, explicit?: string, pageUrl?: string): string;
|
|
17
|
+
/** Longest time `seedRoutes` lets `last_seen_at` lag behind a sighting. */
|
|
18
|
+
export declare const SEED_REFRESH_WINDOW_MS: number;
|
|
19
|
+
/**
|
|
20
|
+
* Resolve how long an unchanged seeded row may skip its `last_seen_at` write.
|
|
21
|
+
* With pruning on, the window stays at half of pruneTtl so the lag stays small
|
|
22
|
+
* against the TTL. Pass the same value to `pruneStaleRoutes`.
|
|
23
|
+
*/
|
|
24
|
+
export declare function resolveSeedRefreshWindowMs(pruneTtlSeconds: number): number;
|
|
25
|
+
export interface SeedRoutesOptions {
|
|
26
|
+
/**
|
|
27
|
+
* An existing row is rewritten only when its `last_seen_at` is older than
|
|
28
|
+
* this, when it is an error row, or when its locale changed.
|
|
29
|
+
* @default SEED_REFRESH_WINDOW_MS
|
|
30
|
+
*/
|
|
31
|
+
refreshWindowMs?: number;
|
|
32
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { getRequestHost } from "#nuxtseo/h3";
|
|
2
|
+
import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
3
|
+
import { createUniversalContext } from "../utils/context.js";
|
|
4
|
+
import { resolveI18nDomain, resolveLocaleFromRoute } from "../utils/i18n.js";
|
|
5
|
+
function hostFromUrl(url) {
|
|
6
|
+
if (!url)
|
|
7
|
+
return void 0;
|
|
8
|
+
try {
|
|
9
|
+
return new URL(url).host || void 0;
|
|
10
|
+
} catch {
|
|
11
|
+
return void 0;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
export function deriveLocale(event, route, explicit, pageUrl) {
|
|
15
|
+
if (explicit !== void 0)
|
|
16
|
+
return explicit;
|
|
17
|
+
const cfg = useRuntimeConfig(event);
|
|
18
|
+
const i18n = cfg["nuxt-ai-ready"]?.i18n;
|
|
19
|
+
if (!i18n)
|
|
20
|
+
return "";
|
|
21
|
+
const entryHost = hostFromUrl(pageUrl);
|
|
22
|
+
if (entryHost)
|
|
23
|
+
return resolveLocaleFromRoute(route, i18n, { host: entryHost }).locale;
|
|
24
|
+
let requestHost;
|
|
25
|
+
if (event) {
|
|
26
|
+
try {
|
|
27
|
+
requestHost = getRequestHost(event, { xForwardedHost: true });
|
|
28
|
+
} catch {
|
|
29
|
+
requestHost = void 0;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const host = requestHost && resolveI18nDomain(requestHost, i18n)._tag === "known" ? requestHost : hostFromUrl(createUniversalContext(event).siteUrl);
|
|
33
|
+
return resolveLocaleFromRoute(route, i18n, host ? { host } : void 0).locale;
|
|
34
|
+
}
|
|
35
|
+
export const SEED_REFRESH_WINDOW_MS = 24 * 60 * 60 * 1e3;
|
|
36
|
+
export function resolveSeedRefreshWindowMs(pruneTtlSeconds) {
|
|
37
|
+
if (!(pruneTtlSeconds > 0))
|
|
38
|
+
return SEED_REFRESH_WINDOW_MS;
|
|
39
|
+
return Math.min(SEED_REFRESH_WINDOW_MS, Math.floor(pruneTtlSeconds * 1e3 / 2));
|
|
40
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
2
2
|
import { trackDrizzleWork } from "../db/drizzle/client.js";
|
|
3
|
-
import { getPageLastmods, getSitemapLastCrawledAt, markSitemapSeeded, seedRoutes } from "../db/queries.js";
|
|
3
|
+
import { getPageLastmods, getSitemapLastCrawledAt, markSitemapSeeded, resolveSeedRefreshWindowMs, seedRoutes } from "../db/queries.js";
|
|
4
4
|
import { logger } from "../logger.js";
|
|
5
5
|
import { mapSitemapRoutes } from "../utils/sitemap-routes.js";
|
|
6
6
|
const DIAGNOSTICS_KEY = "_aiReadySitemapDiagnostics";
|
|
@@ -96,7 +96,8 @@ export default function sitemapSeederPlugin(nitroApp) {
|
|
|
96
96
|
const urlCount = urls.length;
|
|
97
97
|
const seed = async () => {
|
|
98
98
|
const seedStart = Date.now();
|
|
99
|
-
const
|
|
99
|
+
const pruneTtl = useRuntimeConfig(event)["nuxt-ai-ready"]?.runtimeSync?.pruneTtl ?? 0;
|
|
100
|
+
const seeded = await seedRoutes(event, routes, { refreshWindowMs: resolveSeedRefreshWindowMs(pruneTtl) }).catch((e) => {
|
|
100
101
|
logger.warn(`[sitemap-seeder] Failed to seed routes: ${e.message}`);
|
|
101
102
|
return 0;
|
|
102
103
|
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { eventHandler, getQuery } from "#nuxtseo/h3";
|
|
2
2
|
import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
3
|
-
import { getStaleRoutes, pruneStaleRoutes } from "../../db/queries.js";
|
|
3
|
+
import { getStaleRoutes, pruneStaleRoutes, resolveSeedRefreshWindowMs } from "../../db/queries.js";
|
|
4
4
|
export default eventHandler(async (event) => {
|
|
5
5
|
const config = useRuntimeConfig()["nuxt-ai-ready"];
|
|
6
6
|
const query = getQuery(event);
|
|
@@ -10,10 +10,11 @@ export default eventHandler(async (event) => {
|
|
|
10
10
|
requireAuth(event);
|
|
11
11
|
}
|
|
12
12
|
const ttl = query.ttl ? Math.max(0, Math.trunc(Number(query.ttl)) || config.runtimeSync.pruneTtl) : config.runtimeSync.pruneTtl;
|
|
13
|
+
const refreshWindowMs = resolveSeedRefreshWindowMs(config.runtimeSync.pruneTtl);
|
|
13
14
|
if (dry) {
|
|
14
|
-
const routes = await getStaleRoutes(event, ttl);
|
|
15
|
+
const routes = await getStaleRoutes(event, ttl, refreshWindowMs);
|
|
15
16
|
return { routes, count: routes.length, ttl, dry: true };
|
|
16
17
|
}
|
|
17
|
-
const pruned = await pruneStaleRoutes(event, ttl);
|
|
18
|
+
const pruned = await pruneStaleRoutes(event, ttl, void 0, refreshWindowMs);
|
|
18
19
|
return { pruned, ttl, dry: false };
|
|
19
20
|
});
|
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
import type { RuntimeI18nConfig } from 'nuxtseo-shared/i18n-runtime';
|
|
2
|
-
export { computeLocaleAlternates, localePath, resolveLocaleAlternates, resolveLocaleFromRoute, } from '#ai-ready-virtual/i18n-runtime.mjs';
|
|
2
|
+
export { computeLocaleAlternates, localePath, resolveI18nDomain, resolveLocaleAlternates, resolveLocaleFromRoute, } from '#ai-ready-virtual/i18n-runtime.mjs';
|
|
3
3
|
export type { LocaleAlternate, LocaleAlternateResolution, LocalePages, RouteLocaleInfo, RuntimeI18nConfig, RuntimeLocale, RuntimeRouteContext, } from 'nuxtseo-shared/i18n-runtime';
|
|
4
4
|
/** Get the runtime i18n config from runtimeConfig, or null when disabled. */
|
|
5
5
|
export declare function getRuntimeI18n(aiReadyConfig: {
|
|
6
6
|
i18n?: RuntimeI18nConfig | null;
|
|
7
7
|
}): RuntimeI18nConfig | null;
|
|
8
|
-
/** Normalize a host or URL into a comparable hostname (lowercase, no scheme, no path). */
|
|
9
|
-
export declare function normalizeHost(value: string): string;
|
|
10
|
-
/** True when the host equals a domain configured on any i18n locale. */
|
|
11
|
-
export declare function hostMatchesLocaleDomain(host: string | undefined, i18n: RuntimeI18nConfig): boolean;
|
|
@@ -1,22 +1,10 @@
|
|
|
1
1
|
export {
|
|
2
2
|
computeLocaleAlternates,
|
|
3
3
|
localePath,
|
|
4
|
+
resolveI18nDomain,
|
|
4
5
|
resolveLocaleAlternates,
|
|
5
6
|
resolveLocaleFromRoute
|
|
6
7
|
} from "#ai-ready-virtual/i18n-runtime.mjs";
|
|
7
8
|
export function getRuntimeI18n(aiReadyConfig) {
|
|
8
9
|
return aiReadyConfig.i18n || null;
|
|
9
10
|
}
|
|
10
|
-
export function normalizeHost(value) {
|
|
11
|
-
return value.trim().toLowerCase().replace(/^[a-z][a-z\d+.-]*:\/\//, "").split("/")[0];
|
|
12
|
-
}
|
|
13
|
-
export function hostMatchesLocaleDomain(host, i18n) {
|
|
14
|
-
if (!host)
|
|
15
|
-
return false;
|
|
16
|
-
const normalized = normalizeHost(host);
|
|
17
|
-
if (!normalized)
|
|
18
|
-
return false;
|
|
19
|
-
return i18n.locales.some(
|
|
20
|
-
(locale) => [...locale.domains ?? [], locale.domain, ...locale.defaultForDomains ?? []].some((domain) => domain && normalizeHost(domain) === normalized)
|
|
21
|
-
);
|
|
22
|
-
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createNitroRouteRuleMatcher } from "nuxtseo-shared/server";
|
|
2
2
|
import { localAgentSkillArtifacts } from "#ai-ready-virtual/agent-skills.mjs";
|
|
3
|
-
import { appendHeader, createError,
|
|
3
|
+
import { appendHeader, createError, getRequestHost, getResponseHeader, sendRedirect, setHeader } from "#nuxtseo/h3";
|
|
4
4
|
import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
5
5
|
import { withSiteUrl } from "#site-config/server/composables/utils";
|
|
6
6
|
import initSiteConfig from "#site-config/server/middleware/init";
|
|
@@ -64,7 +64,7 @@ export function buildNegotiationContext(event, path) {
|
|
|
64
64
|
path,
|
|
65
65
|
resolvePath,
|
|
66
66
|
resolveUrl: (target) => withSiteUrl(event, resolvePath(target)),
|
|
67
|
-
routeContext: { host:
|
|
67
|
+
routeContext: { host: getRequestHost(event, { xForwardedHost: true }) }
|
|
68
68
|
};
|
|
69
69
|
}
|
|
70
70
|
let matcherCache;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { useRuntimeConfig } from "#nuxtseo/nitro";
|
|
2
|
-
import { completeCronRun, getCronFastPathStatus, getNextSitemapToCrawl, markSitemapCrawled, markSitemapCrawlPartial, markSitemapError, pruneCronRunsByAge, pruneStaleRoutes, releaseCronLock, seedRoutes, startCronRun, syncSitemaps, tryAcquireCronLock } from "../db/queries.js";
|
|
2
|
+
import { completeCronRun, getCronFastPathStatus, getNextSitemapToCrawl, markSitemapCrawled, markSitemapCrawlPartial, markSitemapError, pruneCronRunsByAge, pruneStaleRoutes, releaseCronLock, resolveSeedRefreshWindowMs, seedRoutes, startCronRun, syncSitemaps, tryAcquireCronLock } from "../db/queries.js";
|
|
3
3
|
import { logger } from "../logger.js";
|
|
4
4
|
import { batchIndexPages } from "./batchIndex.js";
|
|
5
5
|
import { checkAndHandleStale, STALE_CHECK_INTERVAL_MS } from "./checkStale.js";
|
|
@@ -146,6 +146,7 @@ async function executeCron(event, options) {
|
|
|
146
146
|
}
|
|
147
147
|
async function pingSitemap(event, config, sitemapIntervalMinutes, debug) {
|
|
148
148
|
const { pruneTtl } = config.runtimeSync;
|
|
149
|
+
const refreshWindowMs = resolveSeedRefreshWindowMs(pruneTtl);
|
|
149
150
|
const sitemaps = getSitemapsFromConfig(event);
|
|
150
151
|
if (sitemaps.length === 0) {
|
|
151
152
|
sitemaps.push({ name: "sitemap.xml", route: "/sitemap.xml" });
|
|
@@ -157,7 +158,7 @@ async function pingSitemap(event, config, sitemapIntervalMinutes, debug) {
|
|
|
157
158
|
logger.info("[cron] No sitemaps to ping");
|
|
158
159
|
let pruned2 = 0;
|
|
159
160
|
if (pruneTtl > 0) {
|
|
160
|
-
pruned2 = await pruneStaleRoutes(event, pruneTtl);
|
|
161
|
+
pruned2 = await pruneStaleRoutes(event, pruneTtl, void 0, refreshWindowMs);
|
|
161
162
|
}
|
|
162
163
|
return { pinged: false, pruned: pruned2 };
|
|
163
164
|
}
|
|
@@ -171,7 +172,7 @@ async function pingSitemap(event, config, sitemapIntervalMinutes, debug) {
|
|
|
171
172
|
});
|
|
172
173
|
const routes = [...mapSitemapRoutes(result.urls).entries()].map(([route, url]) => ({ route, url: url.loc }));
|
|
173
174
|
if (routes.length > 0)
|
|
174
|
-
await seedRoutes(event, routes);
|
|
175
|
+
await seedRoutes(event, routes, { refreshWindowMs });
|
|
175
176
|
if (result._tag === "failed") {
|
|
176
177
|
await markSitemapError(event, nextSitemap.name, result.error);
|
|
177
178
|
return {
|
|
@@ -197,7 +198,7 @@ async function pingSitemap(event, config, sitemapIntervalMinutes, debug) {
|
|
|
197
198
|
await markSitemapCrawled(event, nextSitemap.name, result.urlsObserved);
|
|
198
199
|
let pruned = 0;
|
|
199
200
|
if (pruneTtl > 0) {
|
|
200
|
-
pruned = await pruneStaleRoutes(event, pruneTtl, result.startedAt);
|
|
201
|
+
pruned = await pruneStaleRoutes(event, pruneTtl, result.startedAt, refreshWindowMs);
|
|
201
202
|
}
|
|
202
203
|
return {
|
|
203
204
|
name: nextSitemap.name,
|
|
@@ -241,6 +241,7 @@ declare module '#ai-ready-virtual/i18n-runtime.mjs' {
|
|
|
241
241
|
export {
|
|
242
242
|
computeLocaleAlternates,
|
|
243
243
|
localePath,
|
|
244
|
+
resolveI18nDomain,
|
|
244
245
|
resolveLocaleAlternates,
|
|
245
246
|
resolveLocaleFromRoute,
|
|
246
247
|
} from 'nuxtseo-shared/i18n-runtime'
|
|
@@ -1689,11 +1690,11 @@ ${details}`);
|
|
|
1689
1690
|
nitroConfig.virtual = nitroConfig.virtual || {};
|
|
1690
1691
|
nitroConfig.virtual["#ai-ready-virtual/i18n-runtime.mjs"] = () => {
|
|
1691
1692
|
const runtimeConfig = nitroConfig.runtimeConfig?.["nuxt-ai-ready"];
|
|
1692
|
-
return runtimeConfig?.i18n ? `export { computeLocaleAlternates, localePath, resolveLocaleAlternates, resolveLocaleFromRoute } from ${JSON.stringify(nuxtSeoSharedI18nRuntimePath)}` : `
|
|
1693
|
+
return runtimeConfig?.i18n ? `export { computeLocaleAlternates, localePath, resolveI18nDomain, resolveLocaleAlternates, resolveLocaleFromRoute } from ${JSON.stringify(nuxtSeoSharedI18nRuntimePath)}` : `
|
|
1693
1694
|
const unavailable = () => {
|
|
1694
1695
|
throw new Error('[nuxt-ai-ready] i18n runtime called without i18n configuration.')
|
|
1695
1696
|
}
|
|
1696
|
-
export { unavailable as computeLocaleAlternates, unavailable as localePath, unavailable as resolveLocaleAlternates, unavailable as resolveLocaleFromRoute }
|
|
1697
|
+
export { unavailable as computeLocaleAlternates, unavailable as localePath, unavailable as resolveI18nDomain, unavailable as resolveLocaleAlternates, unavailable as resolveLocaleFromRoute }
|
|
1697
1698
|
`;
|
|
1698
1699
|
};
|
|
1699
1700
|
nitroConfig.virtual["#ai-ready-virtual/site-tools.mjs"] = `export default ${JSON.stringify(siteToolsConfig)}`;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nuxt-ai-ready",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "2.3.
|
|
4
|
+
"version": "2.3.4",
|
|
5
5
|
"description": "Best practice AI & LLM discoverability for Nuxt sites.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
"drizzle-orm": "1.0.0-rc.4",
|
|
75
75
|
"mdream": "^1.7.1",
|
|
76
76
|
"nuxt-site-config": "^4.2.3",
|
|
77
|
-
"nuxtseo-shared": "^5.3.
|
|
77
|
+
"nuxtseo-shared": "^5.3.15",
|
|
78
78
|
"pathe": "^2.0.3",
|
|
79
79
|
"pkg-types": "^2.3.2",
|
|
80
80
|
"site-config-stack": "^4.2.3",
|