nuxt-ai-ready 1.7.6 → 1.7.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -122,18 +122,7 @@ async function initCrawler(state) {
|
|
|
122
122
|
if (state.dbPath) {
|
|
123
123
|
logger.debug(`Creating directory for SQLite: ${dirname(state.dbPath)}`);
|
|
124
124
|
await mkdir(dirname(state.dbPath), { recursive: true });
|
|
125
|
-
const
|
|
126
|
-
const sqlite = new Database(state.dbPath);
|
|
127
|
-
const db = {
|
|
128
|
-
all: async (sql, params = []) => sqlite.prepare(sql).all(...params),
|
|
129
|
-
first: async (sql, params = []) => sqlite.prepare(sql).get(...params),
|
|
130
|
-
exec: async (sql, params = []) => {
|
|
131
|
-
sqlite.prepare(sql).run(...params);
|
|
132
|
-
},
|
|
133
|
-
close: async () => {
|
|
134
|
-
sqlite.close();
|
|
135
|
-
}
|
|
136
|
-
};
|
|
125
|
+
const db = await createPrerenderDatabase(state.dbPath);
|
|
137
126
|
state.db = db;
|
|
138
127
|
await initSchema(db, { ftsTokenizer: state.ftsTokenizer });
|
|
139
128
|
logger.debug(`Crawler initialized with SQLite at ${state.dbPath} (tokenizer: ${state.ftsTokenizer || "default"})`);
|
|
@@ -148,6 +137,38 @@ async function initCrawler(state) {
|
|
|
148
137
|
}
|
|
149
138
|
state.initialized = true;
|
|
150
139
|
}
|
|
140
|
+
async function createPrerenderDatabase(dbPath) {
|
|
141
|
+
const nodeVersion = Number.parseInt(process.versions.node?.split(".")[0] || "0");
|
|
142
|
+
if (nodeVersion >= 22) {
|
|
143
|
+
const { DatabaseSync } = await import('node:sqlite');
|
|
144
|
+
const sqlite2 = new DatabaseSync(dbPath);
|
|
145
|
+
return {
|
|
146
|
+
all: async (sql, params = []) => sqlite2.prepare(sql).all(...params),
|
|
147
|
+
first: async (sql, params = []) => sqlite2.prepare(sql).get(...params),
|
|
148
|
+
exec: async (sql, params = []) => {
|
|
149
|
+
if (params.length)
|
|
150
|
+
sqlite2.prepare(sql).run(...params);
|
|
151
|
+
else
|
|
152
|
+
sqlite2.exec(sql);
|
|
153
|
+
},
|
|
154
|
+
close: async () => {
|
|
155
|
+
sqlite2.close();
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
const { default: Database } = await import('better-sqlite3');
|
|
160
|
+
const sqlite = new Database(dbPath);
|
|
161
|
+
return {
|
|
162
|
+
all: async (sql, params = []) => sqlite.prepare(sql).all(...params),
|
|
163
|
+
first: async (sql, params = []) => sqlite.prepare(sql).get(...params),
|
|
164
|
+
exec: async (sql, params = []) => {
|
|
165
|
+
sqlite.prepare(sql).run(...params);
|
|
166
|
+
},
|
|
167
|
+
close: async () => {
|
|
168
|
+
sqlite.close();
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
}
|
|
151
172
|
function flattenHeadings(headings) {
|
|
152
173
|
return (headings || []).map((h) => Object.entries(h).map(([tag, text]) => `${tag}:${text}`).join("")).join("|");
|
|
153
174
|
}
|
|
@@ -488,4 +509,4 @@ function setupPrerenderHandler(options, dbPath, siteInfo, llmsTxtConfig, indexNo
|
|
|
488
509
|
});
|
|
489
510
|
}
|
|
490
511
|
|
|
491
|
-
export { detectSitemapPrerender, setupPrerenderHandler };
|
|
512
|
+
export { createPrerenderDatabase, detectSitemapPrerender, setupPrerenderHandler };
|
package/dist/module.json
CHANGED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { CronFastPathStatus } from '../db/queries.js';
|
|
2
|
+
export interface CronPlanInput {
|
|
3
|
+
status: CronFastPathStatus;
|
|
4
|
+
runtimeSyncTtlSeconds: number;
|
|
5
|
+
indexNowEnabled: boolean;
|
|
6
|
+
staleCheckNeeded: boolean;
|
|
7
|
+
now: number;
|
|
8
|
+
}
|
|
9
|
+
export interface CronPlan {
|
|
10
|
+
hasWork: boolean;
|
|
11
|
+
runIndexNow: boolean;
|
|
12
|
+
sitemapIntervalMinutes: number;
|
|
13
|
+
}
|
|
14
|
+
export declare function resolveSitemapIntervalMinutes(runtimeSyncTtlSeconds: number): number;
|
|
15
|
+
export declare function resolveCronPlan(input: CronPlanInput): CronPlan;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export function resolveSitemapIntervalMinutes(runtimeSyncTtlSeconds) {
|
|
2
|
+
return Math.max(1, Math.ceil(runtimeSyncTtlSeconds / 60));
|
|
3
|
+
}
|
|
4
|
+
export function resolveCronPlan(input) {
|
|
5
|
+
const { status, indexNowEnabled, staleCheckNeeded, now } = input;
|
|
6
|
+
const indexNowInBackoff = status.indexNowBackoff !== null && now < status.indexNowBackoff.until;
|
|
7
|
+
const runIndexNow = indexNowEnabled && status.indexNowPending > 0 && !indexNowInBackoff;
|
|
8
|
+
return {
|
|
9
|
+
sitemapIntervalMinutes: resolveSitemapIntervalMinutes(input.runtimeSyncTtlSeconds),
|
|
10
|
+
runIndexNow,
|
|
11
|
+
hasWork: staleCheckNeeded || status.pendingPages > 0 || status.sitemapsNeedCrawl > 0 || runIndexNow
|
|
12
|
+
};
|
|
13
|
+
}
|
|
@@ -3,6 +3,7 @@ import { completeCronRun, getCronFastPathStatus, getNextSitemapToCrawl, markSite
|
|
|
3
3
|
import { logger } from "../logger.js";
|
|
4
4
|
import { batchIndexPages } from "./batchIndex.js";
|
|
5
5
|
import { checkAndHandleStale, STALE_CHECK_INTERVAL_MS } from "./checkStale.js";
|
|
6
|
+
import { resolveCronPlan, resolveSitemapIntervalMinutes } from "./cron-plan.js";
|
|
6
7
|
import { syncToIndexNow } from "./indexnow.js";
|
|
7
8
|
import { crawlSitemapByRoute, getSitemapsFromConfig, mapSitemapRoutes } from "./sitemap.js";
|
|
8
9
|
export async function runCron(event, options) {
|
|
@@ -13,6 +14,8 @@ export async function runCron(event, options) {
|
|
|
13
14
|
const startTime = Date.now();
|
|
14
15
|
const results = {};
|
|
15
16
|
const allErrors = [];
|
|
17
|
+
const sitemapIntervalMinutes = resolveSitemapIntervalMinutes(config.runtimeSync.ttl);
|
|
18
|
+
let runIndexNow = !!config.indexNow;
|
|
16
19
|
const acquired = await tryAcquireCronLock(event);
|
|
17
20
|
if (!acquired) {
|
|
18
21
|
if (debug) {
|
|
@@ -25,13 +28,19 @@ export async function runCron(event, options) {
|
|
|
25
28
|
}
|
|
26
29
|
try {
|
|
27
30
|
if (config.runtimeSync.enabled) {
|
|
28
|
-
const status = await getCronFastPathStatus(event);
|
|
31
|
+
const status = await getCronFastPathStatus(event, sitemapIntervalMinutes);
|
|
29
32
|
if (status) {
|
|
30
33
|
const now = Date.now();
|
|
31
34
|
const staleCheckNeeded = !status.lastStaleCheck || now - status.lastStaleCheck >= STALE_CHECK_INTERVAL_MS;
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
|
|
35
|
+
const plan = resolveCronPlan({
|
|
36
|
+
status,
|
|
37
|
+
runtimeSyncTtlSeconds: config.runtimeSync.ttl,
|
|
38
|
+
indexNowEnabled: !!config.indexNow,
|
|
39
|
+
staleCheckNeeded,
|
|
40
|
+
now
|
|
41
|
+
});
|
|
42
|
+
runIndexNow = plan.runIndexNow;
|
|
43
|
+
if (!plan.hasWork) {
|
|
35
44
|
if (debug) {
|
|
36
45
|
const duration = Date.now() - startTime;
|
|
37
46
|
logger.info(`[cron] Fast path: no work needed (${duration}ms)`);
|
|
@@ -58,7 +67,7 @@ export async function runCron(event, options) {
|
|
|
58
67
|
}
|
|
59
68
|
}
|
|
60
69
|
if (config.runtimeSync.enabled) {
|
|
61
|
-
const sitemapResult = await pingSitemap(event, config, debug).catch((err) => {
|
|
70
|
+
const sitemapResult = await pingSitemap(event, config, sitemapIntervalMinutes, debug).catch((err) => {
|
|
62
71
|
const msg = err instanceof Error ? err.message : String(err);
|
|
63
72
|
logger.warn("[ai-ready:cron] Sitemap ping failed:", msg);
|
|
64
73
|
allErrors.push(`sitemap: ${msg}`);
|
|
@@ -95,7 +104,7 @@ export async function runCron(event, options) {
|
|
|
95
104
|
logger.info(`[cron] Index: ${indexResult.indexed} pages (${indexResult.remaining} remaining${indexResult.errors.length > 0 ? `, ${indexResult.errors.length} errors` : ""})`);
|
|
96
105
|
}
|
|
97
106
|
}
|
|
98
|
-
if (
|
|
107
|
+
if (runIndexNow) {
|
|
99
108
|
const indexNowResult = await syncToIndexNow(event, 100).catch((err) => {
|
|
100
109
|
logger.warn("[ai-ready:cron] IndexNow sync failed:", err.message);
|
|
101
110
|
return { success: false, submitted: 0, remaining: 0, error: err.message };
|
|
@@ -145,14 +154,14 @@ export async function runCron(event, options) {
|
|
|
145
154
|
});
|
|
146
155
|
}
|
|
147
156
|
}
|
|
148
|
-
async function pingSitemap(event, config, debug) {
|
|
157
|
+
async function pingSitemap(event, config, sitemapIntervalMinutes, debug) {
|
|
149
158
|
const { pruneTtl } = config.runtimeSync;
|
|
150
159
|
const sitemaps = getSitemapsFromConfig(event);
|
|
151
160
|
if (sitemaps.length === 0) {
|
|
152
161
|
sitemaps.push({ name: "sitemap.xml", route: "/sitemap.xml" });
|
|
153
162
|
}
|
|
154
163
|
await syncSitemaps(event, sitemaps);
|
|
155
|
-
const nextSitemap = await getNextSitemapToCrawl(event);
|
|
164
|
+
const nextSitemap = await getNextSitemapToCrawl(event, sitemapIntervalMinutes);
|
|
156
165
|
if (!nextSitemap) {
|
|
157
166
|
if (debug)
|
|
158
167
|
logger.info("[cron] No sitemaps to ping");
|