nuxt-ai-ready 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.mts +1 -0
- package/dist/cli.mjs +299 -0
- package/dist/module.json +1 -1
- package/dist/module.mjs +38 -85
- package/dist/runtime/server/routes/__ai-ready/cron.get.js +12 -2
- package/dist/runtime/server/routes/__ai-ready/status.get.js +7 -1
- package/dist/runtime/server/utils/indexPage.js +5 -2
- package/dist/runtime/server/utils/indexnow-shared.d.ts +57 -0
- package/dist/runtime/server/utils/indexnow-shared.js +77 -0
- package/dist/runtime/server/utils/indexnow.d.ts +1 -1
- package/dist/runtime/server/utils/indexnow.js +1 -1
- package/dist/runtime/server/utils.js +2 -1
- package/package.json +8 -4
package/dist/cli.d.mts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
package/dist/cli.mjs
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import fsp from 'node:fs/promises';
|
|
4
|
+
import { defineCommand, runMain } from 'citty';
|
|
5
|
+
import { consola } from 'consola';
|
|
6
|
+
import { colors } from 'consola/utils';
|
|
7
|
+
import { resolve, join } from 'pathe';
|
|
8
|
+
|
|
9
|
+
async function getSecret(cwd) {
|
|
10
|
+
const secretPath = join(cwd, "node_modules/.cache/nuxt/ai-ready/secret");
|
|
11
|
+
if (!existsSync(secretPath)) {
|
|
12
|
+
return null;
|
|
13
|
+
}
|
|
14
|
+
return fsp.readFile(secretPath, "utf-8").then((s) => s.trim()).catch(() => null);
|
|
15
|
+
}
|
|
16
|
+
const main = defineCommand({
|
|
17
|
+
meta: {
|
|
18
|
+
name: "nuxt-ai-ready",
|
|
19
|
+
description: "Nuxt AI Ready CLI"
|
|
20
|
+
},
|
|
21
|
+
subCommands: {
|
|
22
|
+
status: defineCommand({
|
|
23
|
+
meta: {
|
|
24
|
+
name: "status",
|
|
25
|
+
description: "Show indexing status and IndexNow sync progress"
|
|
26
|
+
},
|
|
27
|
+
args: {
|
|
28
|
+
url: {
|
|
29
|
+
type: "string",
|
|
30
|
+
alias: "u",
|
|
31
|
+
description: "Site URL (default: http://localhost:3000)",
|
|
32
|
+
default: "http://localhost:3000"
|
|
33
|
+
},
|
|
34
|
+
cwd: {
|
|
35
|
+
type: "string",
|
|
36
|
+
description: "Working directory",
|
|
37
|
+
default: "."
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
async run({ args }) {
|
|
41
|
+
const cwd = resolve(args.cwd || ".");
|
|
42
|
+
const secret = await getSecret(cwd);
|
|
43
|
+
if (!secret) {
|
|
44
|
+
consola.error("No secret found. Run `nuxi dev` or `nuxi build` first to generate one.");
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const url = `${args.url}/__ai-ready/status?secret=${secret}`;
|
|
48
|
+
consola.info(`Fetching status from ${args.url}...`);
|
|
49
|
+
const res = await fetch(url).then((r) => r.json()).catch((err) => {
|
|
50
|
+
consola.error(`Failed to connect: ${err.message}`);
|
|
51
|
+
return null;
|
|
52
|
+
});
|
|
53
|
+
if (!res)
|
|
54
|
+
return;
|
|
55
|
+
consola.box("AI Ready Status");
|
|
56
|
+
consola.info(`Total pages: ${colors.cyan(res.total?.toString() || "0")}`);
|
|
57
|
+
consola.info(`Indexed: ${colors.green(res.indexed?.toString() || "0")}`);
|
|
58
|
+
consola.info(`Pending: ${colors.yellow(res.pending?.toString() || "0")}`);
|
|
59
|
+
if (res.indexNow) {
|
|
60
|
+
consola.log("");
|
|
61
|
+
consola.info(colors.bold("IndexNow:"));
|
|
62
|
+
consola.info(` Pending: ${colors.yellow(res.indexNow.pending?.toString() || "0")}`);
|
|
63
|
+
consola.info(` Total submitted: ${colors.green(res.indexNow.totalSubmitted?.toString() || "0")}`);
|
|
64
|
+
if (res.indexNow.lastSubmittedAt) {
|
|
65
|
+
const date = new Date(res.indexNow.lastSubmittedAt);
|
|
66
|
+
consola.info(` Last submitted: ${colors.dim(date.toISOString())}`);
|
|
67
|
+
}
|
|
68
|
+
if (res.indexNow.lastError) {
|
|
69
|
+
consola.info(` Last error: ${colors.red(res.indexNow.lastError)}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}),
|
|
74
|
+
poll: defineCommand({
|
|
75
|
+
meta: {
|
|
76
|
+
name: "poll",
|
|
77
|
+
description: "Trigger page indexing"
|
|
78
|
+
},
|
|
79
|
+
args: {
|
|
80
|
+
url: {
|
|
81
|
+
type: "string",
|
|
82
|
+
alias: "u",
|
|
83
|
+
description: "Site URL (default: http://localhost:3000)",
|
|
84
|
+
default: "http://localhost:3000"
|
|
85
|
+
},
|
|
86
|
+
limit: {
|
|
87
|
+
type: "string",
|
|
88
|
+
alias: "l",
|
|
89
|
+
description: "Max pages to process",
|
|
90
|
+
default: "10"
|
|
91
|
+
},
|
|
92
|
+
all: {
|
|
93
|
+
type: "boolean",
|
|
94
|
+
alias: "a",
|
|
95
|
+
description: "Process all pending pages"
|
|
96
|
+
},
|
|
97
|
+
cwd: {
|
|
98
|
+
type: "string",
|
|
99
|
+
description: "Working directory",
|
|
100
|
+
default: "."
|
|
101
|
+
}
|
|
102
|
+
},
|
|
103
|
+
async run({ args }) {
|
|
104
|
+
const cwd = resolve(args.cwd || ".");
|
|
105
|
+
const secret = await getSecret(cwd);
|
|
106
|
+
if (!secret) {
|
|
107
|
+
consola.error("No secret found. Run `nuxi dev` or `nuxi build` first.");
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
const params = new URLSearchParams({ secret });
|
|
111
|
+
if (args.all) {
|
|
112
|
+
params.set("all", "true");
|
|
113
|
+
} else {
|
|
114
|
+
params.set("limit", args.limit || "10");
|
|
115
|
+
}
|
|
116
|
+
const url = `${args.url}/__ai-ready/poll?${params}`;
|
|
117
|
+
consola.info(`Triggering poll at ${args.url}...`);
|
|
118
|
+
const res = await fetch(url, { method: "POST" }).then((r) => r.json()).catch((err) => {
|
|
119
|
+
consola.error(`Failed: ${err.message}`);
|
|
120
|
+
return null;
|
|
121
|
+
});
|
|
122
|
+
if (!res)
|
|
123
|
+
return;
|
|
124
|
+
consola.success(`Indexed: ${colors.green(res.indexed?.toString() || "0")} pages`);
|
|
125
|
+
consola.info(`Remaining: ${colors.yellow(res.remaining?.toString() || "0")}`);
|
|
126
|
+
if (res.errors?.length) {
|
|
127
|
+
consola.warn(`Errors: ${res.errors.length}`);
|
|
128
|
+
}
|
|
129
|
+
if (res.duration) {
|
|
130
|
+
consola.info(`Duration: ${colors.dim(`${res.duration}ms`)}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}),
|
|
134
|
+
restore: defineCommand({
|
|
135
|
+
meta: {
|
|
136
|
+
name: "restore",
|
|
137
|
+
description: "Restore database from prerendered dump"
|
|
138
|
+
},
|
|
139
|
+
args: {
|
|
140
|
+
url: {
|
|
141
|
+
type: "string",
|
|
142
|
+
alias: "u",
|
|
143
|
+
description: "Site URL (default: http://localhost:3000)",
|
|
144
|
+
default: "http://localhost:3000"
|
|
145
|
+
},
|
|
146
|
+
clear: {
|
|
147
|
+
type: "boolean",
|
|
148
|
+
description: "Clear existing pages first (default: true)",
|
|
149
|
+
default: true
|
|
150
|
+
},
|
|
151
|
+
cwd: {
|
|
152
|
+
type: "string",
|
|
153
|
+
description: "Working directory",
|
|
154
|
+
default: "."
|
|
155
|
+
}
|
|
156
|
+
},
|
|
157
|
+
async run({ args }) {
|
|
158
|
+
const cwd = resolve(args.cwd || ".");
|
|
159
|
+
const secret = await getSecret(cwd);
|
|
160
|
+
if (!secret) {
|
|
161
|
+
consola.error("No secret found. Run `nuxi dev` or `nuxi build` first.");
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const params = new URLSearchParams({ secret });
|
|
165
|
+
if (!args.clear) {
|
|
166
|
+
params.set("clear", "false");
|
|
167
|
+
}
|
|
168
|
+
const url = `${args.url}/__ai-ready/restore?${params}`;
|
|
169
|
+
consola.info(`Restoring database at ${args.url}...`);
|
|
170
|
+
const res = await fetch(url, { method: "POST" }).then((r) => r.json()).catch((err) => {
|
|
171
|
+
consola.error(`Failed: ${err.message}`);
|
|
172
|
+
return null;
|
|
173
|
+
});
|
|
174
|
+
if (!res)
|
|
175
|
+
return;
|
|
176
|
+
consola.success(`Restored: ${colors.green(res.restored?.toString() || "0")} pages`);
|
|
177
|
+
if (res.cleared) {
|
|
178
|
+
consola.info(`Cleared: ${colors.yellow(res.cleared?.toString() || "0")} existing pages`);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}),
|
|
182
|
+
prune: defineCommand({
|
|
183
|
+
meta: {
|
|
184
|
+
name: "prune",
|
|
185
|
+
description: "Remove stale routes from database"
|
|
186
|
+
},
|
|
187
|
+
args: {
|
|
188
|
+
url: {
|
|
189
|
+
type: "string",
|
|
190
|
+
alias: "u",
|
|
191
|
+
description: "Site URL (default: http://localhost:3000)",
|
|
192
|
+
default: "http://localhost:3000"
|
|
193
|
+
},
|
|
194
|
+
dry: {
|
|
195
|
+
type: "boolean",
|
|
196
|
+
alias: "d",
|
|
197
|
+
description: "Preview without deleting"
|
|
198
|
+
},
|
|
199
|
+
ttl: {
|
|
200
|
+
type: "string",
|
|
201
|
+
description: "Override pruneTtl config"
|
|
202
|
+
},
|
|
203
|
+
cwd: {
|
|
204
|
+
type: "string",
|
|
205
|
+
description: "Working directory",
|
|
206
|
+
default: "."
|
|
207
|
+
}
|
|
208
|
+
},
|
|
209
|
+
async run({ args }) {
|
|
210
|
+
const cwd = resolve(args.cwd || ".");
|
|
211
|
+
const secret = await getSecret(cwd);
|
|
212
|
+
if (!secret && !args.dry) {
|
|
213
|
+
consola.error("No secret found. Run `nuxi dev` or `nuxi build` first.");
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
const params = new URLSearchParams();
|
|
217
|
+
if (secret)
|
|
218
|
+
params.set("secret", secret);
|
|
219
|
+
if (args.dry)
|
|
220
|
+
params.set("dry", "true");
|
|
221
|
+
if (args.ttl)
|
|
222
|
+
params.set("ttl", args.ttl);
|
|
223
|
+
const url = `${args.url}/__ai-ready/prune?${params}`;
|
|
224
|
+
consola.info(`${args.dry ? "Previewing" : "Pruning"} stale routes at ${args.url}...`);
|
|
225
|
+
const res = await fetch(url, { method: "POST" }).then((r) => r.json()).catch((err) => {
|
|
226
|
+
consola.error(`Failed: ${err.message}`);
|
|
227
|
+
return null;
|
|
228
|
+
});
|
|
229
|
+
if (!res)
|
|
230
|
+
return;
|
|
231
|
+
if (args.dry) {
|
|
232
|
+
consola.info(`Would prune: ${colors.yellow(res.count?.toString() || "0")} routes`);
|
|
233
|
+
if (res.routes?.length) {
|
|
234
|
+
for (const route of res.routes.slice(0, 20)) {
|
|
235
|
+
consola.log(` ${colors.dim("\u2022")} ${route}`);
|
|
236
|
+
}
|
|
237
|
+
if (res.routes.length > 20) {
|
|
238
|
+
consola.log(` ${colors.dim(`... and ${res.routes.length - 20} more`)}`);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
} else {
|
|
242
|
+
consola.success(`Pruned: ${colors.green(res.pruned?.toString() || "0")} routes`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
}),
|
|
246
|
+
indexnow: defineCommand({
|
|
247
|
+
meta: {
|
|
248
|
+
name: "indexnow",
|
|
249
|
+
description: "Trigger IndexNow sync"
|
|
250
|
+
},
|
|
251
|
+
args: {
|
|
252
|
+
url: {
|
|
253
|
+
type: "string",
|
|
254
|
+
alias: "u",
|
|
255
|
+
description: "Site URL (default: http://localhost:3000)",
|
|
256
|
+
default: "http://localhost:3000"
|
|
257
|
+
},
|
|
258
|
+
limit: {
|
|
259
|
+
type: "string",
|
|
260
|
+
alias: "l",
|
|
261
|
+
description: "Max URLs to submit",
|
|
262
|
+
default: "100"
|
|
263
|
+
},
|
|
264
|
+
cwd: {
|
|
265
|
+
type: "string",
|
|
266
|
+
description: "Working directory",
|
|
267
|
+
default: "."
|
|
268
|
+
}
|
|
269
|
+
},
|
|
270
|
+
async run({ args }) {
|
|
271
|
+
const cwd = resolve(args.cwd || ".");
|
|
272
|
+
const secret = await getSecret(cwd);
|
|
273
|
+
if (!secret) {
|
|
274
|
+
consola.error("No secret found. Run `nuxi dev` or `nuxi build` first.");
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
const params = new URLSearchParams({
|
|
278
|
+
secret,
|
|
279
|
+
limit: args.limit || "100"
|
|
280
|
+
});
|
|
281
|
+
const url = `${args.url}/__ai-ready/indexnow?${params}`;
|
|
282
|
+
consola.info(`Triggering IndexNow sync at ${args.url}...`);
|
|
283
|
+
const res = await fetch(url, { method: "POST" }).then((r) => r.json()).catch((err) => {
|
|
284
|
+
consola.error(`Failed: ${err.message}`);
|
|
285
|
+
return null;
|
|
286
|
+
});
|
|
287
|
+
if (!res)
|
|
288
|
+
return;
|
|
289
|
+
if (res.success) {
|
|
290
|
+
consola.success(`Submitted: ${colors.green(res.submitted?.toString() || "0")} URLs`);
|
|
291
|
+
consola.info(`Remaining: ${colors.yellow(res.remaining?.toString() || "0")}`);
|
|
292
|
+
} else {
|
|
293
|
+
consola.error(`Failed: ${res.error || "Unknown error"}`);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
})
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
runMain(main);
|
package/dist/module.json
CHANGED
package/dist/module.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { randomBytes } from 'node:crypto';
|
|
1
2
|
import { mkdir, writeFile, appendFile, stat, access } from 'node:fs/promises';
|
|
2
3
|
import { join, dirname } from 'node:path';
|
|
3
4
|
import { useLogger, useNuxt, addTemplate, addTypeTemplate, defineNuxtModule, createResolver, hasNuxtModule, addServerHandler, addPlugin } from '@nuxt/kit';
|
|
@@ -11,11 +12,16 @@ import { parseSitemapXml } from '@nuxtjs/sitemap/utils';
|
|
|
11
12
|
import { colorize } from 'consola/utils';
|
|
12
13
|
import { withBase } from 'ufo';
|
|
13
14
|
import { createAdapter, initSchema, computeContentHash, insertPage, queryAllPages, exportDbDump } from '../dist/runtime/server/db/shared.js';
|
|
15
|
+
import { comparePageHashes, submitToIndexNowShared } from '../dist/runtime/server/utils/indexnow-shared.js';
|
|
14
16
|
import { buildLlmsFullTxtHeader, formatPageForLlmsFullTxt } from '../dist/runtime/server/utils/llms-full.js';
|
|
15
17
|
import { join as join$1, isAbsolute } from 'pathe';
|
|
16
18
|
|
|
17
19
|
const logger = useLogger("nuxt-ai-ready");
|
|
18
20
|
|
|
21
|
+
const moduleRegistrations = [];
|
|
22
|
+
function registerModule(registration) {
|
|
23
|
+
moduleRegistrations.push(registration);
|
|
24
|
+
}
|
|
19
25
|
function hookNuxtSeoProLicense() {
|
|
20
26
|
const nuxt = useNuxt();
|
|
21
27
|
const isBuild = !nuxt.options.dev && !nuxt.options._prepare;
|
|
@@ -43,7 +49,13 @@ function hookNuxtSeoProLicense() {
|
|
|
43
49
|
const siteName = siteConfig.name || void 0;
|
|
44
50
|
const res = await $fetch("https://nuxtseo.com/api/pro/verify", {
|
|
45
51
|
method: "POST",
|
|
46
|
-
body: {
|
|
52
|
+
body: {
|
|
53
|
+
apiKey: license,
|
|
54
|
+
siteUrl,
|
|
55
|
+
siteName,
|
|
56
|
+
// Include registered modules for dashboard integration
|
|
57
|
+
modules: moduleRegistrations.length > 0 ? moduleRegistrations : void 0
|
|
58
|
+
}
|
|
47
59
|
}).catch((err) => {
|
|
48
60
|
if (err?.response?.status === 401) {
|
|
49
61
|
spinner.stop("\u274C Invalid API key");
|
|
@@ -67,84 +79,6 @@ function hookNuxtSeoProLicense() {
|
|
|
67
79
|
}
|
|
68
80
|
}
|
|
69
81
|
|
|
70
|
-
const INDEXNOW_HOSTS = ["api.indexnow.org", "www.bing.com"];
|
|
71
|
-
function getIndexNowEndpoints() {
|
|
72
|
-
const testEndpoint = process.env.INDEXNOW_TEST_ENDPOINT;
|
|
73
|
-
if (testEndpoint) {
|
|
74
|
-
return [testEndpoint];
|
|
75
|
-
}
|
|
76
|
-
return INDEXNOW_HOSTS.map((host) => `https://${host}/indexnow`);
|
|
77
|
-
}
|
|
78
|
-
function buildIndexNowBody(routes, key, siteUrl) {
|
|
79
|
-
const urlList = routes.map(
|
|
80
|
-
(route) => route.startsWith("http") ? route : `${siteUrl}${route}`
|
|
81
|
-
);
|
|
82
|
-
return {
|
|
83
|
-
host: new URL(siteUrl).host,
|
|
84
|
-
key,
|
|
85
|
-
keyLocation: `${siteUrl}/${key}.txt`,
|
|
86
|
-
urlList
|
|
87
|
-
};
|
|
88
|
-
}
|
|
89
|
-
async function submitToIndexNow(routes, key, siteUrl, options) {
|
|
90
|
-
if (!siteUrl) {
|
|
91
|
-
return { success: false, error: "Site URL not configured" };
|
|
92
|
-
}
|
|
93
|
-
const fetchFn = options?.fetchFn ?? globalThis.fetch;
|
|
94
|
-
const log = options?.logger;
|
|
95
|
-
const body = buildIndexNowBody(routes, key, siteUrl);
|
|
96
|
-
const endpoints = getIndexNowEndpoints();
|
|
97
|
-
let lastError;
|
|
98
|
-
for (const endpoint of endpoints) {
|
|
99
|
-
log?.debug(`[indexnow] Submitting ${body.urlList.length} URLs to ${endpoint}`);
|
|
100
|
-
const response = await fetchFn(endpoint, {
|
|
101
|
-
method: "POST",
|
|
102
|
-
headers: { "Content-Type": "application/json" },
|
|
103
|
-
body: JSON.stringify(body)
|
|
104
|
-
}).then((r) => r.ok ? { ok: true } : { error: `HTTP ${r.status}` }).catch((err) => ({ error: err.message }));
|
|
105
|
-
if ("error" in response) {
|
|
106
|
-
lastError = response.error;
|
|
107
|
-
if (lastError?.includes("429")) {
|
|
108
|
-
log?.warn(`[indexnow] Rate limited on ${endpoint}, trying fallback...`);
|
|
109
|
-
continue;
|
|
110
|
-
}
|
|
111
|
-
log?.warn(`[indexnow] Submission failed on ${endpoint}: ${lastError}`);
|
|
112
|
-
return { success: false, error: lastError, host: endpoint };
|
|
113
|
-
}
|
|
114
|
-
log?.debug(`[indexnow] Successfully submitted ${body.urlList.length} URLs via ${endpoint}`);
|
|
115
|
-
return { success: true, host: endpoint };
|
|
116
|
-
}
|
|
117
|
-
return {
|
|
118
|
-
success: false,
|
|
119
|
-
error: lastError || "All endpoints rate limited",
|
|
120
|
-
host: endpoints[endpoints.length - 1]
|
|
121
|
-
};
|
|
122
|
-
}
|
|
123
|
-
function comparePageHashes(currentPages, prevMeta) {
|
|
124
|
-
if (!prevMeta?.pages) {
|
|
125
|
-
return { changed: [], added: [], removed: [] };
|
|
126
|
-
}
|
|
127
|
-
const prevHashes = new Map(prevMeta.pages.map((p) => [p.route, p.hash]));
|
|
128
|
-
const currentRoutes = new Set(currentPages.map((p) => p.route));
|
|
129
|
-
const changed = [];
|
|
130
|
-
const added = [];
|
|
131
|
-
for (const page of currentPages) {
|
|
132
|
-
const prevHash = prevHashes.get(page.route);
|
|
133
|
-
if (!prevHash) {
|
|
134
|
-
added.push(page.route);
|
|
135
|
-
} else if (prevHash !== page.hash) {
|
|
136
|
-
changed.push(page.route);
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
const removed = [];
|
|
140
|
-
for (const route of prevHashes.keys()) {
|
|
141
|
-
if (!currentRoutes.has(route)) {
|
|
142
|
-
removed.push(route);
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
return { changed, added, removed };
|
|
146
|
-
}
|
|
147
|
-
|
|
148
82
|
async function fetchPreviousMeta(siteUrl, indexNowKey) {
|
|
149
83
|
const metaUrl = `${siteUrl}/__ai-ready/pages.meta.json`;
|
|
150
84
|
logger.debug(`[indexnow] Fetching previous meta from ${metaUrl}`);
|
|
@@ -169,7 +103,7 @@ async function handleStaticIndexNow(currentPages, siteUrl, indexNowKey, prevMeta
|
|
|
169
103
|
return;
|
|
170
104
|
}
|
|
171
105
|
logger.info(`[indexnow] Submitting ${totalChanged} changed pages (${changed.length} modified, ${added.length} new)`);
|
|
172
|
-
const result = await
|
|
106
|
+
const result = await submitToIndexNowShared(
|
|
173
107
|
[...changed, ...added],
|
|
174
108
|
indexNowKey,
|
|
175
109
|
siteUrl,
|
|
@@ -701,6 +635,28 @@ const module$1 = defineNuxtModule({
|
|
|
701
635
|
mergedLlmsTxt.notes = llmsTxtPayload.notes.length > 0 ? llmsTxtPayload.notes : void 0;
|
|
702
636
|
const prerenderCacheDir = join(nuxt.options.rootDir, "node_modules/.cache/nuxt-seo/ai-ready/routes");
|
|
703
637
|
const buildDbPath = join(nuxt.options.buildDir, ".data/ai-ready/build.db");
|
|
638
|
+
const runtimeSyncConfig = typeof config.runtimeSync === "object" ? config.runtimeSync : {};
|
|
639
|
+
const runtimeSyncEnabled = !!config.runtimeSync || !!config.cron;
|
|
640
|
+
const indexNowKey = config.indexNowKey || process.env.NUXT_AI_READY_INDEX_NOW_KEY;
|
|
641
|
+
let runtimeSyncSecret = config.runtimeSyncSecret || process.env.NUXT_AI_READY_RUNTIME_SYNC_SECRET;
|
|
642
|
+
if (!runtimeSyncSecret && runtimeSyncEnabled) {
|
|
643
|
+
runtimeSyncSecret = randomBytes(32).toString("hex");
|
|
644
|
+
logger.info(`Generated runtimeSyncSecret (use NUXT_AI_READY_RUNTIME_SYNC_SECRET env to set explicitly)`);
|
|
645
|
+
}
|
|
646
|
+
if (runtimeSyncSecret) {
|
|
647
|
+
const cacheDir = join(nuxt.options.rootDir, "node_modules/.cache/nuxt/ai-ready");
|
|
648
|
+
await mkdir(cacheDir, { recursive: true });
|
|
649
|
+
await writeFile(join(cacheDir, "secret"), runtimeSyncSecret);
|
|
650
|
+
}
|
|
651
|
+
registerModule({
|
|
652
|
+
name: "nuxt-ai-ready",
|
|
653
|
+
secret: runtimeSyncSecret,
|
|
654
|
+
features: {
|
|
655
|
+
cron: !!config.cron,
|
|
656
|
+
indexNow: !!indexNowKey,
|
|
657
|
+
runtimeSync: runtimeSyncEnabled
|
|
658
|
+
}
|
|
659
|
+
});
|
|
704
660
|
nuxt.hooks.hook("nitro:config", (nitroConfig) => {
|
|
705
661
|
nitroConfig.experimental = nitroConfig.experimental || {};
|
|
706
662
|
nitroConfig.experimental.asyncContext = true;
|
|
@@ -713,7 +669,7 @@ const module$1 = defineNuxtModule({
|
|
|
713
669
|
nitroConfig.vercel.config.crons = nitroConfig.vercel.config.crons || [];
|
|
714
670
|
nitroConfig.vercel.config.crons.push({
|
|
715
671
|
schedule: cronSchedule,
|
|
716
|
-
path: "/__ai-ready/cron"
|
|
672
|
+
path: runtimeSyncSecret ? `/__ai-ready/cron?secret=${runtimeSyncSecret}` : "/__ai-ready/cron"
|
|
717
673
|
});
|
|
718
674
|
} else {
|
|
719
675
|
nitroConfig.experimental.tasks = true;
|
|
@@ -774,9 +730,6 @@ export async function readPageDataFromFilesystem() {
|
|
|
774
730
|
export const errorRoutes = []`;
|
|
775
731
|
});
|
|
776
732
|
const database = refineDatabaseConfig(config.database || {}, nuxt.options.rootDir);
|
|
777
|
-
const runtimeSyncConfig = typeof config.runtimeSync === "object" ? config.runtimeSync : {};
|
|
778
|
-
const runtimeSyncEnabled = !!config.runtimeSync || !!config.cron;
|
|
779
|
-
const indexNowKey = config.indexNowKey || process.env.NUXT_AI_READY_INDEX_NOW_KEY;
|
|
780
733
|
nuxt.options.runtimeConfig["nuxt-ai-ready"] = {
|
|
781
734
|
version: version || "0.0.0",
|
|
782
735
|
debug: config.debug || false,
|
|
@@ -795,7 +748,7 @@ export const errorRoutes = []`;
|
|
|
795
748
|
batchSize: runtimeSyncConfig.batchSize ?? 20,
|
|
796
749
|
pruneTtl: runtimeSyncConfig.pruneTtl ?? 0
|
|
797
750
|
},
|
|
798
|
-
runtimeSyncSecret
|
|
751
|
+
runtimeSyncSecret,
|
|
799
752
|
indexNowKey
|
|
800
753
|
};
|
|
801
754
|
addServerHandler({
|
|
@@ -1,3 +1,13 @@
|
|
|
1
|
-
import { eventHandler } from "h3";
|
|
1
|
+
import { createError, eventHandler, getQuery } from "h3";
|
|
2
|
+
import { useRuntimeConfig } from "nitropack/runtime";
|
|
2
3
|
import { runCron } from "../../utils/runCron.js";
|
|
3
|
-
export default eventHandler((event) =>
|
|
4
|
+
export default eventHandler((event) => {
|
|
5
|
+
const config = useRuntimeConfig(event)["nuxt-ai-ready"];
|
|
6
|
+
if (config.runtimeSyncSecret) {
|
|
7
|
+
const { secret } = getQuery(event);
|
|
8
|
+
if (secret !== config.runtimeSyncSecret) {
|
|
9
|
+
throw createError({ statusCode: 401, message: "Unauthorized" });
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
return runCron(event);
|
|
13
|
+
});
|
|
@@ -1,8 +1,14 @@
|
|
|
1
|
-
import { eventHandler } from "h3";
|
|
1
|
+
import { createError, eventHandler, getQuery } from "h3";
|
|
2
2
|
import { useRuntimeConfig } from "nitropack/runtime";
|
|
3
3
|
import { countPages, countPagesNeedingIndexNowSync, getIndexNowStats } from "../../db/queries.js";
|
|
4
4
|
export default eventHandler(async (event) => {
|
|
5
5
|
const config = useRuntimeConfig(event)["nuxt-ai-ready"];
|
|
6
|
+
if (config.runtimeSyncSecret) {
|
|
7
|
+
const { secret } = getQuery(event);
|
|
8
|
+
if (secret !== config.runtimeSyncSecret) {
|
|
9
|
+
throw createError({ statusCode: 401, message: "Unauthorized" });
|
|
10
|
+
}
|
|
11
|
+
}
|
|
6
12
|
const [total, pending] = await Promise.all([
|
|
7
13
|
countPages(event),
|
|
8
14
|
countPages(event, { where: { pending: true } })
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { useSiteConfig } from "#site-config/server/composables/useSiteConfig";
|
|
2
|
+
import { useEvent, useNitroApp, useRuntimeConfig } from "nitropack/runtime";
|
|
2
3
|
import { getPageHash, isPageFresh, queryPages, upsertPage } from "../db/queries.js";
|
|
3
4
|
import { computeContentHash } from "../db/shared.js";
|
|
4
5
|
import { logger } from "../logger.js";
|
|
@@ -15,7 +16,9 @@ export async function indexPage(route, html, options = {}, event) {
|
|
|
15
16
|
const existing = await queryPages(event, { route });
|
|
16
17
|
const isUpdate = !!existing;
|
|
17
18
|
const isError = html.includes("__NUXT_ERROR__") || html.includes("nuxt-error-page");
|
|
18
|
-
const
|
|
19
|
+
const siteConfig = useSiteConfig(event || useEvent());
|
|
20
|
+
const fullUrl = `${siteConfig.url}${route}`;
|
|
21
|
+
const result = await convertHtmlToMarkdown(html, fullUrl, config.mdreamOptions, { extractUpdatedAt: true });
|
|
19
22
|
const updatedAt = result.updatedAt || (/* @__PURE__ */ new Date()).toISOString();
|
|
20
23
|
const headings = JSON.stringify(result.headings);
|
|
21
24
|
const keywords = extractKeywords(result.textContent, result.metaKeywords);
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared IndexNow utilities for build-time and runtime
|
|
3
|
+
* This module has no Nuxt/Nitro dependencies so it can be used in both contexts
|
|
4
|
+
*/
|
|
5
|
+
export declare const INDEXNOW_HOSTS: string[];
|
|
6
|
+
/**
|
|
7
|
+
* Get IndexNow endpoints, with test override support
|
|
8
|
+
*/
|
|
9
|
+
export declare function getIndexNowEndpoints(): string[];
|
|
10
|
+
export interface IndexNowSubmitResult {
|
|
11
|
+
success: boolean;
|
|
12
|
+
error?: string;
|
|
13
|
+
host?: string;
|
|
14
|
+
}
|
|
15
|
+
export interface IndexNowRequestBody {
|
|
16
|
+
host: string;
|
|
17
|
+
key: string;
|
|
18
|
+
keyLocation: string;
|
|
19
|
+
urlList: string[];
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Build the IndexNow API request body
|
|
23
|
+
*/
|
|
24
|
+
export declare function buildIndexNowBody(routes: string[], key: string, siteUrl: string): IndexNowRequestBody;
|
|
25
|
+
export interface SubmitOptions {
|
|
26
|
+
/** Custom fetch implementation (defaults to globalThis.fetch) */
|
|
27
|
+
fetchFn?: typeof fetch;
|
|
28
|
+
/** Logger for debug/warn messages (optional) */
|
|
29
|
+
logger?: {
|
|
30
|
+
debug: (msg: string) => void;
|
|
31
|
+
warn: (msg: string) => void;
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Submit URLs to IndexNow API with fallback on rate limit
|
|
36
|
+
* Works in both build-time (native fetch) and runtime ($fetch) contexts
|
|
37
|
+
*/
|
|
38
|
+
export declare function submitToIndexNowShared(routes: string[], key: string, siteUrl: string, options?: SubmitOptions): Promise<IndexNowSubmitResult>;
|
|
39
|
+
export interface PageHashMeta {
|
|
40
|
+
route: string;
|
|
41
|
+
hash: string;
|
|
42
|
+
}
|
|
43
|
+
export interface BuildMeta {
|
|
44
|
+
buildId: string;
|
|
45
|
+
pageCount: number;
|
|
46
|
+
createdAt: string;
|
|
47
|
+
pages: PageHashMeta[];
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Compare page hashes between current and previous builds
|
|
51
|
+
* Returns changed, added, and removed routes
|
|
52
|
+
*/
|
|
53
|
+
export declare function comparePageHashes(currentPages: PageHashMeta[], prevMeta: BuildMeta | null | undefined): {
|
|
54
|
+
changed: string[];
|
|
55
|
+
added: string[];
|
|
56
|
+
removed: string[];
|
|
57
|
+
};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
export const INDEXNOW_HOSTS = ["api.indexnow.org", "www.bing.com"];
|
|
2
|
+
export function getIndexNowEndpoints() {
|
|
3
|
+
const testEndpoint = process.env.INDEXNOW_TEST_ENDPOINT;
|
|
4
|
+
if (testEndpoint) {
|
|
5
|
+
return [testEndpoint];
|
|
6
|
+
}
|
|
7
|
+
return INDEXNOW_HOSTS.map((host) => `https://${host}/indexnow`);
|
|
8
|
+
}
|
|
9
|
+
export function buildIndexNowBody(routes, key, siteUrl) {
|
|
10
|
+
const urlList = routes.map(
|
|
11
|
+
(route) => route.startsWith("http") ? route : `${siteUrl}${route}`
|
|
12
|
+
);
|
|
13
|
+
return {
|
|
14
|
+
host: new URL(siteUrl).host,
|
|
15
|
+
key,
|
|
16
|
+
keyLocation: `${siteUrl}/${key}.txt`,
|
|
17
|
+
urlList
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
export async function submitToIndexNowShared(routes, key, siteUrl, options) {
|
|
21
|
+
if (!siteUrl) {
|
|
22
|
+
return { success: false, error: "Site URL not configured" };
|
|
23
|
+
}
|
|
24
|
+
const fetchFn = options?.fetchFn ?? globalThis.fetch;
|
|
25
|
+
const log = options?.logger;
|
|
26
|
+
const body = buildIndexNowBody(routes, key, siteUrl);
|
|
27
|
+
const endpoints = getIndexNowEndpoints();
|
|
28
|
+
let lastError;
|
|
29
|
+
for (const endpoint of endpoints) {
|
|
30
|
+
log?.debug(`[indexnow] Submitting ${body.urlList.length} URLs to ${endpoint}`);
|
|
31
|
+
const response = await fetchFn(endpoint, {
|
|
32
|
+
method: "POST",
|
|
33
|
+
headers: { "Content-Type": "application/json" },
|
|
34
|
+
body: JSON.stringify(body)
|
|
35
|
+
}).then((r) => r.ok ? { ok: true } : { error: `HTTP ${r.status}` }).catch((err) => ({ error: err.message }));
|
|
36
|
+
if ("error" in response) {
|
|
37
|
+
lastError = response.error;
|
|
38
|
+
if (lastError?.includes("429")) {
|
|
39
|
+
log?.warn(`[indexnow] Rate limited on ${endpoint}, trying fallback...`);
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
log?.warn(`[indexnow] Submission failed on ${endpoint}: ${lastError}`);
|
|
43
|
+
return { success: false, error: lastError, host: endpoint };
|
|
44
|
+
}
|
|
45
|
+
log?.debug(`[indexnow] Successfully submitted ${body.urlList.length} URLs via ${endpoint}`);
|
|
46
|
+
return { success: true, host: endpoint };
|
|
47
|
+
}
|
|
48
|
+
return {
|
|
49
|
+
success: false,
|
|
50
|
+
error: lastError || "All endpoints rate limited",
|
|
51
|
+
host: endpoints[endpoints.length - 1]
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
export function comparePageHashes(currentPages, prevMeta) {
|
|
55
|
+
if (!prevMeta?.pages) {
|
|
56
|
+
return { changed: [], added: [], removed: [] };
|
|
57
|
+
}
|
|
58
|
+
const prevHashes = new Map(prevMeta.pages.map((p) => [p.route, p.hash]));
|
|
59
|
+
const currentRoutes = new Set(currentPages.map((p) => p.route));
|
|
60
|
+
const changed = [];
|
|
61
|
+
const added = [];
|
|
62
|
+
for (const page of currentPages) {
|
|
63
|
+
const prevHash = prevHashes.get(page.route);
|
|
64
|
+
if (!prevHash) {
|
|
65
|
+
added.push(page.route);
|
|
66
|
+
} else if (prevHash !== page.hash) {
|
|
67
|
+
changed.push(page.route);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const removed = [];
|
|
71
|
+
for (const route of prevHashes.keys()) {
|
|
72
|
+
if (!currentRoutes.has(route)) {
|
|
73
|
+
removed.push(route);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return { changed, added, removed };
|
|
77
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { H3Event } from 'h3';
|
|
2
|
-
export type { BuildMeta, IndexNowSubmitResult, PageHashMeta } from '
|
|
2
|
+
export type { BuildMeta, IndexNowSubmitResult, PageHashMeta } from './indexnow-shared.js';
|
|
3
3
|
export interface IndexNowResult {
|
|
4
4
|
success: boolean;
|
|
5
5
|
submitted: number;
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { getSiteConfig } from "#site-config/server/composables";
|
|
2
2
|
import { useRuntimeConfig } from "nitropack/runtime";
|
|
3
|
-
import { submitToIndexNow as submitToIndexNowShared } from "../../../shared/indexnow";
|
|
4
3
|
import { useDatabase } from "../db/index.js";
|
|
5
4
|
import {
|
|
6
5
|
batchIndexNowUpdate,
|
|
@@ -10,6 +9,7 @@ import {
|
|
|
10
9
|
updateIndexNowStats
|
|
11
10
|
} from "../db/queries.js";
|
|
12
11
|
import { logger } from "../logger.js";
|
|
12
|
+
import { submitToIndexNowShared } from "./indexnow-shared.js";
|
|
13
13
|
const BACKOFF_MINUTES = [5, 10, 20, 40, 60];
|
|
14
14
|
async function getBackoffInfo(event) {
|
|
15
15
|
const db = await useDatabase(event).catch(() => null);
|
|
@@ -35,7 +35,8 @@ function buildMdreamOptions(url, mdreamOptions, meta, extractUpdatedAt = false)
|
|
|
35
35
|
}
|
|
36
36
|
}
|
|
37
37
|
});
|
|
38
|
-
|
|
38
|
+
const origin = new URL(url).origin;
|
|
39
|
+
let options = { origin, ...mdreamOptions };
|
|
39
40
|
if (mdreamOptions?.preset === "minimal") {
|
|
40
41
|
options = withMinimalPreset(options);
|
|
41
42
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nuxt-ai-ready",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.8.
|
|
4
|
+
"version": "0.8.2",
|
|
5
5
|
"description": "Best practice AI & LLM discoverability for Nuxt sites.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -32,6 +32,9 @@
|
|
|
32
32
|
}
|
|
33
33
|
},
|
|
34
34
|
"main": "./dist/module.mjs",
|
|
35
|
+
"bin": {
|
|
36
|
+
"nuxt-ai-ready": "./dist/cli.mjs"
|
|
37
|
+
},
|
|
35
38
|
"files": [
|
|
36
39
|
"dist",
|
|
37
40
|
"mcp.d.ts"
|
|
@@ -48,6 +51,7 @@
|
|
|
48
51
|
"dependencies": {
|
|
49
52
|
"@clack/prompts": "^0.11.0",
|
|
50
53
|
"@nuxt/kit": "^4.2.2",
|
|
54
|
+
"citty": "^0.1.6",
|
|
51
55
|
"consola": "^3.4.2",
|
|
52
56
|
"db0": "^0.3.4",
|
|
53
57
|
"defu": "^6.1.4",
|
|
@@ -78,12 +82,12 @@
|
|
|
78
82
|
"@vue/test-utils": "^2.4.6",
|
|
79
83
|
"@vueuse/nuxt": "^14.1.0",
|
|
80
84
|
"agents": "^0.3.5",
|
|
81
|
-
"ai": "^6.0.
|
|
85
|
+
"ai": "^6.0.38",
|
|
82
86
|
"better-sqlite3": "^12.6.0",
|
|
83
87
|
"bumpp": "^10.4.0",
|
|
84
88
|
"eslint": "^9.39.2",
|
|
85
89
|
"execa": "^9.6.1",
|
|
86
|
-
"happy-dom": "^20.3.
|
|
90
|
+
"happy-dom": "^20.3.1",
|
|
87
91
|
"nuxt": "^4.2.2",
|
|
88
92
|
"nuxt-site-config": "^3.2.17",
|
|
89
93
|
"playwright": "^1.57.0",
|
|
@@ -94,7 +98,7 @@
|
|
|
94
98
|
"vue": "^3.5.26",
|
|
95
99
|
"vue-router": "^4.6.4",
|
|
96
100
|
"vue-tsc": "^3.2.2",
|
|
97
|
-
"wrangler": "^4.59.
|
|
101
|
+
"wrangler": "^4.59.2",
|
|
98
102
|
"zod": "^4.3.5"
|
|
99
103
|
},
|
|
100
104
|
"resolutions": {
|