@goflag/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +207 -0
- package/dist/chunk-75OFMTOC.js +3719 -0
- package/dist/chunk-75OFMTOC.js.map +1 -0
- package/dist/cli.js +612 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +589 -0
- package/dist/index.js +28 -0
- package/dist/index.js.map +1 -0
- package/package.json +88 -0
|
@@ -0,0 +1,3719 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/report/fingerprint.ts
|
|
4
|
+
import { createHash } from "crypto";
|
|
5
|
+
function routeKey(rawUrl) {
|
|
6
|
+
try {
|
|
7
|
+
const u = new URL(rawUrl);
|
|
8
|
+
const path = u.pathname.replace(/\/+$/, "") || "/";
|
|
9
|
+
return `${path}${u.search}`;
|
|
10
|
+
} catch {
|
|
11
|
+
return rawUrl;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
function targetKey(rawUrl) {
|
|
15
|
+
try {
|
|
16
|
+
const u = new URL(rawUrl);
|
|
17
|
+
const path = u.pathname.replace(/\/+$/, "") || "/";
|
|
18
|
+
return `${u.origin}${path}${u.search}`;
|
|
19
|
+
} catch {
|
|
20
|
+
return rawUrl;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function fingerprint(category, ...parts) {
|
|
24
|
+
const digest = createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 10);
|
|
25
|
+
return `${category}-${digest}`;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// src/lib/core/extract/static.ts
|
|
29
|
+
import * as cheerio from "cheerio";
|
|
30
|
+
|
|
31
|
+
// src/lib/core/types.ts
|
|
32
|
+
var PAGE_SCHEMA_VERSION = 2;
|
|
33
|
+
|
|
34
|
+
// src/lib/core/extract/json-ld.ts
|
|
35
|
+
function parseJsonLdScripts(scripts) {
|
|
36
|
+
const blocks = [];
|
|
37
|
+
let visibleIndex = 0;
|
|
38
|
+
for (const s of scripts) {
|
|
39
|
+
if (s.type?.toLowerCase() !== "application/ld+json") continue;
|
|
40
|
+
const raw = s.content?.trim() ?? "";
|
|
41
|
+
if (raw.length === 0) {
|
|
42
|
+
blocks.push({
|
|
43
|
+
index: visibleIndex++,
|
|
44
|
+
raw,
|
|
45
|
+
data: null,
|
|
46
|
+
parseError: "empty",
|
|
47
|
+
types: []
|
|
48
|
+
});
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
const decoded = decodeHtmlEntities(raw);
|
|
52
|
+
let parsed;
|
|
53
|
+
let parseError;
|
|
54
|
+
try {
|
|
55
|
+
parsed = JSON.parse(decoded);
|
|
56
|
+
} catch (err) {
|
|
57
|
+
parsed = null;
|
|
58
|
+
parseError = err instanceof Error ? err.message : String(err);
|
|
59
|
+
}
|
|
60
|
+
blocks.push({
|
|
61
|
+
index: visibleIndex++,
|
|
62
|
+
raw,
|
|
63
|
+
data: parsed,
|
|
64
|
+
parseError,
|
|
65
|
+
types: parsed === null ? [] : extractTypes(parsed)
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
return blocks;
|
|
69
|
+
}
|
|
70
|
+
function extractTypes(node) {
|
|
71
|
+
const out = /* @__PURE__ */ new Set();
|
|
72
|
+
visit(node, out);
|
|
73
|
+
return [...out];
|
|
74
|
+
}
|
|
75
|
+
function visit(node, out) {
|
|
76
|
+
if (node === null || node === void 0) return;
|
|
77
|
+
if (Array.isArray(node)) {
|
|
78
|
+
for (const item of node) visit(item, out);
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
if (typeof node !== "object") return;
|
|
82
|
+
const obj = node;
|
|
83
|
+
const t = obj["@type"];
|
|
84
|
+
if (typeof t === "string") {
|
|
85
|
+
out.add(t);
|
|
86
|
+
} else if (Array.isArray(t)) {
|
|
87
|
+
for (const v of t) {
|
|
88
|
+
if (typeof v === "string") out.add(v);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const graph = obj["@graph"];
|
|
92
|
+
if (Array.isArray(graph)) {
|
|
93
|
+
for (const item of graph) visit(item, out);
|
|
94
|
+
}
|
|
95
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
96
|
+
if (k === "@type" || k === "@graph") continue;
|
|
97
|
+
if (v && typeof v === "object") visit(v, out);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
function decodeHtmlEntities(s) {
|
|
101
|
+
return s.replaceAll(""", '"').replaceAll("'", "'").replaceAll("'", "'").replaceAll("'", "'").replaceAll("<", "<").replaceAll(">", ">").replaceAll(" ", " ").replaceAll("&", "&");
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// src/lib/core/extract/static.ts
|
|
105
|
+
function extractStatic(html, options) {
|
|
106
|
+
const $ = cheerio.load(html);
|
|
107
|
+
const base = resolveBase($, options.baseUrl);
|
|
108
|
+
const raw = extractRaw($);
|
|
109
|
+
const meta = parseGenericMeta(raw, base);
|
|
110
|
+
const openGraph = parseOpenGraph(raw);
|
|
111
|
+
const twitter = parseTwitter(raw);
|
|
112
|
+
const links = parseLinks(raw, base);
|
|
113
|
+
const jsonLd = parseJsonLdScripts(raw.scripts);
|
|
114
|
+
return {
|
|
115
|
+
schemaVersion: PAGE_SCHEMA_VERSION,
|
|
116
|
+
raw,
|
|
117
|
+
meta,
|
|
118
|
+
openGraph,
|
|
119
|
+
twitter,
|
|
120
|
+
links,
|
|
121
|
+
jsonLd
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
function extractRaw($) {
|
|
125
|
+
const html = $("html").first();
|
|
126
|
+
const baseEl = $("head base").first();
|
|
127
|
+
const head = {
|
|
128
|
+
title: pickTitle($),
|
|
129
|
+
htmlLang: attr(html, "lang"),
|
|
130
|
+
htmlDir: attr(html, "dir"),
|
|
131
|
+
baseHref: attr(baseEl, "href"),
|
|
132
|
+
metas: [],
|
|
133
|
+
links: [],
|
|
134
|
+
scripts: []
|
|
135
|
+
};
|
|
136
|
+
$("meta").each((_, el) => {
|
|
137
|
+
head.metas.push(toRawMeta($, el));
|
|
138
|
+
});
|
|
139
|
+
$("link").each((_, el) => {
|
|
140
|
+
head.links.push(toRawLink($, el));
|
|
141
|
+
});
|
|
142
|
+
$('script[type="application/ld+json"]').each((_, el) => {
|
|
143
|
+
head.scripts.push(toRawScript($, el));
|
|
144
|
+
});
|
|
145
|
+
return head;
|
|
146
|
+
}
|
|
147
|
+
function pickTitle($) {
|
|
148
|
+
const headTitle = trim($("head title").first().text());
|
|
149
|
+
if (headTitle) return headTitle;
|
|
150
|
+
let found;
|
|
151
|
+
$("title").each((_, el) => {
|
|
152
|
+
if (found) return;
|
|
153
|
+
if ($(el).parents("svg").length > 0) return;
|
|
154
|
+
const text = trim($(el).text());
|
|
155
|
+
if (text) found = text;
|
|
156
|
+
});
|
|
157
|
+
return found || void 0;
|
|
158
|
+
}
|
|
159
|
+
function toRawMeta($, el) {
|
|
160
|
+
const attributes = collectAttributes($, el);
|
|
161
|
+
return {
|
|
162
|
+
name: attributes.name,
|
|
163
|
+
property: attributes.property,
|
|
164
|
+
httpEquiv: attributes["http-equiv"],
|
|
165
|
+
content: attributes.content,
|
|
166
|
+
charset: attributes.charset,
|
|
167
|
+
attributes
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
function toRawLink($, el) {
|
|
171
|
+
const attributes = collectAttributes($, el);
|
|
172
|
+
return {
|
|
173
|
+
rel: attributes.rel ?? "",
|
|
174
|
+
href: attributes.href,
|
|
175
|
+
hreflang: attributes.hreflang,
|
|
176
|
+
type: attributes.type,
|
|
177
|
+
sizes: attributes.sizes,
|
|
178
|
+
media: attributes.media,
|
|
179
|
+
crossorigin: attributes.crossorigin,
|
|
180
|
+
attributes
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
function toRawScript($, el) {
|
|
184
|
+
const attributes = collectAttributes($, el);
|
|
185
|
+
return {
|
|
186
|
+
type: attributes.type,
|
|
187
|
+
src: attributes.src,
|
|
188
|
+
content: trim($(el).text()) || void 0,
|
|
189
|
+
attributes
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
function parseGenericMeta(raw, base) {
|
|
193
|
+
const meta = {};
|
|
194
|
+
if (raw.title) {
|
|
195
|
+
meta.title = sourced(raw.title, { kind: "title" }, raw.title);
|
|
196
|
+
}
|
|
197
|
+
for (const m of raw.metas) {
|
|
198
|
+
const name = m.name?.toLowerCase();
|
|
199
|
+
if (name === "description" && m.content !== void 0) {
|
|
200
|
+
meta.description = sourcedMeta(m.content, m);
|
|
201
|
+
} else if (name === "keywords" && m.content !== void 0) {
|
|
202
|
+
const list = m.content.split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
203
|
+
meta.keywords = sourcedMeta(list, m);
|
|
204
|
+
} else if (name === "author" && m.content !== void 0) {
|
|
205
|
+
meta.author = sourcedMeta(m.content, m);
|
|
206
|
+
} else if (name === "theme-color" && m.content !== void 0) {
|
|
207
|
+
meta.themeColor = sourcedMeta(m.content, m);
|
|
208
|
+
} else if (name === "color-scheme" && m.content !== void 0) {
|
|
209
|
+
meta.colorScheme = sourcedMeta(m.content, m);
|
|
210
|
+
} else if (name === "viewport" && m.content !== void 0) {
|
|
211
|
+
meta.viewport = sourcedMeta(m.content, m);
|
|
212
|
+
} else if (name === "robots" && m.content !== void 0) {
|
|
213
|
+
meta.robots = sourcedMeta(m.content, m);
|
|
214
|
+
} else if (name === "googlebot" && m.content !== void 0) {
|
|
215
|
+
meta.googlebot = sourcedMeta(m.content, m);
|
|
216
|
+
} else if (name === "generator" && m.content !== void 0) {
|
|
217
|
+
meta.generator = sourcedMeta(m.content, m);
|
|
218
|
+
} else if (name === "referrer" && m.content !== void 0) {
|
|
219
|
+
meta.referrer = sourcedMeta(m.content, m);
|
|
220
|
+
} else if (name === "application-name" && m.content !== void 0) {
|
|
221
|
+
meta.applicationName = sourcedMeta(m.content, m);
|
|
222
|
+
} else if (m.charset !== void 0) {
|
|
223
|
+
meta.charset = sourced(m.charset, { kind: "meta", name: "charset" }, m.charset);
|
|
224
|
+
} else if (m.httpEquiv?.toLowerCase() === "content-type" && m.content !== void 0) {
|
|
225
|
+
const charset = /charset=([^;]+)/i.exec(m.content)?.[1]?.trim();
|
|
226
|
+
if (charset && !meta.charset) {
|
|
227
|
+
meta.charset = sourced(charset, { kind: "meta", httpEquiv: "content-type" }, m.content);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
for (const l of raw.links) {
|
|
232
|
+
const rel = l.rel?.toLowerCase();
|
|
233
|
+
if (rel === "canonical" && l.href) {
|
|
234
|
+
const resolved = safeResolve(l.href, base);
|
|
235
|
+
meta.canonical = sourced(resolved, { kind: "link", rel: "canonical" }, l.href);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return meta;
|
|
239
|
+
}
|
|
240
|
+
function parseOpenGraph(raw) {
|
|
241
|
+
const og = {
|
|
242
|
+
localeAlternates: [],
|
|
243
|
+
images: [],
|
|
244
|
+
unknown: []
|
|
245
|
+
};
|
|
246
|
+
let current;
|
|
247
|
+
for (const m of raw.metas) {
|
|
248
|
+
const property = m.property?.toLowerCase();
|
|
249
|
+
if (!property?.startsWith("og:") || m.content === void 0) continue;
|
|
250
|
+
const value = m.content;
|
|
251
|
+
switch (property) {
|
|
252
|
+
case "og:title":
|
|
253
|
+
og.title = sourcedMeta(value, m);
|
|
254
|
+
break;
|
|
255
|
+
case "og:type":
|
|
256
|
+
og.type = sourcedMeta(value, m);
|
|
257
|
+
break;
|
|
258
|
+
case "og:url":
|
|
259
|
+
og.url = sourcedMeta(value, m);
|
|
260
|
+
break;
|
|
261
|
+
case "og:description":
|
|
262
|
+
og.description = sourcedMeta(value, m);
|
|
263
|
+
break;
|
|
264
|
+
case "og:site_name":
|
|
265
|
+
og.siteName = sourcedMeta(value, m);
|
|
266
|
+
break;
|
|
267
|
+
case "og:locale":
|
|
268
|
+
og.locale = sourcedMeta(value, m);
|
|
269
|
+
break;
|
|
270
|
+
case "og:locale:alternate":
|
|
271
|
+
og.localeAlternates.push(sourcedMeta(value, m));
|
|
272
|
+
break;
|
|
273
|
+
case "og:image":
|
|
274
|
+
case "og:image:url":
|
|
275
|
+
current = { url: sourcedMeta(value, m) };
|
|
276
|
+
og.images.push(current);
|
|
277
|
+
break;
|
|
278
|
+
case "og:image:secure_url":
|
|
279
|
+
if (current) current.secureUrl = sourcedMeta(value, m);
|
|
280
|
+
break;
|
|
281
|
+
case "og:image:type":
|
|
282
|
+
if (current) current.type = sourcedMeta(value, m);
|
|
283
|
+
break;
|
|
284
|
+
case "og:image:width": {
|
|
285
|
+
const n = Number.parseInt(value, 10);
|
|
286
|
+
if (current && Number.isFinite(n)) current.width = sourcedMeta(n, m);
|
|
287
|
+
break;
|
|
288
|
+
}
|
|
289
|
+
case "og:image:height": {
|
|
290
|
+
const n = Number.parseInt(value, 10);
|
|
291
|
+
if (current && Number.isFinite(n)) current.height = sourcedMeta(n, m);
|
|
292
|
+
break;
|
|
293
|
+
}
|
|
294
|
+
case "og:image:alt":
|
|
295
|
+
if (current) current.alt = sourcedMeta(value, m);
|
|
296
|
+
break;
|
|
297
|
+
default:
|
|
298
|
+
og.unknown.push({ property, value: sourcedMeta(value, m) });
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
return og;
|
|
302
|
+
}
|
|
303
|
+
function parseTwitter(raw) {
|
|
304
|
+
const tw = {};
|
|
305
|
+
for (const m of raw.metas) {
|
|
306
|
+
const name = m.name?.toLowerCase();
|
|
307
|
+
if (!name?.startsWith("twitter:") || m.content === void 0) continue;
|
|
308
|
+
switch (name) {
|
|
309
|
+
case "twitter:card":
|
|
310
|
+
tw.card = sourcedMeta(m.content, m);
|
|
311
|
+
break;
|
|
312
|
+
case "twitter:site":
|
|
313
|
+
tw.site = sourcedMeta(m.content, m);
|
|
314
|
+
break;
|
|
315
|
+
case "twitter:creator":
|
|
316
|
+
tw.creator = sourcedMeta(m.content, m);
|
|
317
|
+
break;
|
|
318
|
+
case "twitter:title":
|
|
319
|
+
tw.title = sourcedMeta(m.content, m);
|
|
320
|
+
break;
|
|
321
|
+
case "twitter:description":
|
|
322
|
+
tw.description = sourcedMeta(m.content, m);
|
|
323
|
+
break;
|
|
324
|
+
case "twitter:image":
|
|
325
|
+
case "twitter:image:src":
|
|
326
|
+
tw.image = sourcedMeta(m.content, m);
|
|
327
|
+
break;
|
|
328
|
+
case "twitter:image:alt":
|
|
329
|
+
tw.imageAlt = sourcedMeta(m.content, m);
|
|
330
|
+
break;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
return tw;
|
|
334
|
+
}
|
|
335
|
+
function parseLinks(raw, base) {
|
|
336
|
+
const out = {
|
|
337
|
+
alternates: [],
|
|
338
|
+
icons: [],
|
|
339
|
+
feeds: [],
|
|
340
|
+
preconnects: [],
|
|
341
|
+
dnsPrefetches: []
|
|
342
|
+
};
|
|
343
|
+
for (const l of raw.links) {
|
|
344
|
+
const rel = l.rel.toLowerCase();
|
|
345
|
+
if (rel === "canonical" && l.href) {
|
|
346
|
+
out.canonical = safeResolve(l.href, base);
|
|
347
|
+
} else if (rel === "alternate" && l.hreflang && l.href) {
|
|
348
|
+
const alt = {
|
|
349
|
+
hreflang: l.hreflang,
|
|
350
|
+
href: safeResolve(l.href, base),
|
|
351
|
+
isXDefault: l.hreflang.toLowerCase() === "x-default"
|
|
352
|
+
};
|
|
353
|
+
out.alternates.push(alt);
|
|
354
|
+
} else if (rel === "alternate" && l.type && l.href && isFeedType(l.type)) {
|
|
355
|
+
out.feeds.push({
|
|
356
|
+
rel: "alternate",
|
|
357
|
+
type: l.type.toLowerCase(),
|
|
358
|
+
href: safeResolve(l.href, base),
|
|
359
|
+
title: l.attributes.title
|
|
360
|
+
});
|
|
361
|
+
} else if ((rel === "icon" || rel === "apple-touch-icon" || rel === "shortcut icon") && l.href) {
|
|
362
|
+
const icon = {
|
|
363
|
+
rel,
|
|
364
|
+
href: safeResolve(l.href, base),
|
|
365
|
+
sizes: l.sizes,
|
|
366
|
+
type: l.type,
|
|
367
|
+
parsedSizes: parseSizes(l.sizes)
|
|
368
|
+
};
|
|
369
|
+
out.icons.push(icon);
|
|
370
|
+
} else if (rel === "manifest" && l.href) {
|
|
371
|
+
out.manifest = { href: safeResolve(l.href, base), crossorigin: l.crossorigin };
|
|
372
|
+
} else if (rel === "preconnect" && l.href) {
|
|
373
|
+
out.preconnects.push(safeResolve(l.href, base));
|
|
374
|
+
} else if (rel === "dns-prefetch" && l.href) {
|
|
375
|
+
out.dnsPrefetches.push(safeResolve(l.href, base));
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return out;
|
|
379
|
+
}
|
|
380
|
+
function isFeedType(type) {
|
|
381
|
+
const t = type.toLowerCase();
|
|
382
|
+
return t === "application/rss+xml" || t === "application/atom+xml" || t === "application/feed+json";
|
|
383
|
+
}
|
|
384
|
+
function parseSizes(sizes) {
|
|
385
|
+
if (!sizes) return [];
|
|
386
|
+
return sizes.split(/\s+/).filter(Boolean).map((token) => {
|
|
387
|
+
if (token.toLowerCase() === "any") return "any";
|
|
388
|
+
const m = /^(\d+)x(\d+)$/i.exec(token);
|
|
389
|
+
if (!m) return void 0;
|
|
390
|
+
const w = Number.parseInt(m[1], 10);
|
|
391
|
+
const h = Number.parseInt(m[2], 10);
|
|
392
|
+
if (!Number.isFinite(w) || !Number.isFinite(h)) return void 0;
|
|
393
|
+
return { width: w, height: h };
|
|
394
|
+
}).filter((v) => v !== void 0);
|
|
395
|
+
}
|
|
396
|
+
function collectAttributes(_$, el) {
|
|
397
|
+
const result = {};
|
|
398
|
+
const attribs = el.attribs;
|
|
399
|
+
if (attribs) {
|
|
400
|
+
for (const [k, v] of Object.entries(attribs)) {
|
|
401
|
+
result[k.toLowerCase()] = v;
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
return result;
|
|
405
|
+
}
|
|
406
|
+
function attr(el, name) {
|
|
407
|
+
const v = el.attr(name);
|
|
408
|
+
return v === void 0 || v === "" ? void 0 : v;
|
|
409
|
+
}
|
|
410
|
+
function trim(s) {
|
|
411
|
+
return (s ?? "").replace(/\s+/g, " ").trim();
|
|
412
|
+
}
|
|
413
|
+
function resolveBase($, requestUrl) {
|
|
414
|
+
let base;
|
|
415
|
+
try {
|
|
416
|
+
base = new URL(requestUrl);
|
|
417
|
+
} catch {
|
|
418
|
+
base = new URL("http://invalid.local/");
|
|
419
|
+
}
|
|
420
|
+
const baseHref = $("head base").first().attr("href");
|
|
421
|
+
if (baseHref) {
|
|
422
|
+
try {
|
|
423
|
+
base = new URL(baseHref, base);
|
|
424
|
+
} catch {
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
return base;
|
|
428
|
+
}
|
|
429
|
+
function safeResolve(href, base) {
|
|
430
|
+
try {
|
|
431
|
+
return new URL(href, base).toString();
|
|
432
|
+
} catch {
|
|
433
|
+
return href;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
function sourced(value, origin, raw) {
|
|
437
|
+
return { value, origin, raw };
|
|
438
|
+
}
|
|
439
|
+
function sourcedMeta(value, m) {
|
|
440
|
+
const origin = m.property ? { kind: "meta", property: m.property } : m.name ? { kind: "meta", name: m.name } : m.httpEquiv ? { kind: "meta", httpEquiv: m.httpEquiv } : { kind: "meta" };
|
|
441
|
+
return { value, origin, raw: m.content };
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// src/lib/core/fetch/static.ts
|
|
445
|
+
var DEFAULT_UA = "Goflag/0.0 (+https://github.com/tancredesimonin-indie/goflag)";
|
|
446
|
+
var FetchError = class extends Error {
|
|
447
|
+
url;
|
|
448
|
+
cause;
|
|
449
|
+
constructor(message, url, options) {
|
|
450
|
+
super(message);
|
|
451
|
+
this.name = "FetchError";
|
|
452
|
+
this.url = url;
|
|
453
|
+
this.cause = options?.cause;
|
|
454
|
+
}
|
|
455
|
+
};
|
|
456
|
+
async function fetchStatic(url, options = {}) {
|
|
457
|
+
const timeoutMs = options.timeoutMs ?? 15e3;
|
|
458
|
+
const maxRedirects = options.maxRedirects ?? 10;
|
|
459
|
+
const userAgent = options.userAgent ?? DEFAULT_UA;
|
|
460
|
+
let parsed;
|
|
461
|
+
try {
|
|
462
|
+
parsed = new URL(url);
|
|
463
|
+
} catch {
|
|
464
|
+
throw new FetchError(`Invalid URL: ${url}`, url);
|
|
465
|
+
}
|
|
466
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
467
|
+
throw new FetchError(
|
|
468
|
+
`Unsupported protocol "${parsed.protocol}" \u2014 only http(s) is supported`,
|
|
469
|
+
url
|
|
470
|
+
);
|
|
471
|
+
}
|
|
472
|
+
const abort = new AbortController();
|
|
473
|
+
const onCallerAbort = () => abort.abort(options.signal?.reason);
|
|
474
|
+
options.signal?.addEventListener("abort", onCallerAbort, { once: true });
|
|
475
|
+
const timeout = setTimeout(
|
|
476
|
+
() => abort.abort(new FetchError("Request timed out", url)),
|
|
477
|
+
timeoutMs
|
|
478
|
+
);
|
|
479
|
+
const restoreTls = relaxTlsIfRequested(options.allowInsecureTls);
|
|
480
|
+
const start = performance.now();
|
|
481
|
+
try {
|
|
482
|
+
let currentUrl = parsed.toString();
|
|
483
|
+
let redirects = 0;
|
|
484
|
+
let response;
|
|
485
|
+
while (true) {
|
|
486
|
+
try {
|
|
487
|
+
response = await fetch(currentUrl, {
|
|
488
|
+
method: "GET",
|
|
489
|
+
redirect: "manual",
|
|
490
|
+
headers: {
|
|
491
|
+
"user-agent": userAgent,
|
|
492
|
+
accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5"
|
|
493
|
+
},
|
|
494
|
+
signal: abort.signal
|
|
495
|
+
});
|
|
496
|
+
} catch (err) {
|
|
497
|
+
throw new FetchError(`Network error fetching ${currentUrl}: ${describe(err)}`, url, {
|
|
498
|
+
cause: err
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
if (isRedirect(response.status)) {
|
|
502
|
+
const location = response.headers.get("location");
|
|
503
|
+
if (!location) {
|
|
504
|
+
throw new FetchError(
|
|
505
|
+
`Redirect ${response.status} from ${currentUrl} missing Location header`,
|
|
506
|
+
url
|
|
507
|
+
);
|
|
508
|
+
}
|
|
509
|
+
if (redirects >= maxRedirects) {
|
|
510
|
+
throw new FetchError(
|
|
511
|
+
`Exceeded maximum of ${maxRedirects} redirects (last hop: ${currentUrl})`,
|
|
512
|
+
url
|
|
513
|
+
);
|
|
514
|
+
}
|
|
515
|
+
await response.body?.cancel().catch(() => void 0);
|
|
516
|
+
currentUrl = new URL(location, currentUrl).toString();
|
|
517
|
+
redirects += 1;
|
|
518
|
+
continue;
|
|
519
|
+
}
|
|
520
|
+
const body = await response.text();
|
|
521
|
+
const headers = headersToObject(response.headers);
|
|
522
|
+
const contentType = stripContentTypeParams(headers["content-type"]);
|
|
523
|
+
const meta = {
|
|
524
|
+
requestedUrl: parsed.toString(),
|
|
525
|
+
finalUrl: currentUrl,
|
|
526
|
+
status: response.status,
|
|
527
|
+
statusText: response.statusText,
|
|
528
|
+
headers,
|
|
529
|
+
redirectCount: redirects,
|
|
530
|
+
durationMs: Math.round(performance.now() - start),
|
|
531
|
+
bodyBytes: Buffer.byteLength(body, "utf8"),
|
|
532
|
+
contentType
|
|
533
|
+
};
|
|
534
|
+
return { meta, body };
|
|
535
|
+
}
|
|
536
|
+
} finally {
|
|
537
|
+
clearTimeout(timeout);
|
|
538
|
+
options.signal?.removeEventListener("abort", onCallerAbort);
|
|
539
|
+
restoreTls();
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
function isRedirect(status) {
|
|
543
|
+
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
|
|
544
|
+
}
|
|
545
|
+
function headersToObject(headers) {
|
|
546
|
+
const out = {};
|
|
547
|
+
headers.forEach((v, k) => {
|
|
548
|
+
out[k.toLowerCase()] = v;
|
|
549
|
+
});
|
|
550
|
+
return out;
|
|
551
|
+
}
|
|
552
|
+
function stripContentTypeParams(value) {
|
|
553
|
+
if (!value) return void 0;
|
|
554
|
+
const semi = value.indexOf(";");
|
|
555
|
+
return (semi === -1 ? value : value.slice(0, semi)).trim().toLowerCase();
|
|
556
|
+
}
|
|
557
|
+
function relaxTlsIfRequested(enabled) {
|
|
558
|
+
if (!enabled) return () => void 0;
|
|
559
|
+
const previous = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
|
|
560
|
+
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
|
|
561
|
+
return () => {
|
|
562
|
+
if (previous === void 0) {
|
|
563
|
+
delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
|
|
564
|
+
} else {
|
|
565
|
+
process.env.NODE_TLS_REJECT_UNAUTHORIZED = previous;
|
|
566
|
+
}
|
|
567
|
+
};
|
|
568
|
+
}
|
|
569
|
+
function describe(err) {
|
|
570
|
+
if (err instanceof Error) return err.message;
|
|
571
|
+
return String(err);
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
// src/lib/core/extract/headless.ts
|
|
575
|
+
var DEFAULT_UA2 = "Goflag/0.0 (+https://github.com/tancredesimonin-indie/goflag; headless)";
|
|
576
|
+
var HeadlessUnavailableError = class extends Error {
|
|
577
|
+
cause;
|
|
578
|
+
constructor(message, options) {
|
|
579
|
+
super(message);
|
|
580
|
+
this.name = "HeadlessUnavailableError";
|
|
581
|
+
this.cause = options?.cause;
|
|
582
|
+
}
|
|
583
|
+
};
|
|
584
|
+
async function extractHeadless(url, options = {}) {
|
|
585
|
+
const launcher = options.launcher ?? defaultLauncher();
|
|
586
|
+
const timeoutMs = options.timeoutMs ?? 3e4;
|
|
587
|
+
const userAgent = options.userAgent ?? DEFAULT_UA2;
|
|
588
|
+
const waitUntil = options.waitUntil ?? "networkidle";
|
|
589
|
+
if (options.signal?.aborted) {
|
|
590
|
+
throw new HeadlessUnavailableError("Headless render cancelled before launch");
|
|
591
|
+
}
|
|
592
|
+
const start = performance.now();
|
|
593
|
+
const browser = await launcher.launch({ allowInsecureTls: options.allowInsecureTls });
|
|
594
|
+
let page;
|
|
595
|
+
try {
|
|
596
|
+
page = await browser.newPage({ userAgent });
|
|
597
|
+
const nav = await page.goto(url, { waitUntil, timeoutMs });
|
|
598
|
+
const body = await page.content();
|
|
599
|
+
const meta = {
|
|
600
|
+
requestedUrl: url,
|
|
601
|
+
finalUrl: nav.finalUrl,
|
|
602
|
+
status: nav.status,
|
|
603
|
+
statusText: nav.statusText,
|
|
604
|
+
headers: nav.headers,
|
|
605
|
+
redirectCount: 0,
|
|
606
|
+
durationMs: Math.round(performance.now() - start),
|
|
607
|
+
bodyBytes: Buffer.byteLength(body, "utf8"),
|
|
608
|
+
contentType: nav.headers["content-type"]?.split(";")[0]?.trim().toLowerCase()
|
|
609
|
+
};
|
|
610
|
+
return { renderedHtml: body, fetch: meta };
|
|
611
|
+
} finally {
|
|
612
|
+
await page?.close().catch(() => void 0);
|
|
613
|
+
await browser.close().catch(() => void 0);
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
function defaultLauncher(loader = () => import("playwright")) {
|
|
617
|
+
return {
|
|
618
|
+
async launch({ allowInsecureTls }) {
|
|
619
|
+
let mod;
|
|
620
|
+
try {
|
|
621
|
+
mod = await loader();
|
|
622
|
+
} catch (err) {
|
|
623
|
+
throw new HeadlessUnavailableError(
|
|
624
|
+
"Headless mode requires the 'playwright' package. Install it with `pnpm add -D playwright` and `pnpm exec playwright install chromium`.",
|
|
625
|
+
{ cause: err }
|
|
626
|
+
);
|
|
627
|
+
}
|
|
628
|
+
let pw;
|
|
629
|
+
try {
|
|
630
|
+
pw = await mod.chromium.launch({
|
|
631
|
+
headless: true,
|
|
632
|
+
args: allowInsecureTls ? ["--ignore-certificate-errors"] : []
|
|
633
|
+
});
|
|
634
|
+
} catch (err) {
|
|
635
|
+
throw new HeadlessUnavailableError(
|
|
636
|
+
"Could not launch Chromium. Install the browser binary with `pnpm exec playwright install chromium`.",
|
|
637
|
+
{ cause: err }
|
|
638
|
+
);
|
|
639
|
+
}
|
|
640
|
+
return {
|
|
641
|
+
async newPage({ userAgent }) {
|
|
642
|
+
const ctx = await pw.newContext({
|
|
643
|
+
userAgent,
|
|
644
|
+
ignoreHTTPSErrors: !!allowInsecureTls
|
|
645
|
+
});
|
|
646
|
+
const p = await ctx.newPage();
|
|
647
|
+
return {
|
|
648
|
+
async goto(url, opts) {
|
|
649
|
+
const response = await p.goto(url, {
|
|
650
|
+
waitUntil: opts.waitUntil,
|
|
651
|
+
timeout: opts.timeoutMs
|
|
652
|
+
});
|
|
653
|
+
const status = response?.status() ?? 0;
|
|
654
|
+
const statusText = response?.statusText() ?? "";
|
|
655
|
+
const headers = response ? await response.allHeaders() : {};
|
|
656
|
+
return {
|
|
657
|
+
status,
|
|
658
|
+
statusText,
|
|
659
|
+
finalUrl: p.url(),
|
|
660
|
+
headers
|
|
661
|
+
};
|
|
662
|
+
},
|
|
663
|
+
async content() {
|
|
664
|
+
return p.content();
|
|
665
|
+
},
|
|
666
|
+
async close() {
|
|
667
|
+
await ctx.close();
|
|
668
|
+
}
|
|
669
|
+
};
|
|
670
|
+
},
|
|
671
|
+
async close() {
|
|
672
|
+
await pw.close();
|
|
673
|
+
}
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// src/lib/core/extract/heuristics.ts
|
|
680
|
+
var PLACEHOLDER_TITLES = /* @__PURE__ */ new Set([
|
|
681
|
+
"react app",
|
|
682
|
+
"vue app",
|
|
683
|
+
"vite app",
|
|
684
|
+
"create react app",
|
|
685
|
+
"next.js",
|
|
686
|
+
"nuxt",
|
|
687
|
+
"angular",
|
|
688
|
+
"ember",
|
|
689
|
+
"webpack app",
|
|
690
|
+
"document",
|
|
691
|
+
"untitled",
|
|
692
|
+
""
|
|
693
|
+
]);
|
|
694
|
+
function looksClientRendered(partial) {
|
|
695
|
+
const reasons = [];
|
|
696
|
+
const titleRaw = partial.raw.title?.trim().toLowerCase() ?? "";
|
|
697
|
+
const titleMissing2 = titleRaw === "";
|
|
698
|
+
const titlePlaceholder = !titleMissing2 && PLACEHOLDER_TITLES.has(titleRaw);
|
|
699
|
+
if (titleMissing2) reasons.push("title missing");
|
|
700
|
+
else if (titlePlaceholder) reasons.push(`placeholder title "${partial.raw.title}"`);
|
|
701
|
+
const descMissing = partial.meta.description === void 0;
|
|
702
|
+
if (descMissing) reasons.push("no description");
|
|
703
|
+
const canonicalMissing2 = partial.meta.canonical === void 0;
|
|
704
|
+
if (canonicalMissing2) reasons.push("no canonical");
|
|
705
|
+
const ogMissing = partial.openGraph.title === void 0 && partial.openGraph.description === void 0 && partial.openGraph.images.length === 0 && partial.openGraph.type === void 0 && partial.openGraph.url === void 0 && partial.openGraph.siteName === void 0 && partial.openGraph.unknown.length === 0;
|
|
706
|
+
if (ogMissing) reasons.push("no og:*");
|
|
707
|
+
const twitterMissing = partial.twitter.card === void 0 && partial.twitter.title === void 0 && partial.twitter.image === void 0;
|
|
708
|
+
if (twitterMissing) reasons.push("no twitter:*");
|
|
709
|
+
const jsonLdMissing = partial.jsonLd.length === 0;
|
|
710
|
+
if (jsonLdMissing) reasons.push("no JSON-LD");
|
|
711
|
+
const hreflangMissing2 = partial.links.alternates.length === 0;
|
|
712
|
+
if (hreflangMissing2) reasons.push("no hreflang");
|
|
713
|
+
const likely = (titleMissing2 || titlePlaceholder) && descMissing && canonicalMissing2 && ogMissing && twitterMissing && jsonLdMissing && hreflangMissing2;
|
|
714
|
+
return {
|
|
715
|
+
likely,
|
|
716
|
+
reason: reasons.join(", ") || "all metadata present"
|
|
717
|
+
};
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
// src/lib/core/extract/hydration.ts
|
|
721
|
+
function computeHydrationDelta(staticPass, renderedPass) {
|
|
722
|
+
const staticMetaKeys = new Set(staticPass.raw.metas.map(metaKey));
|
|
723
|
+
const renderedMetaKeys = new Set(renderedPass.raw.metas.map(metaKey));
|
|
724
|
+
const clientInjectedMetas = renderedPass.raw.metas.filter((m) => !staticMetaKeys.has(metaKey(m))).map(toMetaSummary);
|
|
725
|
+
const clientRemovedMetas = staticPass.raw.metas.filter((m) => !renderedMetaKeys.has(metaKey(m))).map(toMetaSummary);
|
|
726
|
+
const staticLinkKeys = new Set(staticPass.raw.links.map(linkKey));
|
|
727
|
+
const renderedLinkKeys = new Set(renderedPass.raw.links.map(linkKey));
|
|
728
|
+
const clientInjectedLinks = renderedPass.raw.links.filter((l) => !staticLinkKeys.has(linkKey(l))).map(toLinkSummary);
|
|
729
|
+
const clientRemovedLinks = staticPass.raw.links.filter((l) => !renderedLinkKeys.has(linkKey(l))).map(toLinkSummary);
|
|
730
|
+
return {
|
|
731
|
+
fromMode: "static",
|
|
732
|
+
toMode: "headless",
|
|
733
|
+
titleChanged: (staticPass.raw.title ?? "") !== (renderedPass.raw.title ?? ""),
|
|
734
|
+
htmlLangChanged: (staticPass.raw.htmlLang ?? "") !== (renderedPass.raw.htmlLang ?? ""),
|
|
735
|
+
clientInjectedMetas,
|
|
736
|
+
clientRemovedMetas,
|
|
737
|
+
clientInjectedLinks,
|
|
738
|
+
clientRemovedLinks,
|
|
739
|
+
jsonLdBlocksAdded: Math.max(0, renderedPass.jsonLd.length - staticPass.jsonLd.length)
|
|
740
|
+
};
|
|
741
|
+
}
|
|
742
|
+
function metaKey(m) {
|
|
743
|
+
return [
|
|
744
|
+
"n=" + (m.name ?? ""),
|
|
745
|
+
"p=" + (m.property ?? ""),
|
|
746
|
+
"he=" + (m.httpEquiv ?? ""),
|
|
747
|
+
"ch=" + (m.charset ?? ""),
|
|
748
|
+
"c=" + (m.content ?? "")
|
|
749
|
+
].join("|");
|
|
750
|
+
}
|
|
751
|
+
function linkKey(l) {
|
|
752
|
+
return [
|
|
753
|
+
"r=" + (l.rel ?? ""),
|
|
754
|
+
"h=" + (l.href ?? ""),
|
|
755
|
+
"hl=" + (l.hreflang ?? ""),
|
|
756
|
+
"t=" + (l.type ?? ""),
|
|
757
|
+
"s=" + (l.sizes ?? "")
|
|
758
|
+
].join("|");
|
|
759
|
+
}
|
|
760
|
+
function toMetaSummary(m) {
|
|
761
|
+
const out = {};
|
|
762
|
+
if (m.name) out.name = m.name;
|
|
763
|
+
if (m.property) out.property = m.property;
|
|
764
|
+
if (m.httpEquiv) out.httpEquiv = m.httpEquiv;
|
|
765
|
+
if (m.content !== void 0) out.content = m.content;
|
|
766
|
+
return out;
|
|
767
|
+
}
|
|
768
|
+
function toLinkSummary(l) {
|
|
769
|
+
const out = { rel: l.rel };
|
|
770
|
+
if (l.href !== void 0) out.href = l.href;
|
|
771
|
+
if (l.hreflang !== void 0) out.hreflang = l.hreflang;
|
|
772
|
+
return out;
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
// src/lib/core/probes/abort.ts
|
|
776
|
+
function combineSignals(caller, timeoutMs) {
|
|
777
|
+
const controller = new AbortController();
|
|
778
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
779
|
+
if (!caller) {
|
|
780
|
+
return { signal: controller.signal, cleanup: () => clearTimeout(timer) };
|
|
781
|
+
}
|
|
782
|
+
if (caller.aborted) {
|
|
783
|
+
controller.abort();
|
|
784
|
+
return { signal: controller.signal, cleanup: () => clearTimeout(timer) };
|
|
785
|
+
}
|
|
786
|
+
const onAbort = () => controller.abort();
|
|
787
|
+
caller.addEventListener("abort", onAbort, { once: true });
|
|
788
|
+
return {
|
|
789
|
+
signal: controller.signal,
|
|
790
|
+
cleanup: () => {
|
|
791
|
+
clearTimeout(timer);
|
|
792
|
+
caller.removeEventListener("abort", onAbort);
|
|
793
|
+
}
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
// src/lib/core/probes/manifest.ts
|
|
798
|
+
async function probeManifest(href, options = {}) {
|
|
799
|
+
const { signal, cleanup } = combineSignals(options.signal, options.timeoutMs ?? 5e3);
|
|
800
|
+
try {
|
|
801
|
+
const res = await fetch(href, { signal, redirect: "follow" });
|
|
802
|
+
if (!res.ok) return { url: href, status: res.status, found: false };
|
|
803
|
+
const raw = await res.text();
|
|
804
|
+
let data;
|
|
805
|
+
let parseError;
|
|
806
|
+
try {
|
|
807
|
+
data = JSON.parse(raw);
|
|
808
|
+
} catch (err) {
|
|
809
|
+
parseError = err instanceof Error ? err.message : String(err);
|
|
810
|
+
}
|
|
811
|
+
return { url: href, status: res.status, found: true, raw, data, parseError };
|
|
812
|
+
} catch {
|
|
813
|
+
return { url: href, status: 0, found: false };
|
|
814
|
+
} finally {
|
|
815
|
+
cleanup();
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
// src/lib/core/probes/robots.ts
|
|
820
|
+
async function probeRobots(origin, options = {}) {
|
|
821
|
+
const url = new URL("/robots.txt", origin).toString();
|
|
822
|
+
const { signal, cleanup } = combineSignals(options.signal, options.timeoutMs ?? 5e3);
|
|
823
|
+
try {
|
|
824
|
+
const res = await fetch(url, { signal, redirect: "follow" });
|
|
825
|
+
if (!res.ok) {
|
|
826
|
+
return { url, status: res.status, found: false, sitemaps: [], blocksAll: false };
|
|
827
|
+
}
|
|
828
|
+
const body = await res.text();
|
|
829
|
+
return {
|
|
830
|
+
url,
|
|
831
|
+
status: res.status,
|
|
832
|
+
found: true,
|
|
833
|
+
raw: body,
|
|
834
|
+
sitemaps: extractSitemaps(body),
|
|
835
|
+
blocksAll: blocksAllUserAgents(body)
|
|
836
|
+
};
|
|
837
|
+
} catch {
|
|
838
|
+
return { url, status: 0, found: false, sitemaps: [], blocksAll: false };
|
|
839
|
+
} finally {
|
|
840
|
+
cleanup();
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
function extractSitemaps(body) {
|
|
844
|
+
const out = [];
|
|
845
|
+
for (const line of body.split(/\r?\n/)) {
|
|
846
|
+
const m = /^\s*Sitemap:\s*(\S+)\s*$/i.exec(line);
|
|
847
|
+
if (m?.[1]) out.push(m[1]);
|
|
848
|
+
}
|
|
849
|
+
return out;
|
|
850
|
+
}
|
|
851
|
+
function blocksAllUserAgents(body) {
|
|
852
|
+
let inWildcard = false;
|
|
853
|
+
for (const rawLine of body.split(/\r?\n/)) {
|
|
854
|
+
const line = rawLine.replace(/#.*$/, "").trim();
|
|
855
|
+
if (line.length === 0) continue;
|
|
856
|
+
const ua = /^User-agent:\s*(.+)$/i.exec(line);
|
|
857
|
+
if (ua) {
|
|
858
|
+
inWildcard = ua[1].trim() === "*";
|
|
859
|
+
continue;
|
|
860
|
+
}
|
|
861
|
+
if (!inWildcard) continue;
|
|
862
|
+
const dis = /^Disallow:\s*(.*)$/i.exec(line);
|
|
863
|
+
if (dis && dis[1].trim() === "/") return true;
|
|
864
|
+
}
|
|
865
|
+
return false;
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
// src/lib/core/probes/sitemap.ts
|
|
869
|
+
async function probeSitemap(origin, options = {}) {
|
|
870
|
+
const url = new URL("/sitemap.xml", origin).toString();
|
|
871
|
+
const { signal, cleanup } = combineSignals(options.signal, options.timeoutMs ?? 5e3);
|
|
872
|
+
try {
|
|
873
|
+
const res = await fetch(url, { signal, redirect: "follow" });
|
|
874
|
+
if (!res.ok) return { url, status: res.status, found: false, isIndex: false, entryCount: 0 };
|
|
875
|
+
const body = await res.text();
|
|
876
|
+
return {
|
|
877
|
+
url,
|
|
878
|
+
status: res.status,
|
|
879
|
+
found: true,
|
|
880
|
+
isIndex: /<sitemapindex\b/i.test(body),
|
|
881
|
+
entryCount: countEntries(body)
|
|
882
|
+
};
|
|
883
|
+
} catch {
|
|
884
|
+
return { url, status: 0, found: false, isIndex: false, entryCount: 0 };
|
|
885
|
+
} finally {
|
|
886
|
+
cleanup();
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
function countEntries(body) {
|
|
890
|
+
const url = body.match(/<url\b/gi)?.length ?? 0;
|
|
891
|
+
const sm = body.match(/<sitemap\b/gi)?.length ?? 0;
|
|
892
|
+
return url + sm;
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
// src/lib/core/inspect.ts
|
|
896
|
+
async function inspect(url, options = {}) {
|
|
897
|
+
const {
|
|
898
|
+
probes: enableProbes = true,
|
|
899
|
+
mode = "auto",
|
|
900
|
+
headless: headlessOpts,
|
|
901
|
+
...fetchOptions
|
|
902
|
+
} = options;
|
|
903
|
+
const fetchedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
904
|
+
if (mode === "headless") {
|
|
905
|
+
return inspectHeadlessOnly({
|
|
906
|
+
url,
|
|
907
|
+
fetchedAt,
|
|
908
|
+
enableProbes,
|
|
909
|
+
fetchOptions,
|
|
910
|
+
headlessOpts
|
|
911
|
+
});
|
|
912
|
+
}
|
|
913
|
+
const { meta: staticMeta, body: staticBody } = await fetchStatic(url, fetchOptions);
|
|
914
|
+
const staticParsed = extractStatic(staticBody, { baseUrl: staticMeta.finalUrl });
|
|
915
|
+
if (mode === "static") {
|
|
916
|
+
const probes2 = await runProbes({
|
|
917
|
+
enabled: enableProbes,
|
|
918
|
+
origin: new URL(staticMeta.finalUrl).origin,
|
|
919
|
+
manifestHref: staticParsed.links.manifest?.href,
|
|
920
|
+
signal: fetchOptions.signal
|
|
921
|
+
});
|
|
922
|
+
return {
|
|
923
|
+
...staticParsed,
|
|
924
|
+
fetchedAt,
|
|
925
|
+
fetch: staticMeta,
|
|
926
|
+
extractor: { mode: "static", escalated: false },
|
|
927
|
+
html: { static: staticBody },
|
|
928
|
+
probes: probes2
|
|
929
|
+
};
|
|
930
|
+
}
|
|
931
|
+
const heuristic = looksClientRendered(staticParsed);
|
|
932
|
+
if (!heuristic.likely) {
|
|
933
|
+
const probes2 = await runProbes({
|
|
934
|
+
enabled: enableProbes,
|
|
935
|
+
origin: new URL(staticMeta.finalUrl).origin,
|
|
936
|
+
manifestHref: staticParsed.links.manifest?.href,
|
|
937
|
+
signal: fetchOptions.signal
|
|
938
|
+
});
|
|
939
|
+
return {
|
|
940
|
+
...staticParsed,
|
|
941
|
+
fetchedAt,
|
|
942
|
+
fetch: staticMeta,
|
|
943
|
+
extractor: { mode: "static", escalated: false },
|
|
944
|
+
html: { static: staticBody },
|
|
945
|
+
probes: probes2
|
|
946
|
+
};
|
|
947
|
+
}
|
|
948
|
+
let renderedBody;
|
|
949
|
+
let renderedFetch;
|
|
950
|
+
try {
|
|
951
|
+
const result = await extractHeadless(url, {
|
|
952
|
+
...headlessOpts,
|
|
953
|
+
allowInsecureTls: fetchOptions.allowInsecureTls,
|
|
954
|
+
signal: fetchOptions.signal
|
|
955
|
+
});
|
|
956
|
+
renderedBody = result.renderedHtml;
|
|
957
|
+
renderedFetch = result.fetch;
|
|
958
|
+
} catch (err) {
|
|
959
|
+
if (!(err instanceof HeadlessUnavailableError)) throw err;
|
|
960
|
+
const probes2 = await runProbes({
|
|
961
|
+
enabled: enableProbes,
|
|
962
|
+
origin: new URL(staticMeta.finalUrl).origin,
|
|
963
|
+
manifestHref: staticParsed.links.manifest?.href,
|
|
964
|
+
signal: fetchOptions.signal
|
|
965
|
+
});
|
|
966
|
+
return {
|
|
967
|
+
...staticParsed,
|
|
968
|
+
fetchedAt,
|
|
969
|
+
fetch: staticMeta,
|
|
970
|
+
extractor: {
|
|
971
|
+
mode: "static",
|
|
972
|
+
escalated: false,
|
|
973
|
+
escalationReason: `would-have-escalated (${heuristic.reason}) but ${err.message}`
|
|
974
|
+
},
|
|
975
|
+
html: { static: staticBody },
|
|
976
|
+
probes: probes2
|
|
977
|
+
};
|
|
978
|
+
}
|
|
979
|
+
const renderedParsed = extractStatic(renderedBody, { baseUrl: renderedFetch.finalUrl });
|
|
980
|
+
const hydration = computeHydrationDelta(staticParsed, renderedParsed);
|
|
981
|
+
const probes = await runProbes({
|
|
982
|
+
enabled: enableProbes,
|
|
983
|
+
origin: new URL(renderedFetch.finalUrl).origin,
|
|
984
|
+
manifestHref: renderedParsed.links.manifest?.href,
|
|
985
|
+
signal: fetchOptions.signal
|
|
986
|
+
});
|
|
987
|
+
return {
|
|
988
|
+
...renderedParsed,
|
|
989
|
+
fetchedAt,
|
|
990
|
+
fetch: renderedFetch,
|
|
991
|
+
extractor: {
|
|
992
|
+
mode: "headless",
|
|
993
|
+
escalated: true,
|
|
994
|
+
escalationReason: heuristic.reason
|
|
995
|
+
},
|
|
996
|
+
html: { static: staticBody, rendered: renderedBody },
|
|
997
|
+
hydration,
|
|
998
|
+
probes
|
|
999
|
+
};
|
|
1000
|
+
}
|
|
1001
|
+
async function inspectHeadlessOnly(args) {
|
|
1002
|
+
const { url, fetchedAt, enableProbes, fetchOptions, headlessOpts } = args;
|
|
1003
|
+
const result = await extractHeadless(url, {
|
|
1004
|
+
...headlessOpts,
|
|
1005
|
+
allowInsecureTls: fetchOptions.allowInsecureTls,
|
|
1006
|
+
signal: fetchOptions.signal
|
|
1007
|
+
});
|
|
1008
|
+
const parsed = extractStatic(result.renderedHtml, { baseUrl: result.fetch.finalUrl });
|
|
1009
|
+
const probes = await runProbes({
|
|
1010
|
+
enabled: enableProbes,
|
|
1011
|
+
origin: new URL(result.fetch.finalUrl).origin,
|
|
1012
|
+
manifestHref: parsed.links.manifest?.href,
|
|
1013
|
+
signal: fetchOptions.signal
|
|
1014
|
+
});
|
|
1015
|
+
return {
|
|
1016
|
+
...parsed,
|
|
1017
|
+
fetchedAt,
|
|
1018
|
+
fetch: result.fetch,
|
|
1019
|
+
extractor: { mode: "headless", escalated: false },
|
|
1020
|
+
html: { static: "", rendered: result.renderedHtml },
|
|
1021
|
+
probes
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
async function runProbes(args) {
|
|
1025
|
+
if (!args.enabled) return {};
|
|
1026
|
+
const [robots, sitemap, manifest] = await Promise.all([
|
|
1027
|
+
probeRobots(args.origin, { signal: args.signal }),
|
|
1028
|
+
probeSitemap(args.origin, { signal: args.signal }),
|
|
1029
|
+
args.manifestHref ? probeManifest(args.manifestHref, { signal: args.signal }) : Promise.resolve(void 0)
|
|
1030
|
+
]);
|
|
1031
|
+
const probes = { robots, sitemap };
|
|
1032
|
+
if (manifest) probes.manifest = manifest;
|
|
1033
|
+
return probes;
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
// src/lib/core/glob.ts
|
|
1037
|
+
var cache = /* @__PURE__ */ new Map();
|
|
1038
|
+
function compileGlob(pattern) {
|
|
1039
|
+
const cached = cache.get(pattern);
|
|
1040
|
+
if (cached) return cached;
|
|
1041
|
+
let re = "^";
|
|
1042
|
+
let i = 0;
|
|
1043
|
+
while (i < pattern.length) {
|
|
1044
|
+
const c = pattern[i];
|
|
1045
|
+
if (c === "*") {
|
|
1046
|
+
if (pattern[i + 1] === "*") {
|
|
1047
|
+
re += ".*";
|
|
1048
|
+
i += 2;
|
|
1049
|
+
continue;
|
|
1050
|
+
}
|
|
1051
|
+
re += "[^/]*";
|
|
1052
|
+
i += 1;
|
|
1053
|
+
continue;
|
|
1054
|
+
}
|
|
1055
|
+
if (c === "?") {
|
|
1056
|
+
re += "[^/]";
|
|
1057
|
+
i += 1;
|
|
1058
|
+
continue;
|
|
1059
|
+
}
|
|
1060
|
+
if (/[.+^${}()|\\[\]/]/.test(c)) {
|
|
1061
|
+
re += `\\${c}`;
|
|
1062
|
+
} else {
|
|
1063
|
+
re += c;
|
|
1064
|
+
}
|
|
1065
|
+
i += 1;
|
|
1066
|
+
}
|
|
1067
|
+
re += "$";
|
|
1068
|
+
const compiled = new RegExp(re);
|
|
1069
|
+
cache.set(pattern, compiled);
|
|
1070
|
+
return compiled;
|
|
1071
|
+
}
|
|
1072
|
+
function matchesAny(value, patterns) {
|
|
1073
|
+
for (const p of patterns) {
|
|
1074
|
+
if (compileGlob(p).test(value)) return true;
|
|
1075
|
+
}
|
|
1076
|
+
return false;
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
// src/lib/core/discover.ts
|
|
1080
|
+
var ANCHOR_HREF_RE = /<a\b[^>]*?\bhref\s*=\s*("([^"]*)"|'([^']*)')/gi;
|
|
1081
|
+
function extractCandidateLinks(page) {
|
|
1082
|
+
const out = [];
|
|
1083
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1084
|
+
const base = page.fetch.finalUrl;
|
|
1085
|
+
for (const alt of page.links.alternates) {
|
|
1086
|
+
push(out, seen, alt.href, "hreflang");
|
|
1087
|
+
}
|
|
1088
|
+
for (const link of page.raw.links) {
|
|
1089
|
+
const rel = link.rel.toLowerCase();
|
|
1090
|
+
if ((rel === "next" || rel === "prev") && link.href) {
|
|
1091
|
+
push(out, seen, link.href, "head-link");
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
if (page.html.static) {
|
|
1095
|
+
for (const match of page.html.static.matchAll(ANCHOR_HREF_RE)) {
|
|
1096
|
+
const raw = match[2] ?? match[3];
|
|
1097
|
+
if (!raw) continue;
|
|
1098
|
+
const trimmed = raw.trim();
|
|
1099
|
+
if (!trimmed) continue;
|
|
1100
|
+
let absolute;
|
|
1101
|
+
try {
|
|
1102
|
+
absolute = new URL(trimmed, base).toString();
|
|
1103
|
+
} catch {
|
|
1104
|
+
continue;
|
|
1105
|
+
}
|
|
1106
|
+
push(out, seen, absolute, "body-anchor");
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
return out;
|
|
1110
|
+
}
|
|
1111
|
+
function push(out, seen, href, source) {
|
|
1112
|
+
if (seen.has(href)) return;
|
|
1113
|
+
seen.add(href);
|
|
1114
|
+
out.push({ href, source });
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
// src/lib/core/crawl.ts
|
|
1118
|
+
async function crawl(options) {
|
|
1119
|
+
const depthLimit = options.depth ?? 1;
|
|
1120
|
+
const concurrency = Math.max(1, options.concurrency ?? 4);
|
|
1121
|
+
const maxPages = Math.max(1, options.maxPages ?? 200);
|
|
1122
|
+
const followHreflang = options.followHreflang !== false;
|
|
1123
|
+
const include = options.include ?? [];
|
|
1124
|
+
const exclude = options.exclude ?? [];
|
|
1125
|
+
const entry = canonicaliseUrl(options.entryUrl);
|
|
1126
|
+
if (!entry) {
|
|
1127
|
+
return {
|
|
1128
|
+
pages: [],
|
|
1129
|
+
visited: [],
|
|
1130
|
+
errors: [{ url: options.entryUrl, message: "entry URL is not a valid http(s) URL" }],
|
|
1131
|
+
truncated: false
|
|
1132
|
+
};
|
|
1133
|
+
}
|
|
1134
|
+
const origin = new URL(entry).origin;
|
|
1135
|
+
const visitedSet = /* @__PURE__ */ new Set([entry]);
|
|
1136
|
+
const queue = [{ url: entry, depth: 0 }];
|
|
1137
|
+
for (const seed of options.seedUrls ?? []) {
|
|
1138
|
+
const normalised = canonicaliseUrl(seed);
|
|
1139
|
+
if (!normalised || visitedSet.has(normalised)) continue;
|
|
1140
|
+
if (new URL(normalised).origin !== origin) continue;
|
|
1141
|
+
visitedSet.add(normalised);
|
|
1142
|
+
queue.push({ url: normalised, depth: 0 });
|
|
1143
|
+
}
|
|
1144
|
+
const pages = [];
|
|
1145
|
+
const errors = [];
|
|
1146
|
+
let truncated = false;
|
|
1147
|
+
while (queue.length > 0) {
|
|
1148
|
+
if (options.inspectOptions?.signal?.aborted) break;
|
|
1149
|
+
const wave = queue.splice(0, concurrency);
|
|
1150
|
+
const inspections = wave.map(async (item) => {
|
|
1151
|
+
try {
|
|
1152
|
+
const page = await inspect(item.url, options.inspectOptions);
|
|
1153
|
+
return { kind: "ok", item, page };
|
|
1154
|
+
} catch (err) {
|
|
1155
|
+
return {
|
|
1156
|
+
kind: "err",
|
|
1157
|
+
item,
|
|
1158
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1159
|
+
};
|
|
1160
|
+
}
|
|
1161
|
+
});
|
|
1162
|
+
const results = await Promise.all(inspections);
|
|
1163
|
+
for (const result of results) {
|
|
1164
|
+
if (result.kind === "err") {
|
|
1165
|
+
errors.push({ url: result.item.url, message: result.error });
|
|
1166
|
+
continue;
|
|
1167
|
+
}
|
|
1168
|
+
pages.push(result.page);
|
|
1169
|
+
if (options.onPage) {
|
|
1170
|
+
options.onPage(result.page, { visited: pages.length, queued: queue.length });
|
|
1171
|
+
}
|
|
1172
|
+
if (pages.length >= maxPages) {
|
|
1173
|
+
truncated = true;
|
|
1174
|
+
break;
|
|
1175
|
+
}
|
|
1176
|
+
if (result.item.depth >= depthLimit) continue;
|
|
1177
|
+
const candidates = extractCandidateLinks(result.page);
|
|
1178
|
+
for (const candidate of candidates) {
|
|
1179
|
+
const normalised = canonicaliseUrl(candidate.href, result.page.fetch.finalUrl);
|
|
1180
|
+
if (!normalised) continue;
|
|
1181
|
+
if (visitedSet.has(normalised)) continue;
|
|
1182
|
+
const parsed = new URL(normalised);
|
|
1183
|
+
if (parsed.origin !== origin) continue;
|
|
1184
|
+
const passesIncludeExclude = filterAllows(parsed.pathname, include, exclude);
|
|
1185
|
+
const isHreflang = candidate.source === "hreflang";
|
|
1186
|
+
if (!passesIncludeExclude && !(followHreflang && isHreflang)) continue;
|
|
1187
|
+
visitedSet.add(normalised);
|
|
1188
|
+
queue.push({ url: normalised, depth: result.item.depth + 1 });
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
if (truncated) break;
|
|
1192
|
+
}
|
|
1193
|
+
return {
|
|
1194
|
+
pages,
|
|
1195
|
+
visited: [...visitedSet],
|
|
1196
|
+
errors,
|
|
1197
|
+
truncated
|
|
1198
|
+
};
|
|
1199
|
+
}
|
|
1200
|
+
function filterAllows(pathname, include, exclude) {
|
|
1201
|
+
if (exclude.length > 0 && matchesAny(pathname, exclude)) return false;
|
|
1202
|
+
if (include.length === 0) return true;
|
|
1203
|
+
return matchesAny(pathname, include);
|
|
1204
|
+
}
|
|
1205
|
+
function canonicaliseUrl(href, base, options = {}) {
|
|
1206
|
+
let url;
|
|
1207
|
+
try {
|
|
1208
|
+
url = new URL(href, base);
|
|
1209
|
+
} catch {
|
|
1210
|
+
return null;
|
|
1211
|
+
}
|
|
1212
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
|
1213
|
+
url.hash = "";
|
|
1214
|
+
if (!options.preserveTrailingSlash && url.pathname.length > 1 && url.pathname.endsWith("/")) {
|
|
1215
|
+
url.pathname = url.pathname.slice(0, -1);
|
|
1216
|
+
}
|
|
1217
|
+
return url.toString();
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
// src/lib/rules/index.ts
|
|
1221
|
+
var TITLE_MIN = 10;
|
|
1222
|
+
var TITLE_MAX = 60;
|
|
1223
|
+
var DESC_MIN = 50;
|
|
1224
|
+
var DESC_MAX = 160;
|
|
1225
|
+
function tokens(value) {
|
|
1226
|
+
if (!value) return [];
|
|
1227
|
+
return value.split(",").map((t) => t.trim().toLowerCase()).filter(Boolean);
|
|
1228
|
+
}
|
|
1229
|
+
var titleMissing = {
|
|
1230
|
+
id: "title.missing",
|
|
1231
|
+
severity: "error",
|
|
1232
|
+
summary: "Every page needs a non-empty `<title>`",
|
|
1233
|
+
check: ({ page, issue }) => {
|
|
1234
|
+
const t = page.meta.title?.value?.trim();
|
|
1235
|
+
if (t && t.length > 0) return;
|
|
1236
|
+
return issue({
|
|
1237
|
+
message: "Page is missing a `<title>` element (or it is empty).",
|
|
1238
|
+
origin: { kind: "title" },
|
|
1239
|
+
fix: {
|
|
1240
|
+
title: "Add a <title> to <head>",
|
|
1241
|
+
snippet: [
|
|
1242
|
+
"// app/\u2026/page.tsx \u2014 App Router owns the <head>; never hand-write the tag.",
|
|
1243
|
+
'export const metadata = { title: "Page name \u2014 Site name" };'
|
|
1244
|
+
].join("\n"),
|
|
1245
|
+
language: "tsx"
|
|
1246
|
+
}
|
|
1247
|
+
});
|
|
1248
|
+
}
|
|
1249
|
+
};
|
|
1250
|
+
var titleLength = {
|
|
1251
|
+
id: "title.length",
|
|
1252
|
+
severity: "warning",
|
|
1253
|
+
summary: `Keep \`<title>\` between ${TITLE_MIN} and ${TITLE_MAX} characters`,
|
|
1254
|
+
check: ({ page, issue }) => {
|
|
1255
|
+
const t = page.meta.title?.value?.trim();
|
|
1256
|
+
if (!t) return;
|
|
1257
|
+
if (t.length >= TITLE_MIN && t.length <= TITLE_MAX) return;
|
|
1258
|
+
const direction = t.length < TITLE_MIN ? "short" : "long";
|
|
1259
|
+
return issue({
|
|
1260
|
+
message: `Title is ${t.length} characters \u2014 ${direction} of the recommended ${TITLE_MIN}\u2013${TITLE_MAX} window.`,
|
|
1261
|
+
origin: { kind: "title" }
|
|
1262
|
+
});
|
|
1263
|
+
}
|
|
1264
|
+
};
|
|
1265
|
+
var descriptionMissing = {
|
|
1266
|
+
id: "description.missing",
|
|
1267
|
+
severity: "warning",
|
|
1268
|
+
summary: 'Provide a `<meta name="description">` on every indexable page',
|
|
1269
|
+
check: ({ page, issue }) => {
|
|
1270
|
+
const d = page.meta.description?.value?.trim();
|
|
1271
|
+
if (d && d.length > 0) return;
|
|
1272
|
+
return issue({
|
|
1273
|
+
message: 'Page has no `<meta name="description">`.',
|
|
1274
|
+
origin: { kind: "meta", name: "description" },
|
|
1275
|
+
fix: {
|
|
1276
|
+
title: "Add a meta description",
|
|
1277
|
+
snippet: [
|
|
1278
|
+
"// app/\u2026/page.tsx",
|
|
1279
|
+
"export const metadata = {",
|
|
1280
|
+
' description: "One sentence that promises what this page delivers.",',
|
|
1281
|
+
"};"
|
|
1282
|
+
].join("\n"),
|
|
1283
|
+
language: "tsx"
|
|
1284
|
+
}
|
|
1285
|
+
});
|
|
1286
|
+
}
|
|
1287
|
+
};
|
|
1288
|
+
var descriptionLength = {
|
|
1289
|
+
id: "description.length",
|
|
1290
|
+
severity: "warning",
|
|
1291
|
+
summary: `Keep descriptions between ${DESC_MIN} and ${DESC_MAX} characters`,
|
|
1292
|
+
check: ({ page, issue }) => {
|
|
1293
|
+
const d = page.meta.description?.value?.trim();
|
|
1294
|
+
if (!d) return;
|
|
1295
|
+
if (d.length >= DESC_MIN && d.length <= DESC_MAX) return;
|
|
1296
|
+
const direction = d.length < DESC_MIN ? "short" : "long";
|
|
1297
|
+
return issue({
|
|
1298
|
+
message: `Description is ${d.length} characters \u2014 ${direction} of the recommended ${DESC_MIN}\u2013${DESC_MAX} window.`,
|
|
1299
|
+
origin: { kind: "meta", name: "description" }
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1302
|
+
};
|
|
1303
|
+
var canonicalMissing = {
|
|
1304
|
+
id: "canonical.missing",
|
|
1305
|
+
severity: "warning",
|
|
1306
|
+
summary: 'Declare a `<link rel="canonical">` so search engines pick the right URL',
|
|
1307
|
+
check: ({ page, issue }) => {
|
|
1308
|
+
if (page.links.canonical) return;
|
|
1309
|
+
return issue({
|
|
1310
|
+
message: 'Page is missing `<link rel="canonical">`.',
|
|
1311
|
+
origin: { kind: "link", rel: "canonical" },
|
|
1312
|
+
fix: {
|
|
1313
|
+
title: "Declare the canonical URL",
|
|
1314
|
+
snippet: [
|
|
1315
|
+
"// app/\u2026/page.tsx \u2014 relative canonicals resolve against metadataBase,",
|
|
1316
|
+
"// which defaults to localhost and silently breaks in production.",
|
|
1317
|
+
"export const metadata = {",
|
|
1318
|
+
" metadataBase: new URL(process.env.NEXT_PUBLIC_SITE_URL!),",
|
|
1319
|
+
' alternates: { canonical: "/the-page" },',
|
|
1320
|
+
"};"
|
|
1321
|
+
].join("\n"),
|
|
1322
|
+
language: "tsx"
|
|
1323
|
+
}
|
|
1324
|
+
});
|
|
1325
|
+
}
|
|
1326
|
+
};
|
|
1327
|
+
var canonicalAbsolute = {
|
|
1328
|
+
id: "canonical.absolute",
|
|
1329
|
+
severity: "error",
|
|
1330
|
+
summary: '`rel="canonical"` must be an absolute, fully-qualified URL',
|
|
1331
|
+
check: ({ page, issue }) => {
|
|
1332
|
+
const link = page.raw.links.find((l) => l.rel.toLowerCase() === "canonical");
|
|
1333
|
+
const raw = link?.href?.trim();
|
|
1334
|
+
if (!raw) return;
|
|
1335
|
+
if (/^https?:\/\//i.test(raw)) return;
|
|
1336
|
+
return issue({
|
|
1337
|
+
message: `Canonical is "${raw}" \u2014 must be an absolute http(s) URL (consumers see the raw value, not the resolved "${page.links.canonical ?? raw}").`,
|
|
1338
|
+
origin: { kind: "link", rel: "canonical" }
|
|
1339
|
+
});
|
|
1340
|
+
}
|
|
1341
|
+
};
|
|
1342
|
+
var viewportMissing = {
|
|
1343
|
+
id: "viewport.missing",
|
|
1344
|
+
severity: "warning",
|
|
1345
|
+
summary: 'Declare a `<meta name="viewport">` so mobile browsers render at the right scale',
|
|
1346
|
+
check: ({ page, issue }) => {
|
|
1347
|
+
const v = page.meta.viewport?.value?.trim();
|
|
1348
|
+
if (v && v.length > 0) return;
|
|
1349
|
+
return issue({
|
|
1350
|
+
message: 'Page has no `<meta name="viewport">` \u2014 mobile browsers will render at desktop width.',
|
|
1351
|
+
origin: { kind: "meta", name: "viewport" },
|
|
1352
|
+
fix: {
|
|
1353
|
+
title: "Add a viewport meta",
|
|
1354
|
+
snippet: [
|
|
1355
|
+
"// app/layout.tsx \u2014 a dedicated export, not part of `metadata`.",
|
|
1356
|
+
'export const viewport = { width: "device-width", initialScale: 1 };'
|
|
1357
|
+
].join("\n"),
|
|
1358
|
+
language: "tsx"
|
|
1359
|
+
}
|
|
1360
|
+
});
|
|
1361
|
+
}
|
|
1362
|
+
};
|
|
1363
|
+
var ogTitleMissing = {
|
|
1364
|
+
id: "og.title.missing",
|
|
1365
|
+
severity: "warning",
|
|
1366
|
+
summary: "Set an explicit `og:title` instead of relying on `<title>` fallback",
|
|
1367
|
+
check: ({ page, issue }) => {
|
|
1368
|
+
if (page.openGraph.title?.value?.trim()) return;
|
|
1369
|
+
return issue({
|
|
1370
|
+
message: "Page has no `og:title`; consumers will fall back to `<title>` (or nothing).",
|
|
1371
|
+
origin: { kind: "meta", property: "og:title" }
|
|
1372
|
+
});
|
|
1373
|
+
}
|
|
1374
|
+
};
|
|
1375
|
+
var ogDescriptionMissing = {
|
|
1376
|
+
id: "og.description.missing",
|
|
1377
|
+
severity: "info",
|
|
1378
|
+
summary: "Set an explicit `og:description` for richer link unfurls",
|
|
1379
|
+
check: ({ page, issue }) => {
|
|
1380
|
+
if (page.openGraph.description?.value?.trim()) return;
|
|
1381
|
+
const hasOtherOg = Boolean(
|
|
1382
|
+
page.openGraph.title?.value || page.openGraph.images.length > 0 || page.openGraph.url?.value
|
|
1383
|
+
);
|
|
1384
|
+
if (!hasOtherOg) return;
|
|
1385
|
+
return issue({
|
|
1386
|
+
message: "Page has `og:*` tags but no `og:description`; unfurls will fall back to the meta description (or nothing).",
|
|
1387
|
+
origin: { kind: "meta", property: "og:description" }
|
|
1388
|
+
});
|
|
1389
|
+
}
|
|
1390
|
+
};
|
|
1391
|
+
var ogImageMissing = {
|
|
1392
|
+
id: "og.image.missing",
|
|
1393
|
+
severity: "warning",
|
|
1394
|
+
summary: "Provide at least one `og:image` so links unfurl with a preview",
|
|
1395
|
+
check: ({ page, issue }) => {
|
|
1396
|
+
if (page.openGraph.images.length > 0) return;
|
|
1397
|
+
return issue({
|
|
1398
|
+
message: "Page has no `og:image`. Link unfurls will fall back to text-only or a random body image.",
|
|
1399
|
+
origin: { kind: "meta", property: "og:image" },
|
|
1400
|
+
fix: {
|
|
1401
|
+
title: "Add an og:image",
|
|
1402
|
+
snippet: [
|
|
1403
|
+
"// app/\u2026/page.tsx",
|
|
1404
|
+
"export const metadata = {",
|
|
1405
|
+
' openGraph: { images: [{ url: "/og.png", width: 1200, height: 630 }] },',
|
|
1406
|
+
"};",
|
|
1407
|
+
"// Or generate one per page: app/\u2026/opengraph-image.tsx with ImageResponse."
|
|
1408
|
+
].join("\n"),
|
|
1409
|
+
language: "tsx"
|
|
1410
|
+
}
|
|
1411
|
+
});
|
|
1412
|
+
}
|
|
1413
|
+
};
|
|
1414
|
+
var robotsConflict = {
|
|
1415
|
+
id: "robots.conflict",
|
|
1416
|
+
severity: "error",
|
|
1417
|
+
summary: "`robots`, `googlebot`, and `X-Robots-Tag` must not contradict each other",
|
|
1418
|
+
check: ({ page, issue }) => {
|
|
1419
|
+
const issues = [];
|
|
1420
|
+
const metaRobots = new Set(tokens(page.meta.robots?.value));
|
|
1421
|
+
const metaGoogle = new Set(tokens(page.meta.googlebot?.value));
|
|
1422
|
+
const header = new Set(tokens(page.fetch.headers["x-robots-tag"]));
|
|
1423
|
+
const allSources = [
|
|
1424
|
+
{ name: "meta robots", tokens: metaRobots, origin: { kind: "meta", name: "robots" } },
|
|
1425
|
+
{ name: "meta googlebot", tokens: metaGoogle, origin: { kind: "meta", name: "googlebot" } },
|
|
1426
|
+
{
|
|
1427
|
+
name: "X-Robots-Tag header",
|
|
1428
|
+
tokens: header,
|
|
1429
|
+
origin: { kind: "header", name: "x-robots-tag" }
|
|
1430
|
+
}
|
|
1431
|
+
];
|
|
1432
|
+
const sources = allSources.filter((s) => s.tokens.size > 0);
|
|
1433
|
+
if (sources.length < 2) return [];
|
|
1434
|
+
const indexers = sources.filter((s) => s.tokens.has("index") && !s.tokens.has("noindex"));
|
|
1435
|
+
const noindexers = sources.filter((s) => s.tokens.has("noindex"));
|
|
1436
|
+
if (indexers.length > 0 && noindexers.length > 0) {
|
|
1437
|
+
issues.push(
|
|
1438
|
+
issue({
|
|
1439
|
+
message: `Conflicting indexing directives: ${noindexers.map((s) => s.name).join(", ")} say \`noindex\`, ${indexers.map((s) => s.name).join(", ")} say \`index\`.`,
|
|
1440
|
+
origin: noindexers[0].origin
|
|
1441
|
+
})
|
|
1442
|
+
);
|
|
1443
|
+
}
|
|
1444
|
+
const followers = sources.filter((s) => s.tokens.has("follow") && !s.tokens.has("nofollow"));
|
|
1445
|
+
const nofollowers = sources.filter((s) => s.tokens.has("nofollow"));
|
|
1446
|
+
if (followers.length > 0 && nofollowers.length > 0) {
|
|
1447
|
+
issues.push(
|
|
1448
|
+
issue({
|
|
1449
|
+
message: `Conflicting follow directives: ${nofollowers.map((s) => s.name).join(", ")} say \`nofollow\`, ${followers.map((s) => s.name).join(", ")} say \`follow\`.`,
|
|
1450
|
+
origin: nofollowers[0].origin
|
|
1451
|
+
})
|
|
1452
|
+
);
|
|
1453
|
+
}
|
|
1454
|
+
return issues;
|
|
1455
|
+
}
|
|
1456
|
+
};
|
|
1457
|
+
var RULES = [
|
|
1458
|
+
canonicalAbsolute,
|
|
1459
|
+
canonicalMissing,
|
|
1460
|
+
descriptionLength,
|
|
1461
|
+
descriptionMissing,
|
|
1462
|
+
ogDescriptionMissing,
|
|
1463
|
+
ogImageMissing,
|
|
1464
|
+
ogTitleMissing,
|
|
1465
|
+
robotsConflict,
|
|
1466
|
+
titleLength,
|
|
1467
|
+
titleMissing,
|
|
1468
|
+
viewportMissing
|
|
1469
|
+
];
|
|
1470
|
+
var RULE_BY_ID = new Map(RULES.map((r) => [r.id, r]));
|
|
1471
|
+
function getRule(id) {
|
|
1472
|
+
return RULE_BY_ID.get(id);
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
// src/lib/core/lint.ts
|
|
1476
|
+
var SEVERITY_RANK = {
|
|
1477
|
+
error: 0,
|
|
1478
|
+
warning: 1,
|
|
1479
|
+
info: 2
|
|
1480
|
+
};
|
|
1481
|
+
function lint(page, rules = RULES) {
|
|
1482
|
+
const issues = [];
|
|
1483
|
+
for (const rule of rules) {
|
|
1484
|
+
if (rule.appliesTo && !rule.appliesTo(page)) continue;
|
|
1485
|
+
let result;
|
|
1486
|
+
try {
|
|
1487
|
+
result = rule.check({
|
|
1488
|
+
page,
|
|
1489
|
+
issue: (input) => ({
|
|
1490
|
+
ruleId: rule.id,
|
|
1491
|
+
severity: input.severity ?? rule.severity,
|
|
1492
|
+
message: input.message,
|
|
1493
|
+
origin: input.origin,
|
|
1494
|
+
fix: input.fix,
|
|
1495
|
+
suggestion: input.suggestion
|
|
1496
|
+
})
|
|
1497
|
+
});
|
|
1498
|
+
} catch (err) {
|
|
1499
|
+
issues.push({
|
|
1500
|
+
ruleId: "engine.rule-crashed",
|
|
1501
|
+
severity: "info",
|
|
1502
|
+
message: `Rule \`${rule.id}\` threw: ${err instanceof Error ? err.message : String(err)}`
|
|
1503
|
+
});
|
|
1504
|
+
continue;
|
|
1505
|
+
}
|
|
1506
|
+
if (!result) continue;
|
|
1507
|
+
if (Array.isArray(result)) issues.push(...result);
|
|
1508
|
+
else issues.push(result);
|
|
1509
|
+
}
|
|
1510
|
+
return sortIssues(issues);
|
|
1511
|
+
}
|
|
1512
|
+
function sortIssues(issues) {
|
|
1513
|
+
return [...issues].sort((a, b) => {
|
|
1514
|
+
const sev = SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
|
|
1515
|
+
if (sev !== 0) return sev;
|
|
1516
|
+
return a.ruleId.localeCompare(b.ruleId);
|
|
1517
|
+
});
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
// src/lib/core/i18n.ts
|
|
1521
|
+
var BCP47_LOOSE = /^[a-z]{2,3}(-[a-z]{2}|-\d{3})?$/i;
|
|
1522
|
+
function looksLikeLocaleSegment(segment) {
|
|
1523
|
+
return BCP47_LOOSE.test(segment);
|
|
1524
|
+
}
|
|
1525
|
+
function isValidLocale(tag) {
|
|
1526
|
+
if (tag === "x-default") return true;
|
|
1527
|
+
return BCP47_LOOSE.test(tag);
|
|
1528
|
+
}
|
|
1529
|
+
function buildI18nMatrix(pages, options = {}) {
|
|
1530
|
+
const inspectedByUrl = /* @__PURE__ */ new Map();
|
|
1531
|
+
for (const page of pages) inspectedByUrl.set(page.fetch.finalUrl, page);
|
|
1532
|
+
const grid = /* @__PURE__ */ new Map();
|
|
1533
|
+
const locales = /* @__PURE__ */ new Set();
|
|
1534
|
+
const routes = /* @__PURE__ */ new Set();
|
|
1535
|
+
for (const locale of options.locales ?? []) {
|
|
1536
|
+
const tag = locale.trim().toLowerCase();
|
|
1537
|
+
if (tag) locales.add(tag);
|
|
1538
|
+
}
|
|
1539
|
+
function record(route, locale, url) {
|
|
1540
|
+
routes.add(route);
|
|
1541
|
+
locales.add(locale);
|
|
1542
|
+
let row = grid.get(route);
|
|
1543
|
+
if (!row) {
|
|
1544
|
+
row = /* @__PURE__ */ new Map();
|
|
1545
|
+
grid.set(route, row);
|
|
1546
|
+
}
|
|
1547
|
+
if (!row.has(locale)) {
|
|
1548
|
+
row.set(locale, { url, inspected: inspectedByUrl.has(url) });
|
|
1549
|
+
}
|
|
1550
|
+
}
|
|
1551
|
+
for (const page of pages) {
|
|
1552
|
+
const url = new URL(page.fetch.finalUrl);
|
|
1553
|
+
const { route: selfRoute, locale: selfLocale } = splitRoute(url.pathname);
|
|
1554
|
+
record(selfRoute, selfLocale, page.fetch.finalUrl);
|
|
1555
|
+
for (const alt of page.links.alternates) {
|
|
1556
|
+
let altUrl;
|
|
1557
|
+
try {
|
|
1558
|
+
altUrl = new URL(alt.href);
|
|
1559
|
+
} catch {
|
|
1560
|
+
continue;
|
|
1561
|
+
}
|
|
1562
|
+
const { route } = splitRoute(altUrl.pathname);
|
|
1563
|
+
const locale = alt.isXDefault ? "x-default" : alt.hreflang;
|
|
1564
|
+
record(route || selfRoute, locale, altUrl.toString());
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
for (const declared of options.declaredUrls ?? []) {
|
|
1568
|
+
let url;
|
|
1569
|
+
try {
|
|
1570
|
+
url = new URL(declared);
|
|
1571
|
+
} catch {
|
|
1572
|
+
continue;
|
|
1573
|
+
}
|
|
1574
|
+
const { route, locale } = splitRoute(url.pathname);
|
|
1575
|
+
record(route, locale, url.toString());
|
|
1576
|
+
}
|
|
1577
|
+
const sortedLocales = sortLocales([...locales]);
|
|
1578
|
+
const sortedRoutes = [...routes].sort((a, b) => a.localeCompare(b));
|
|
1579
|
+
const cells = {};
|
|
1580
|
+
for (const route of sortedRoutes) {
|
|
1581
|
+
cells[route] = {};
|
|
1582
|
+
for (const locale of sortedLocales) {
|
|
1583
|
+
const slot = grid.get(route)?.get(locale);
|
|
1584
|
+
cells[route][locale] = slot ? { url: slot.url, inspected: slot.inspected } : { url: null, inspected: false };
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
return { locales: sortedLocales, routes: sortedRoutes, cells };
|
|
1588
|
+
}
|
|
1589
|
+
function reciprocityIssues(pages) {
|
|
1590
|
+
const issues = [];
|
|
1591
|
+
const byUrl = /* @__PURE__ */ new Map();
|
|
1592
|
+
for (const page of pages) byUrl.set(page.fetch.finalUrl, page);
|
|
1593
|
+
for (const page of pages) {
|
|
1594
|
+
const url = page.fetch.finalUrl;
|
|
1595
|
+
const seenLocalesOnPage = /* @__PURE__ */ new Set();
|
|
1596
|
+
for (const alt of page.links.alternates) {
|
|
1597
|
+
const locale = alt.isXDefault ? "x-default" : alt.hreflang;
|
|
1598
|
+
seenLocalesOnPage.add(locale);
|
|
1599
|
+
if (!isValidLocale(locale)) {
|
|
1600
|
+
issues.push({
|
|
1601
|
+
code: "locale.invalid",
|
|
1602
|
+
url,
|
|
1603
|
+
locale,
|
|
1604
|
+
message: `\`hreflang="${locale}"\` is not a valid BCP 47 tag.`
|
|
1605
|
+
});
|
|
1606
|
+
continue;
|
|
1607
|
+
}
|
|
1608
|
+
let altUrl;
|
|
1609
|
+
try {
|
|
1610
|
+
altUrl = new URL(alt.href).toString();
|
|
1611
|
+
} catch {
|
|
1612
|
+
continue;
|
|
1613
|
+
}
|
|
1614
|
+
const peer = byUrl.get(altUrl);
|
|
1615
|
+
if (!peer) continue;
|
|
1616
|
+
const back = peer.links.alternates.find((p) => {
|
|
1617
|
+
let peerHref;
|
|
1618
|
+
try {
|
|
1619
|
+
peerHref = new URL(p.href).toString();
|
|
1620
|
+
} catch {
|
|
1621
|
+
return false;
|
|
1622
|
+
}
|
|
1623
|
+
return peerHref === url;
|
|
1624
|
+
});
|
|
1625
|
+
if (!back) {
|
|
1626
|
+
issues.push({
|
|
1627
|
+
code: "missing-back-link",
|
|
1628
|
+
url,
|
|
1629
|
+
peerUrl: altUrl,
|
|
1630
|
+
locale,
|
|
1631
|
+
message: `\`${url}\` declares an alternate to \`${altUrl}\` but the peer does not link back.`
|
|
1632
|
+
});
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
const nonDefaultLocales = [...seenLocalesOnPage].filter((l) => l !== "x-default");
|
|
1636
|
+
if (nonDefaultLocales.length >= 2 && !seenLocalesOnPage.has("x-default")) {
|
|
1637
|
+
issues.push({
|
|
1638
|
+
code: "x-default-missing",
|
|
1639
|
+
url,
|
|
1640
|
+
message: `Page advertises ${nonDefaultLocales.length} locales but no \`hreflang="x-default"\`.`
|
|
1641
|
+
});
|
|
1642
|
+
}
|
|
1643
|
+
}
|
|
1644
|
+
return dedupeBy(
|
|
1645
|
+
issues,
|
|
1646
|
+
(i) => `${i.code}\0${i.url}\0${i.peerUrl ?? ""}\0${i.locale ?? ""}`
|
|
1647
|
+
);
|
|
1648
|
+
}
|
|
1649
|
+
function dedupeBy(items, key) {
|
|
1650
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1651
|
+
const out = [];
|
|
1652
|
+
for (const item of items) {
|
|
1653
|
+
const k = key(item);
|
|
1654
|
+
if (seen.has(k)) continue;
|
|
1655
|
+
seen.add(k);
|
|
1656
|
+
out.push(item);
|
|
1657
|
+
}
|
|
1658
|
+
return out;
|
|
1659
|
+
}
|
|
1660
|
+
function splitRoute(pathname) {
|
|
1661
|
+
const segments = pathname.split("/").filter(Boolean);
|
|
1662
|
+
if (segments.length > 0 && looksLikeLocaleSegment(segments[0])) {
|
|
1663
|
+
const locale = segments[0];
|
|
1664
|
+
const rest = segments.slice(1).join("/");
|
|
1665
|
+
return { route: rest ? `/${rest}` : "/", locale };
|
|
1666
|
+
}
|
|
1667
|
+
return { route: pathname || "/", locale: "x-default" };
|
|
1668
|
+
}
|
|
1669
|
+
function sortLocales(locales) {
|
|
1670
|
+
const xDefault = locales.includes("x-default") ? ["x-default"] : [];
|
|
1671
|
+
const rest = locales.filter((l) => l !== "x-default").sort((a, b) => a.localeCompare(b));
|
|
1672
|
+
return [...xDefault, ...rest];
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
// src/lib/rules/site-rules.ts
|
|
1676
|
+
function isMultilingual(site) {
|
|
1677
|
+
return site.localeAxis.multilingual;
|
|
1678
|
+
}
|
|
1679
|
+
function declaredLocales(page) {
|
|
1680
|
+
const out = /* @__PURE__ */ new Set();
|
|
1681
|
+
for (const alt of page.links.alternates) {
|
|
1682
|
+
if (alt.isXDefault) continue;
|
|
1683
|
+
const tag = alt.hreflang?.trim().toLowerCase();
|
|
1684
|
+
if (tag) out.add(tag);
|
|
1685
|
+
}
|
|
1686
|
+
return out;
|
|
1687
|
+
}
|
|
1688
|
+
function sitemapLocalesByRoute(site) {
|
|
1689
|
+
const byRoute = /* @__PURE__ */ new Map();
|
|
1690
|
+
for (const entry of site.discovery?.urls ?? []) {
|
|
1691
|
+
let pathname;
|
|
1692
|
+
try {
|
|
1693
|
+
pathname = new URL(entry.loc).pathname;
|
|
1694
|
+
} catch {
|
|
1695
|
+
continue;
|
|
1696
|
+
}
|
|
1697
|
+
const { route, locale } = splitRoute(pathname);
|
|
1698
|
+
if (locale === "x-default") continue;
|
|
1699
|
+
const set = byRoute.get(route) ?? /* @__PURE__ */ new Set();
|
|
1700
|
+
set.add(locale.toLowerCase());
|
|
1701
|
+
byRoute.set(route, set);
|
|
1702
|
+
}
|
|
1703
|
+
return byRoute;
|
|
1704
|
+
}
|
|
1705
|
+
function sorted(set) {
|
|
1706
|
+
return [...set].sort((a, b) => a.localeCompare(b));
|
|
1707
|
+
}
|
|
1708
|
+
var hreflangMissing = {
|
|
1709
|
+
id: "hreflang.missing",
|
|
1710
|
+
severity: "error",
|
|
1711
|
+
summary: "Pages on a multilingual site must advertise their locale alternates",
|
|
1712
|
+
appliesTo: isMultilingual,
|
|
1713
|
+
check: ({ site, issue }) => {
|
|
1714
|
+
const locales = site.localeAxis.locales.join(", ");
|
|
1715
|
+
return site.pages.filter((page) => page.links.alternates.length === 0).map(
|
|
1716
|
+
(page) => issue({
|
|
1717
|
+
pageUrl: page.fetch.finalUrl,
|
|
1718
|
+
message: `Page declares no \`hreflang\` alternates, but the site serves ${site.localeAxis.locales.length} locales (${locales}, per the ${site.localeAxis.source}). Locale variants of this route cannot be associated with each other.`,
|
|
1719
|
+
origin: { kind: "link", rel: "alternate" },
|
|
1720
|
+
fix: {
|
|
1721
|
+
title: "Emit alternates from generateMetadata()",
|
|
1722
|
+
snippet: [
|
|
1723
|
+
"// app/[locale]/\u2026/page.tsx",
|
|
1724
|
+
"export async function generateMetadata({ params }) {",
|
|
1725
|
+
" const { locale } = await params;",
|
|
1726
|
+
" return {",
|
|
1727
|
+
" alternates: {",
|
|
1728
|
+
" canonical: `${baseUrl}/${locale}${path}`,",
|
|
1729
|
+
" languages: {",
|
|
1730
|
+
...site.localeAxis.locales.map(
|
|
1731
|
+
(l) => ` "${l}": \`\${baseUrl}/${l}\${path}\`,`
|
|
1732
|
+
),
|
|
1733
|
+
' "x-default": `${baseUrl}/${defaultLocale}${path}`,',
|
|
1734
|
+
" },",
|
|
1735
|
+
" },",
|
|
1736
|
+
" };",
|
|
1737
|
+
"}"
|
|
1738
|
+
].join("\n"),
|
|
1739
|
+
language: "tsx"
|
|
1740
|
+
}
|
|
1741
|
+
})
|
|
1742
|
+
);
|
|
1743
|
+
}
|
|
1744
|
+
};
|
|
1745
|
+
var hreflangSitemapMismatch = {
|
|
1746
|
+
id: "hreflang.sitemap-mismatch",
|
|
1747
|
+
severity: "warning",
|
|
1748
|
+
summary: "`<head>` alternates and sitemap locale coverage must agree",
|
|
1749
|
+
appliesTo: (site) => isMultilingual(site) && (site.discovery?.urls.length ?? 0) > 0,
|
|
1750
|
+
check: ({ site, issue }) => {
|
|
1751
|
+
const bySitemap = sitemapLocalesByRoute(site);
|
|
1752
|
+
const findings = [];
|
|
1753
|
+
for (const page of site.pages) {
|
|
1754
|
+
const head = declaredLocales(page);
|
|
1755
|
+
if (head.size === 0) continue;
|
|
1756
|
+
let pathname;
|
|
1757
|
+
try {
|
|
1758
|
+
pathname = new URL(page.fetch.finalUrl).pathname;
|
|
1759
|
+
} catch {
|
|
1760
|
+
continue;
|
|
1761
|
+
}
|
|
1762
|
+
const { route } = splitRoute(pathname);
|
|
1763
|
+
const inSitemap = bySitemap.get(route);
|
|
1764
|
+
if (!inSitemap || inSitemap.size === 0) continue;
|
|
1765
|
+
const onlyInSitemap = sorted(new Set([...inSitemap].filter((l) => !head.has(l))));
|
|
1766
|
+
const onlyInHead = sorted(new Set([...head].filter((l) => !inSitemap.has(l))));
|
|
1767
|
+
if (onlyInSitemap.length === 0 && onlyInHead.length === 0) continue;
|
|
1768
|
+
const parts = [];
|
|
1769
|
+
if (onlyInSitemap.length > 0) {
|
|
1770
|
+
parts.push(
|
|
1771
|
+
`the sitemap lists ${onlyInSitemap.join(", ")} but the \`<head>\` does not advertise ${onlyInSitemap.length > 1 ? "them" : "it"}`
|
|
1772
|
+
);
|
|
1773
|
+
}
|
|
1774
|
+
if (onlyInHead.length > 0) {
|
|
1775
|
+
parts.push(
|
|
1776
|
+
`the \`<head>\` advertises ${onlyInHead.join(", ")} but the sitemap has no entry for ${onlyInHead.length > 1 ? "them" : "it"}`
|
|
1777
|
+
);
|
|
1778
|
+
}
|
|
1779
|
+
findings.push(
|
|
1780
|
+
issue({
|
|
1781
|
+
pageUrl: page.fetch.finalUrl,
|
|
1782
|
+
message: `Route \`${route}\`: ${parts.join("; ")}. Both are derived from the same intent and must not disagree.`,
|
|
1783
|
+
origin: { kind: "link", rel: "alternate" },
|
|
1784
|
+
fix: {
|
|
1785
|
+
title: "Derive both from one locale-availability source",
|
|
1786
|
+
snippet: [
|
|
1787
|
+
"// Compute availability once, feed both the <head> and the sitemap.",
|
|
1788
|
+
"const localesFor = (slug: string) =>",
|
|
1789
|
+
" allDocs.filter((d) => d.slug === slug && !d.draft).map((d) => d.locale);",
|
|
1790
|
+
"",
|
|
1791
|
+
"// generateMetadata(): alternates.languages \u2190 localesFor(slug)",
|
|
1792
|
+
"// sitemap.ts: alternates.languages \u2190 localesFor(slug)"
|
|
1793
|
+
].join("\n"),
|
|
1794
|
+
language: "ts"
|
|
1795
|
+
}
|
|
1796
|
+
})
|
|
1797
|
+
);
|
|
1798
|
+
}
|
|
1799
|
+
return findings;
|
|
1800
|
+
}
|
|
1801
|
+
};
|
|
1802
|
+
function metaRobotsTokens(page) {
|
|
1803
|
+
const raw = page.meta.robots?.value ?? "";
|
|
1804
|
+
return new Set(
|
|
1805
|
+
raw.split(",").map((t) => t.trim().toLowerCase()).filter(Boolean)
|
|
1806
|
+
);
|
|
1807
|
+
}
|
|
1808
|
+
var robotsBlocksSite = {
|
|
1809
|
+
id: "robots.blocks-site",
|
|
1810
|
+
severity: "error",
|
|
1811
|
+
summary: "`robots.txt` must not forbid crawling a site that asks to be indexed",
|
|
1812
|
+
appliesTo: (site) => site.robots?.found === true && site.robots.blocksAll,
|
|
1813
|
+
check: ({ site, issue }) => {
|
|
1814
|
+
const robotsUrl = site.robots?.url ?? `${site.origin}/robots.txt`;
|
|
1815
|
+
const asking = site.pages.filter((page) => metaRobotsTokens(page).has("index"));
|
|
1816
|
+
const detail = asking.length ? `but ${asking.length} crawled page${asking.length === 1 ? "" : "s"} declare \`<meta name="robots" content="index">\`. Both cannot be true: robots.txt wins, so the pages are never fetched and the meta tag is never read.` : "so no search engine will crawl any page on this origin. If this is a staging or preview environment, that is correct \u2014 otherwise the site is invisible.";
|
|
1817
|
+
return issue({
|
|
1818
|
+
pageUrl: robotsUrl,
|
|
1819
|
+
severity: asking.length > 0 ? "error" : "warning",
|
|
1820
|
+
message: `\`robots.txt\` disallows the whole site for \`User-agent: *\`, ${detail}`,
|
|
1821
|
+
origin: { kind: "computed" },
|
|
1822
|
+
fix: {
|
|
1823
|
+
title: "Gate the disallow on the deployed environment",
|
|
1824
|
+
snippet: [
|
|
1825
|
+
"// app/robots.ts \u2014 the flag must be readable at build AND at runtime,",
|
|
1826
|
+
"// or a production container silently serves the staging rules.",
|
|
1827
|
+
'const isProduction = process.env.APP_ENV === "production";',
|
|
1828
|
+
"",
|
|
1829
|
+
"export default function robots(): MetadataRoute.Robots {",
|
|
1830
|
+
' if (!isProduction) return { rules: { userAgent: "*", disallow: "/" } };',
|
|
1831
|
+
" return {",
|
|
1832
|
+
' rules: { userAgent: "*", allow: "/", disallow: ["/api/", "/_next/"] },',
|
|
1833
|
+
" sitemap: `${baseUrl}/sitemap.xml`,",
|
|
1834
|
+
" };",
|
|
1835
|
+
"}"
|
|
1836
|
+
].join("\n"),
|
|
1837
|
+
language: "ts"
|
|
1838
|
+
}
|
|
1839
|
+
});
|
|
1840
|
+
}
|
|
1841
|
+
};
|
|
1842
|
+
var SITE_RULES = [
|
|
1843
|
+
hreflangMissing,
|
|
1844
|
+
hreflangSitemapMismatch,
|
|
1845
|
+
robotsBlocksSite
|
|
1846
|
+
];
|
|
1847
|
+
function getSiteRule(id) {
|
|
1848
|
+
return SITE_RULES.find((rule) => rule.id === id);
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1851
|
+
// src/lib/core/lint-site.ts
|
|
1852
|
+
function lintSite(site, rules = SITE_RULES) {
|
|
1853
|
+
const drafts = [];
|
|
1854
|
+
for (const rule of rules) {
|
|
1855
|
+
if (rule.appliesTo && !rule.appliesTo(site)) continue;
|
|
1856
|
+
let result;
|
|
1857
|
+
try {
|
|
1858
|
+
result = rule.check({
|
|
1859
|
+
site,
|
|
1860
|
+
issue: (input) => ({
|
|
1861
|
+
ruleId: rule.id,
|
|
1862
|
+
severity: input.severity ?? rule.severity,
|
|
1863
|
+
message: input.message,
|
|
1864
|
+
origin: input.origin,
|
|
1865
|
+
fix: input.fix,
|
|
1866
|
+
suggestion: input.suggestion,
|
|
1867
|
+
pageUrl: input.pageUrl
|
|
1868
|
+
})
|
|
1869
|
+
});
|
|
1870
|
+
} catch (err) {
|
|
1871
|
+
drafts.push({
|
|
1872
|
+
ruleId: "engine.site-rule-crashed",
|
|
1873
|
+
severity: "info",
|
|
1874
|
+
message: `Site rule \`${rule.id}\` threw: ${err instanceof Error ? err.message : String(err)}`,
|
|
1875
|
+
pageUrl: site.origin
|
|
1876
|
+
});
|
|
1877
|
+
continue;
|
|
1878
|
+
}
|
|
1879
|
+
if (!result) continue;
|
|
1880
|
+
if (Array.isArray(result)) drafts.push(...result);
|
|
1881
|
+
else drafts.push(result);
|
|
1882
|
+
}
|
|
1883
|
+
return sortSiteIssues(drafts);
|
|
1884
|
+
}
|
|
1885
|
+
function sortSiteIssues(drafts) {
|
|
1886
|
+
const byPage = [...drafts].sort((a, b) => a.pageUrl.localeCompare(b.pageUrl));
|
|
1887
|
+
return sortIssues(byPage);
|
|
1888
|
+
}
|
|
1889
|
+
|
|
1890
|
+
// src/lib/core/locales.ts
|
|
1891
|
+
var ISO_639_1 = new Set(
|
|
1892
|
+
"aa ab ae af ak am an ar as av ay az ba be bg bh bi bm bn bo br bs ca ce ch co cr cs cu cv cy da de dv dz ee el en eo es et eu fa ff fi fj fo fr fy ga gd gl gn gu gv ha he hi ho hr ht hu hy hz ia id ie ig ii ik io is it iu ja jv ka kg ki kj kk kl km kn ko kr ks ku kv kw ky la lb lg li ln lo lt lu lv mg mh mi mk ml mn mr ms mt my na nb nd ne ng nl nn no nr nv ny oc oj om or os pa pi pl ps pt qu rm rn ro ru rw sa sc sd se sg si sk sl sm sn so sq sr ss st su sv sw ta te tg th ti tk tl tn to tr ts tt tw ty ug uk ur uz ve vi vo wa wo xh yi yo za zh zu".split(" ")
|
|
1893
|
+
);
|
|
1894
|
+
function isKnownLanguageTag(tag) {
|
|
1895
|
+
const primary = tag.toLowerCase().split("-")[0] ?? "";
|
|
1896
|
+
return ISO_639_1.has(primary);
|
|
1897
|
+
}
|
|
1898
|
+
function localePrefixOf(url) {
|
|
1899
|
+
let pathname;
|
|
1900
|
+
try {
|
|
1901
|
+
pathname = new URL(url).pathname;
|
|
1902
|
+
} catch {
|
|
1903
|
+
return null;
|
|
1904
|
+
}
|
|
1905
|
+
const first = pathname.split("/").filter(Boolean)[0];
|
|
1906
|
+
if (first && looksLikeLocaleSegment(first)) return first.toLowerCase();
|
|
1907
|
+
return null;
|
|
1908
|
+
}
|
|
1909
|
+
function prefixesIn(urls) {
|
|
1910
|
+
const out = /* @__PURE__ */ new Set();
|
|
1911
|
+
for (const url of urls ?? []) {
|
|
1912
|
+
const locale = localePrefixOf(url);
|
|
1913
|
+
if (locale) out.add(locale);
|
|
1914
|
+
}
|
|
1915
|
+
out.delete("x-default");
|
|
1916
|
+
return out;
|
|
1917
|
+
}
|
|
1918
|
+
function describeCandidates(pages) {
|
|
1919
|
+
const byTag = /* @__PURE__ */ new Map();
|
|
1920
|
+
for (const page of pages) {
|
|
1921
|
+
const tag = localePrefixOf(page.fetch.finalUrl);
|
|
1922
|
+
if (!tag || tag === "x-default") continue;
|
|
1923
|
+
const entry = byTag.get(tag) ?? { pages: 0, langs: /* @__PURE__ */ new Set() };
|
|
1924
|
+
entry.pages += 1;
|
|
1925
|
+
const lang = page.raw.htmlLang?.trim().toLowerCase();
|
|
1926
|
+
if (lang) entry.langs.add(lang);
|
|
1927
|
+
byTag.set(tag, entry);
|
|
1928
|
+
}
|
|
1929
|
+
const candidates = [];
|
|
1930
|
+
for (const [tag, { pages: n, langs }] of byTag) {
|
|
1931
|
+
const observedLangs = [...langs].sort();
|
|
1932
|
+
const htmlLangAgrees = observedLangs.length === 0 ? void 0 : observedLangs.some((l) => l.split("-")[0] === tag.split("-")[0]);
|
|
1933
|
+
candidates.push({
|
|
1934
|
+
tag,
|
|
1935
|
+
pages: n,
|
|
1936
|
+
isKnownLanguage: isKnownLanguageTag(tag),
|
|
1937
|
+
htmlLangAgrees,
|
|
1938
|
+
observedLangs
|
|
1939
|
+
});
|
|
1940
|
+
}
|
|
1941
|
+
return candidates.sort(
|
|
1942
|
+
(a, b) => Number(b.htmlLangAgrees ?? false) - Number(a.htmlLangAgrees ?? false) || Number(b.isKnownLanguage) - Number(a.isKnownLanguage) || b.pages - a.pages || a.tag.localeCompare(b.tag)
|
|
1943
|
+
);
|
|
1944
|
+
}
|
|
1945
|
+
function deriveLocaleAxis(input) {
|
|
1946
|
+
const explicit = new Set(
|
|
1947
|
+
(input.explicit ?? []).map((l) => l.trim().toLowerCase()).filter(Boolean)
|
|
1948
|
+
);
|
|
1949
|
+
explicit.delete("x-default");
|
|
1950
|
+
const fromSitemap = prefixesIn(input.sitemapUrls);
|
|
1951
|
+
if (explicit.size > 0 || fromSitemap.size > 0) {
|
|
1952
|
+
const all = /* @__PURE__ */ new Set([...explicit, ...fromSitemap]);
|
|
1953
|
+
return {
|
|
1954
|
+
locales: [...all].sort((a, b) => a.localeCompare(b)),
|
|
1955
|
+
source: explicit.size > 0 ? "explicit" : "sitemap",
|
|
1956
|
+
multilingual: all.size >= 2,
|
|
1957
|
+
candidates: []
|
|
1958
|
+
};
|
|
1959
|
+
}
|
|
1960
|
+
return {
|
|
1961
|
+
locales: [],
|
|
1962
|
+
source: "none",
|
|
1963
|
+
multilingual: false,
|
|
1964
|
+
candidates: describeCandidates(input.pages ?? [])
|
|
1965
|
+
};
|
|
1966
|
+
}
|
|
1967
|
+
function suggestedLocales(axis) {
|
|
1968
|
+
const plausible = axis.candidates.filter((c) => c.isKnownLanguage && c.htmlLangAgrees !== false);
|
|
1969
|
+
if (plausible.length < 2) return null;
|
|
1970
|
+
return plausible.map((c) => c.tag).sort((a, b) => a.localeCompare(b)).join(",");
|
|
1971
|
+
}
|
|
1972
|
+
|
|
1973
|
+
// src/lib/core/sitemap/discover.ts
|
|
1974
|
+
import { gunzipSync } from "zlib";
|
|
1975
|
+
|
|
1976
|
+
// src/lib/core/sitemap/parse.ts
|
|
1977
|
+
import * as cheerio2 from "cheerio";
|
|
1978
|
+
function parseSitemap(body) {
|
|
1979
|
+
const trimmed = body.trim();
|
|
1980
|
+
if (trimmed.length === 0) {
|
|
1981
|
+
return { kind: "unknown", wellFormed: false, urls: [], sitemaps: [] };
|
|
1982
|
+
}
|
|
1983
|
+
let $;
|
|
1984
|
+
try {
|
|
1985
|
+
$ = cheerio2.load(trimmed, { xmlMode: true });
|
|
1986
|
+
} catch {
|
|
1987
|
+
return { kind: "unknown", wellFormed: false, urls: [], sitemaps: [] };
|
|
1988
|
+
}
|
|
1989
|
+
if ($("sitemapindex").length > 0) {
|
|
1990
|
+
const sitemaps = [];
|
|
1991
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1992
|
+
$("sitemap > loc").each((_, el) => {
|
|
1993
|
+
const loc = $(el).text().trim();
|
|
1994
|
+
if (loc && !seen.has(loc)) {
|
|
1995
|
+
seen.add(loc);
|
|
1996
|
+
sitemaps.push(loc);
|
|
1997
|
+
}
|
|
1998
|
+
});
|
|
1999
|
+
return { kind: "index", wellFormed: true, sitemaps };
|
|
2000
|
+
}
|
|
2001
|
+
if ($("urlset").length > 0 || $("url").length > 0) {
|
|
2002
|
+
const urls = [];
|
|
2003
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2004
|
+
$("url").each((_, el) => {
|
|
2005
|
+
const node = $(el);
|
|
2006
|
+
const loc = node.find("loc").first().text().trim();
|
|
2007
|
+
if (!loc || seen.has(loc)) return;
|
|
2008
|
+
seen.add(loc);
|
|
2009
|
+
const entry = { loc };
|
|
2010
|
+
const lastmod = node.find("lastmod").first().text().trim();
|
|
2011
|
+
if (lastmod) entry.lastmod = lastmod;
|
|
2012
|
+
const changefreq = node.find("changefreq").first().text().trim();
|
|
2013
|
+
if (changefreq) entry.changefreq = changefreq;
|
|
2014
|
+
const priority = node.find("priority").first().text().trim();
|
|
2015
|
+
if (priority) entry.priority = priority;
|
|
2016
|
+
urls.push(entry);
|
|
2017
|
+
});
|
|
2018
|
+
return { kind: "urlset", wellFormed: true, urls };
|
|
2019
|
+
}
|
|
2020
|
+
return { kind: "unknown", wellFormed: false, urls: [], sitemaps: [] };
|
|
2021
|
+
}
|
|
2022
|
+
|
|
2023
|
+
// src/lib/core/sitemap/discover.ts
|
|
2024
|
+
var DEFAULTS = {
|
|
2025
|
+
timeoutMs: 8e3,
|
|
2026
|
+
maxUrls: 5e3,
|
|
2027
|
+
maxSitemaps: 50,
|
|
2028
|
+
crawlDepth: 1,
|
|
2029
|
+
crawlMaxPages: 100
|
|
2030
|
+
};
|
|
2031
|
+
async function discoverSitemap(baseUrl, options = {}) {
|
|
2032
|
+
const origin = new URL(baseUrl).origin;
|
|
2033
|
+
const maxUrls = options.maxUrls ?? DEFAULTS.maxUrls;
|
|
2034
|
+
const maxSitemaps = options.maxSitemaps ?? DEFAULTS.maxSitemaps;
|
|
2035
|
+
const diagnostics = {
|
|
2036
|
+
found: false,
|
|
2037
|
+
status: 0,
|
|
2038
|
+
declaredInRobots: false,
|
|
2039
|
+
robotsFound: false,
|
|
2040
|
+
atWellKnownPath: false,
|
|
2041
|
+
wellFormed: false,
|
|
2042
|
+
isIndex: false,
|
|
2043
|
+
childSitemapCount: 0,
|
|
2044
|
+
childSitemapErrors: 0,
|
|
2045
|
+
urlCount: 0,
|
|
2046
|
+
warnings: []
|
|
2047
|
+
};
|
|
2048
|
+
const robots = await probeRobots(origin, { signal: options.signal });
|
|
2049
|
+
diagnostics.robotsFound = robots.found;
|
|
2050
|
+
const candidates = [];
|
|
2051
|
+
for (const declared of robots.sitemaps) {
|
|
2052
|
+
candidates.push({ url: declared, source: "robots", declaredInRobots: true });
|
|
2053
|
+
}
|
|
2054
|
+
candidates.push({
|
|
2055
|
+
url: new URL("/sitemap.xml", origin).toString(),
|
|
2056
|
+
source: "well-known",
|
|
2057
|
+
declaredInRobots: robots.sitemaps.includes(new URL("/sitemap.xml", origin).toString())
|
|
2058
|
+
});
|
|
2059
|
+
candidates.push({
|
|
2060
|
+
url: new URL("/sitemap_index.xml", origin).toString(),
|
|
2061
|
+
source: "well-known",
|
|
2062
|
+
declaredInRobots: false
|
|
2063
|
+
});
|
|
2064
|
+
for (const candidate of candidates) {
|
|
2065
|
+
const doc = await fetchDoc(candidate.url, options);
|
|
2066
|
+
if (!doc.body) {
|
|
2067
|
+
if (diagnostics.status === 0) diagnostics.status = doc.status;
|
|
2068
|
+
continue;
|
|
2069
|
+
}
|
|
2070
|
+
const parsed = parseSitemap(doc.body);
|
|
2071
|
+
if (parsed.kind === "unknown") {
|
|
2072
|
+
if (diagnostics.status === 0) diagnostics.status = doc.status;
|
|
2073
|
+
diagnostics.warnings.push(`Malformed or unrecognised sitemap at ${candidate.url}`);
|
|
2074
|
+
continue;
|
|
2075
|
+
}
|
|
2076
|
+
diagnostics.found = true;
|
|
2077
|
+
diagnostics.status = doc.status;
|
|
2078
|
+
diagnostics.wellFormed = true;
|
|
2079
|
+
diagnostics.sitemapUrl = candidate.url;
|
|
2080
|
+
diagnostics.declaredInRobots = candidate.declaredInRobots;
|
|
2081
|
+
diagnostics.atWellKnownPath = isWellKnownPath(candidate.url);
|
|
2082
|
+
const collected = [];
|
|
2083
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2084
|
+
let truncated = false;
|
|
2085
|
+
if (parsed.kind === "urlset") {
|
|
2086
|
+
truncated = pushEntries(collected, seen, parsed.urls, maxUrls);
|
|
2087
|
+
} else {
|
|
2088
|
+
diagnostics.isIndex = true;
|
|
2089
|
+
const children = parsed.sitemaps.slice(0, maxSitemaps);
|
|
2090
|
+
diagnostics.childSitemapCount = parsed.sitemaps.length;
|
|
2091
|
+
if (parsed.sitemaps.length > maxSitemaps) {
|
|
2092
|
+
truncated = true;
|
|
2093
|
+
diagnostics.warnings.push(
|
|
2094
|
+
`Sitemap index has ${parsed.sitemaps.length} children; only the first ${maxSitemaps} were followed.`
|
|
2095
|
+
);
|
|
2096
|
+
}
|
|
2097
|
+
for (const childUrl of children) {
|
|
2098
|
+
if (collected.length >= maxUrls) {
|
|
2099
|
+
truncated = true;
|
|
2100
|
+
break;
|
|
2101
|
+
}
|
|
2102
|
+
const childDoc = await fetchDoc(childUrl, options);
|
|
2103
|
+
if (!childDoc.body) {
|
|
2104
|
+
diagnostics.childSitemapErrors += 1;
|
|
2105
|
+
diagnostics.warnings.push(`Child sitemap unreachable: ${childUrl}`);
|
|
2106
|
+
continue;
|
|
2107
|
+
}
|
|
2108
|
+
const childParsed = parseSitemap(childDoc.body);
|
|
2109
|
+
if (childParsed.kind !== "urlset") {
|
|
2110
|
+
diagnostics.childSitemapErrors += 1;
|
|
2111
|
+
diagnostics.warnings.push(`Child sitemap not a urlset: ${childUrl}`);
|
|
2112
|
+
continue;
|
|
2113
|
+
}
|
|
2114
|
+
if (pushEntries(collected, seen, childParsed.urls, maxUrls)) truncated = true;
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
diagnostics.urlCount = collected.length;
|
|
2118
|
+
if (collected.length === 0 && options.crawlFallback !== false) {
|
|
2119
|
+
return crawlSite(baseUrl, origin, options, diagnostics);
|
|
2120
|
+
}
|
|
2121
|
+
return {
|
|
2122
|
+
origin,
|
|
2123
|
+
baseUrl,
|
|
2124
|
+
source: candidate.source,
|
|
2125
|
+
urls: collected,
|
|
2126
|
+
diagnostics,
|
|
2127
|
+
truncated
|
|
2128
|
+
};
|
|
2129
|
+
}
|
|
2130
|
+
if (options.crawlFallback !== false) {
|
|
2131
|
+
return crawlSite(baseUrl, origin, options, diagnostics);
|
|
2132
|
+
}
|
|
2133
|
+
diagnostics.warnings.push("No sitemap found.");
|
|
2134
|
+
return { origin, baseUrl, source: "well-known", urls: [], diagnostics, truncated: false };
|
|
2135
|
+
}
|
|
2136
|
+
async function crawlSite(baseUrl, origin, options, diagnostics) {
|
|
2137
|
+
const result = await crawl({
|
|
2138
|
+
entryUrl: baseUrl,
|
|
2139
|
+
depth: options.crawlDepth ?? DEFAULTS.crawlDepth,
|
|
2140
|
+
maxPages: options.crawlMaxPages ?? DEFAULTS.crawlMaxPages,
|
|
2141
|
+
inspectOptions: {
|
|
2142
|
+
mode: "static",
|
|
2143
|
+
probes: false,
|
|
2144
|
+
allowInsecureTls: options.allowInsecureTls,
|
|
2145
|
+
signal: options.signal
|
|
2146
|
+
}
|
|
2147
|
+
});
|
|
2148
|
+
const urls = dedupe(result.visited).map((loc) => ({ loc }));
|
|
2149
|
+
diagnostics.urlCount = urls.length;
|
|
2150
|
+
diagnostics.warnings.push(
|
|
2151
|
+
diagnostics.found ? "Sitemap found but lists no URLs \u2014 crawled the site instead." : "No sitemap found \u2014 URLs were discovered by crawling links."
|
|
2152
|
+
);
|
|
2153
|
+
return {
|
|
2154
|
+
origin,
|
|
2155
|
+
baseUrl,
|
|
2156
|
+
source: "crawl",
|
|
2157
|
+
urls,
|
|
2158
|
+
diagnostics,
|
|
2159
|
+
truncated: result.truncated
|
|
2160
|
+
};
|
|
2161
|
+
}
|
|
2162
|
+
function pushEntries(out, seen, entries, cap) {
|
|
2163
|
+
for (const entry of entries) {
|
|
2164
|
+
if (out.length >= cap) return true;
|
|
2165
|
+
if (seen.has(entry.loc)) continue;
|
|
2166
|
+
seen.add(entry.loc);
|
|
2167
|
+
out.push(entry);
|
|
2168
|
+
}
|
|
2169
|
+
return out.length >= cap;
|
|
2170
|
+
}
|
|
2171
|
+
function dedupe(urls) {
|
|
2172
|
+
return Array.from(new Set(urls));
|
|
2173
|
+
}
|
|
2174
|
+
function isWellKnownPath(url) {
|
|
2175
|
+
try {
|
|
2176
|
+
const path = new URL(url).pathname.toLowerCase();
|
|
2177
|
+
return path === "/sitemap.xml" || path === "/sitemap_index.xml";
|
|
2178
|
+
} catch {
|
|
2179
|
+
return false;
|
|
2180
|
+
}
|
|
2181
|
+
}
|
|
2182
|
+
async function fetchDoc(url, options) {
|
|
2183
|
+
const { signal, cleanup } = combineSignals(
|
|
2184
|
+
options.signal,
|
|
2185
|
+
options.timeoutMs ?? DEFAULTS.timeoutMs
|
|
2186
|
+
);
|
|
2187
|
+
const restoreTls = relaxTlsIfRequested2(options.allowInsecureTls);
|
|
2188
|
+
try {
|
|
2189
|
+
const res = await fetch(url, {
|
|
2190
|
+
signal,
|
|
2191
|
+
redirect: "follow",
|
|
2192
|
+
headers: {
|
|
2193
|
+
accept: "application/xml,text/xml,application/gzip;q=0.9,*/*;q=0.5"
|
|
2194
|
+
}
|
|
2195
|
+
});
|
|
2196
|
+
if (!res.ok) return { status: res.status };
|
|
2197
|
+
const encoding = res.headers.get("content-encoding")?.toLowerCase() ?? "";
|
|
2198
|
+
const looksGzip = url.toLowerCase().endsWith(".gz") || encoding.includes("gzip");
|
|
2199
|
+
if (looksGzip) {
|
|
2200
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
2201
|
+
try {
|
|
2202
|
+
return { status: res.status, body: gunzipSync(buf).toString("utf8") };
|
|
2203
|
+
} catch {
|
|
2204
|
+
return { status: res.status, body: buf.toString("utf8") };
|
|
2205
|
+
}
|
|
2206
|
+
}
|
|
2207
|
+
return { status: res.status, body: await res.text() };
|
|
2208
|
+
} catch {
|
|
2209
|
+
return { status: 0 };
|
|
2210
|
+
} finally {
|
|
2211
|
+
cleanup();
|
|
2212
|
+
restoreTls();
|
|
2213
|
+
}
|
|
2214
|
+
}
|
|
2215
|
+
function relaxTlsIfRequested2(enabled) {
|
|
2216
|
+
if (!enabled) return () => void 0;
|
|
2217
|
+
const previous = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
|
|
2218
|
+
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
|
|
2219
|
+
return () => {
|
|
2220
|
+
if (previous === void 0) delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
|
|
2221
|
+
else process.env.NODE_TLS_REJECT_UNAUTHORIZED = previous;
|
|
2222
|
+
};
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
// src/lib/core/net/fetch-url.ts
|
|
2226
|
+
var DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
|
2227
|
+
var DEFAULT_TIMEOUT_MS = 8e3;
|
|
2228
|
+
var DEFAULT_MAX_BYTES = 3 * 1024 * 1024;
|
|
2229
|
+
var DEFAULT_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
|
|
2230
|
+
var TEXTY = /(text\/|application\/(xhtml\+xml|xml|json|.*\+xml|.*\+json|javascript))/;
|
|
2231
|
+
async function fetchUrl(url, options = {}) {
|
|
2232
|
+
const method = options.method ?? "GET";
|
|
2233
|
+
const redirect = options.redirect ?? "follow";
|
|
2234
|
+
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
2235
|
+
const startedAt = Date.now();
|
|
2236
|
+
const { signal, cleanup } = combineSignals(
|
|
2237
|
+
options.signal,
|
|
2238
|
+
options.timeoutMs ?? DEFAULT_TIMEOUT_MS
|
|
2239
|
+
);
|
|
2240
|
+
const restoreTls = relaxTlsIfRequested3(options.allowInsecureTls);
|
|
2241
|
+
try {
|
|
2242
|
+
const res = await fetch(url, {
|
|
2243
|
+
method,
|
|
2244
|
+
signal,
|
|
2245
|
+
redirect,
|
|
2246
|
+
headers: {
|
|
2247
|
+
"user-agent": options.userAgent ?? DEFAULT_USER_AGENT,
|
|
2248
|
+
accept: options.accept ?? DEFAULT_ACCEPT
|
|
2249
|
+
}
|
|
2250
|
+
});
|
|
2251
|
+
const contentType = normaliseContentType(res.headers.get("content-type"));
|
|
2252
|
+
const result = {
|
|
2253
|
+
requestedUrl: url,
|
|
2254
|
+
finalUrl: res.url || url,
|
|
2255
|
+
status: res.status,
|
|
2256
|
+
redirected: res.redirected,
|
|
2257
|
+
redirectChain: [],
|
|
2258
|
+
contentType,
|
|
2259
|
+
durationMs: Date.now() - startedAt
|
|
2260
|
+
};
|
|
2261
|
+
const retryAfter = res.headers.get("retry-after");
|
|
2262
|
+
if (retryAfter) result.retryAfter = retryAfter;
|
|
2263
|
+
if (redirect === "manual" && isRedirectStatus(res.status)) {
|
|
2264
|
+
const location = res.headers.get("location");
|
|
2265
|
+
if (location) {
|
|
2266
|
+
try {
|
|
2267
|
+
const next = new URL(location, url).toString();
|
|
2268
|
+
result.finalUrl = next;
|
|
2269
|
+
result.redirectChain = [next];
|
|
2270
|
+
result.redirected = true;
|
|
2271
|
+
} catch {
|
|
2272
|
+
}
|
|
2273
|
+
}
|
|
2274
|
+
await res.body?.cancel().catch(() => void 0);
|
|
2275
|
+
return result;
|
|
2276
|
+
}
|
|
2277
|
+
if (method === "HEAD") {
|
|
2278
|
+
await res.body?.cancel().catch(() => void 0);
|
|
2279
|
+
return result;
|
|
2280
|
+
}
|
|
2281
|
+
if (contentType && !TEXTY.test(contentType)) {
|
|
2282
|
+
await res.body?.cancel().catch(() => void 0);
|
|
2283
|
+
return result;
|
|
2284
|
+
}
|
|
2285
|
+
const { text, truncated } = await readCappedText(res, maxBytes);
|
|
2286
|
+
result.body = text;
|
|
2287
|
+
if (truncated) result.truncated = true;
|
|
2288
|
+
return result;
|
|
2289
|
+
} catch (err) {
|
|
2290
|
+
return {
|
|
2291
|
+
requestedUrl: url,
|
|
2292
|
+
finalUrl: url,
|
|
2293
|
+
status: 0,
|
|
2294
|
+
redirected: false,
|
|
2295
|
+
redirectChain: [],
|
|
2296
|
+
reason: classifyError(err, options.signal),
|
|
2297
|
+
durationMs: Date.now() - startedAt
|
|
2298
|
+
};
|
|
2299
|
+
} finally {
|
|
2300
|
+
cleanup();
|
|
2301
|
+
restoreTls();
|
|
2302
|
+
}
|
|
2303
|
+
}
|
|
2304
|
+
function isRedirectStatus(status) {
|
|
2305
|
+
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
|
|
2306
|
+
}
|
|
2307
|
+
function normaliseContentType(raw) {
|
|
2308
|
+
if (!raw) return void 0;
|
|
2309
|
+
const semi = raw.indexOf(";");
|
|
2310
|
+
return (semi === -1 ? raw : raw.slice(0, semi)).trim().toLowerCase() || void 0;
|
|
2311
|
+
}
|
|
2312
|
+
async function readCappedText(res, maxBytes) {
|
|
2313
|
+
const reader = res.body?.getReader();
|
|
2314
|
+
if (!reader) {
|
|
2315
|
+
const text = await res.text();
|
|
2316
|
+
if (text.length > maxBytes) {
|
|
2317
|
+
return { text: text.slice(0, maxBytes), truncated: true };
|
|
2318
|
+
}
|
|
2319
|
+
return { text, truncated: false };
|
|
2320
|
+
}
|
|
2321
|
+
const chunks = [];
|
|
2322
|
+
let received = 0;
|
|
2323
|
+
let truncated = false;
|
|
2324
|
+
for (; ; ) {
|
|
2325
|
+
const { done, value } = await reader.read();
|
|
2326
|
+
if (done) break;
|
|
2327
|
+
if (!value) continue;
|
|
2328
|
+
chunks.push(value);
|
|
2329
|
+
received += value.byteLength;
|
|
2330
|
+
if (received >= maxBytes) {
|
|
2331
|
+
truncated = true;
|
|
2332
|
+
await reader.cancel().catch(() => void 0);
|
|
2333
|
+
break;
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
const buf = Buffer.concat(chunks.map((c) => Buffer.from(c)));
|
|
2337
|
+
const sliced = buf.byteLength > maxBytes ? buf.subarray(0, maxBytes) : buf;
|
|
2338
|
+
return { text: sliced.toString("utf8"), truncated };
|
|
2339
|
+
}
|
|
2340
|
+
function classifyError(err, callerSignal) {
|
|
2341
|
+
if (isAbortError(err)) {
|
|
2342
|
+
return callerSignal?.aborted ? "abort" : "timeout";
|
|
2343
|
+
}
|
|
2344
|
+
const code = errorCode(err);
|
|
2345
|
+
if (code) {
|
|
2346
|
+
if (code === "ENOTFOUND" || code === "EAI_AGAIN") return "dns";
|
|
2347
|
+
if (code.startsWith("CERT_") || code.includes("SELF_SIGNED") || code.includes("CERT")) {
|
|
2348
|
+
return "tls";
|
|
2349
|
+
}
|
|
2350
|
+
}
|
|
2351
|
+
const message = err instanceof Error ? err.message.toLowerCase() : String(err).toLowerCase();
|
|
2352
|
+
if (message.includes("certificate") || message.includes("tls") || message.includes("ssl")) {
|
|
2353
|
+
return "tls";
|
|
2354
|
+
}
|
|
2355
|
+
return "network";
|
|
2356
|
+
}
|
|
2357
|
+
function isAbortError(err) {
|
|
2358
|
+
return err instanceof Error && err.name === "AbortError";
|
|
2359
|
+
}
|
|
2360
|
+
function errorCode(err) {
|
|
2361
|
+
if (err && typeof err === "object") {
|
|
2362
|
+
const direct = err.code;
|
|
2363
|
+
if (typeof direct === "string") return direct;
|
|
2364
|
+
const cause = err.cause;
|
|
2365
|
+
if (cause && typeof cause === "object") {
|
|
2366
|
+
const causeCode = cause.code;
|
|
2367
|
+
if (typeof causeCode === "string") return causeCode;
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
return void 0;
|
|
2371
|
+
}
|
|
2372
|
+
function relaxTlsIfRequested3(enabled) {
|
|
2373
|
+
if (!enabled) return () => void 0;
|
|
2374
|
+
const previous = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
|
|
2375
|
+
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
|
|
2376
|
+
return () => {
|
|
2377
|
+
if (previous === void 0) delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
|
|
2378
|
+
else process.env.NODE_TLS_REJECT_UNAUTHORIZED = previous;
|
|
2379
|
+
};
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
// src/lib/core/links/extract.ts
|
|
2383
|
+
import * as cheerio3 from "cheerio";
|
|
2384
|
+
var ANCHOR_SPEC = { selector: "a[href]", attr: "href", source: "a" };
|
|
2385
|
+
var ASSET_SPECS = [
|
|
2386
|
+
{ selector: "img[src]", attr: "src", source: "img" },
|
|
2387
|
+
{ selector: "script[src]", attr: "src", source: "script" },
|
|
2388
|
+
{ selector: "link[href]", attr: "href", source: "link" },
|
|
2389
|
+
{ selector: "iframe[src]", attr: "src", source: "iframe" }
|
|
2390
|
+
];
|
|
2391
|
+
function extractLinks(html, options) {
|
|
2392
|
+
const $ = cheerio3.load(html);
|
|
2393
|
+
const base = resolveBase2($, options.baseUrl);
|
|
2394
|
+
const origin = safeOrigin(options.baseUrl);
|
|
2395
|
+
const specs = options.includeAssets ? [ANCHOR_SPEC, ...ASSET_SPECS] : [ANCHOR_SPEC];
|
|
2396
|
+
const out = [];
|
|
2397
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2398
|
+
for (const spec of specs) {
|
|
2399
|
+
$(spec.selector).each((_, el) => {
|
|
2400
|
+
const raw = $(el).attr(spec.attr);
|
|
2401
|
+
if (raw === void 0) return;
|
|
2402
|
+
const rawHref = raw.trim();
|
|
2403
|
+
if (!rawHref) return;
|
|
2404
|
+
const ref = buildRef($, el, rawHref, spec, base, origin);
|
|
2405
|
+
if (!ref) return;
|
|
2406
|
+
const dedupeKey = `${ref.source}|${ref.url}`;
|
|
2407
|
+
if (seen.has(dedupeKey)) return;
|
|
2408
|
+
seen.add(dedupeKey);
|
|
2409
|
+
out.push(ref);
|
|
2410
|
+
});
|
|
2411
|
+
}
|
|
2412
|
+
return out;
|
|
2413
|
+
}
|
|
2414
|
+
function buildRef($, el, rawHref, spec, base, origin) {
|
|
2415
|
+
const fragment = extractFragment(rawHref);
|
|
2416
|
+
const canonical = canonicaliseUrl(rawHref, base, { preserveTrailingSlash: true });
|
|
2417
|
+
const url = canonical ?? rawHref;
|
|
2418
|
+
const kind = canonical ? classify(canonical, origin) : "external";
|
|
2419
|
+
const ref = {
|
|
2420
|
+
rawHref,
|
|
2421
|
+
url,
|
|
2422
|
+
kind,
|
|
2423
|
+
source: spec.source,
|
|
2424
|
+
rel: spec.source === "a" || spec.source === "link" ? parseRel($(el).attr("rel")) : []
|
|
2425
|
+
};
|
|
2426
|
+
if (spec.source === "a") {
|
|
2427
|
+
const text = $(el).text().trim();
|
|
2428
|
+
if (text) ref.anchorText = text;
|
|
2429
|
+
}
|
|
2430
|
+
if (fragment) ref.fragment = fragment;
|
|
2431
|
+
return ref;
|
|
2432
|
+
}
|
|
2433
|
+
function extractFragment(rawHref) {
|
|
2434
|
+
const hash = rawHref.indexOf("#");
|
|
2435
|
+
if (hash === -1) return void 0;
|
|
2436
|
+
const frag = rawHref.slice(hash);
|
|
2437
|
+
return frag.length > 1 ? frag : void 0;
|
|
2438
|
+
}
|
|
2439
|
+
function parseRel(rel) {
|
|
2440
|
+
if (!rel) return [];
|
|
2441
|
+
return rel.split(/\s+/).map((t) => t.trim().toLowerCase()).filter(Boolean);
|
|
2442
|
+
}
|
|
2443
|
+
function classify(url, origin) {
|
|
2444
|
+
if (!origin) return "external";
|
|
2445
|
+
return safeOrigin(url) === origin ? "internal" : "external";
|
|
2446
|
+
}
|
|
2447
|
+
function safeOrigin(url) {
|
|
2448
|
+
try {
|
|
2449
|
+
return new URL(url).origin;
|
|
2450
|
+
} catch {
|
|
2451
|
+
return null;
|
|
2452
|
+
}
|
|
2453
|
+
}
|
|
2454
|
+
function resolveBase2($, pageUrl) {
|
|
2455
|
+
const declared = $("base[href]").first().attr("href")?.trim();
|
|
2456
|
+
if (declared) {
|
|
2457
|
+
try {
|
|
2458
|
+
return new URL(declared, pageUrl).toString();
|
|
2459
|
+
} catch {
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2462
|
+
return pageUrl;
|
|
2463
|
+
}
|
|
2464
|
+
|
|
2465
|
+
// src/lib/core/links/classify.ts
|
|
2466
|
+
function classifyLink(input) {
|
|
2467
|
+
if (input.status === 0) return "broken";
|
|
2468
|
+
if (input.loop) return "broken";
|
|
2469
|
+
if (input.softNotFound) return "warning";
|
|
2470
|
+
if (input.status === 403 || input.status === 429) return "blocked";
|
|
2471
|
+
if (input.status >= 400) return "broken";
|
|
2472
|
+
if (input.status >= 200 && input.status < 300) {
|
|
2473
|
+
return input.redirected ? "redirect" : "ok";
|
|
2474
|
+
}
|
|
2475
|
+
return "broken";
|
|
2476
|
+
}
|
|
2477
|
+
|
|
2478
|
+
// src/lib/core/links/check.ts
|
|
2479
|
+
var DEFAULTS2 = {
|
|
2480
|
+
timeoutMs: 8e3,
|
|
2481
|
+
maxRedirects: 10,
|
|
2482
|
+
retries: 1,
|
|
2483
|
+
backoffBaseMs: 300,
|
|
2484
|
+
maxBackoffMs: 3e3,
|
|
2485
|
+
softBodyBytes: 16 * 1024
|
|
2486
|
+
};
|
|
2487
|
+
var NOT_FOUND_RE = /\b(not\s+found|page\s+doesn'?t\s+exist|404|no\s+such\s+page)\b/i;
|
|
2488
|
+
var SOFT_404_BODY_LIMIT = 2048;
|
|
2489
|
+
async function checkLink(url, options = {}) {
|
|
2490
|
+
const checkedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
2491
|
+
const started = Date.now();
|
|
2492
|
+
const scheme = schemeOf(url);
|
|
2493
|
+
if (scheme !== "http" && scheme !== "https") {
|
|
2494
|
+
return {
|
|
2495
|
+
url,
|
|
2496
|
+
finalUrl: url,
|
|
2497
|
+
status: 0,
|
|
2498
|
+
verdict: "skipped",
|
|
2499
|
+
method: "GET",
|
|
2500
|
+
redirectChain: [],
|
|
2501
|
+
reason: scheme ? `unsupported scheme: ${scheme}` : "not a URL",
|
|
2502
|
+
checkedAt,
|
|
2503
|
+
durationMs: Date.now() - started
|
|
2504
|
+
};
|
|
2505
|
+
}
|
|
2506
|
+
const outcome = await probeWithRetry(url, options);
|
|
2507
|
+
const softNotFound = isSoftNotFound(outcome);
|
|
2508
|
+
const verdict = classifyLink({
|
|
2509
|
+
status: outcome.status,
|
|
2510
|
+
redirected: outcome.redirectChain.length > 0,
|
|
2511
|
+
reason: outcome.reason,
|
|
2512
|
+
softNotFound,
|
|
2513
|
+
loop: outcome.loop
|
|
2514
|
+
});
|
|
2515
|
+
return {
|
|
2516
|
+
url,
|
|
2517
|
+
finalUrl: outcome.finalUrl,
|
|
2518
|
+
status: outcome.status,
|
|
2519
|
+
verdict,
|
|
2520
|
+
method: outcome.method,
|
|
2521
|
+
redirectChain: outcome.redirectChain,
|
|
2522
|
+
reason: describeReason(outcome, softNotFound),
|
|
2523
|
+
checkedAt,
|
|
2524
|
+
durationMs: Date.now() - started
|
|
2525
|
+
};
|
|
2526
|
+
}
|
|
2527
|
+
async function probeWithRetry(url, options) {
|
|
2528
|
+
const retries = options.retries ?? DEFAULTS2.retries;
|
|
2529
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
2530
|
+
let outcome = await probeWithHeadFallback(url, options);
|
|
2531
|
+
for (let attempt = 0; attempt < retries; attempt++) {
|
|
2532
|
+
if (!isTransient(outcome)) break;
|
|
2533
|
+
if (options.signal?.aborted) break;
|
|
2534
|
+
const delay = backoffDelay(outcome, attempt, options);
|
|
2535
|
+
if (delay > 0) await sleep(delay);
|
|
2536
|
+
outcome = await probeWithHeadFallback(url, options);
|
|
2537
|
+
}
|
|
2538
|
+
return outcome;
|
|
2539
|
+
}
|
|
2540
|
+
async function probeWithHeadFallback(url, options) {
|
|
2541
|
+
const head = await followRedirects(url, "HEAD", options);
|
|
2542
|
+
if (headRejected(head) || head.status >= 200 && head.status < 300) {
|
|
2543
|
+
const get = await followRedirects(url, "GET", options);
|
|
2544
|
+
if (get.status === 0 && head.status !== 0 && !headRejected(head)) return head;
|
|
2545
|
+
return get;
|
|
2546
|
+
}
|
|
2547
|
+
return head;
|
|
2548
|
+
}
|
|
2549
|
+
function headRejected(o) {
|
|
2550
|
+
return o.status === 405 || o.status === 501 || o.status === 403 || o.status === 0;
|
|
2551
|
+
}
|
|
2552
|
+
async function followRedirects(url, method, options) {
|
|
2553
|
+
const maxRedirects = options.maxRedirects ?? DEFAULTS2.maxRedirects;
|
|
2554
|
+
const chain = [];
|
|
2555
|
+
const visited = /* @__PURE__ */ new Set([url]);
|
|
2556
|
+
let current = url;
|
|
2557
|
+
for (let hop = 0; hop <= maxRedirects; hop++) {
|
|
2558
|
+
const res = await fetchUrl(current, {
|
|
2559
|
+
method,
|
|
2560
|
+
redirect: "manual",
|
|
2561
|
+
timeoutMs: options.timeoutMs ?? DEFAULTS2.timeoutMs,
|
|
2562
|
+
allowInsecureTls: options.allowInsecureTls,
|
|
2563
|
+
userAgent: options.userAgent,
|
|
2564
|
+
maxBytes: method === "GET" ? options.softBodyBytes ?? DEFAULTS2.softBodyBytes : void 0
|
|
2565
|
+
});
|
|
2566
|
+
if (res.status === 0) {
|
|
2567
|
+
return {
|
|
2568
|
+
status: 0,
|
|
2569
|
+
finalUrl: current,
|
|
2570
|
+
redirectChain: chain,
|
|
2571
|
+
reason: res.reason,
|
|
2572
|
+
loop: false,
|
|
2573
|
+
method
|
|
2574
|
+
};
|
|
2575
|
+
}
|
|
2576
|
+
const next = res.redirectChain[0];
|
|
2577
|
+
if (next) {
|
|
2578
|
+
chain.push(next);
|
|
2579
|
+
if (visited.has(next)) {
|
|
2580
|
+
return { status: res.status, finalUrl: next, redirectChain: chain, loop: true, method };
|
|
2581
|
+
}
|
|
2582
|
+
visited.add(next);
|
|
2583
|
+
current = next;
|
|
2584
|
+
continue;
|
|
2585
|
+
}
|
|
2586
|
+
return {
|
|
2587
|
+
status: res.status,
|
|
2588
|
+
finalUrl: res.finalUrl,
|
|
2589
|
+
redirectChain: chain,
|
|
2590
|
+
loop: false,
|
|
2591
|
+
body: res.body,
|
|
2592
|
+
retryAfter: res.retryAfter,
|
|
2593
|
+
method
|
|
2594
|
+
};
|
|
2595
|
+
}
|
|
2596
|
+
return { status: 0, finalUrl: current, redirectChain: chain, loop: true, method };
|
|
2597
|
+
}
|
|
2598
|
+
function isSoftNotFound(o) {
|
|
2599
|
+
if (o.status < 200 || o.status >= 300) return false;
|
|
2600
|
+
if (o.body === void 0) return false;
|
|
2601
|
+
if (o.body.length > SOFT_404_BODY_LIMIT) return false;
|
|
2602
|
+
return NOT_FOUND_RE.test(o.body);
|
|
2603
|
+
}
|
|
2604
|
+
function isTransient(o) {
|
|
2605
|
+
if (o.status === 0) return true;
|
|
2606
|
+
if (o.status === 429) return true;
|
|
2607
|
+
if (o.status >= 500) return true;
|
|
2608
|
+
return false;
|
|
2609
|
+
}
|
|
2610
|
+
function backoffDelay(o, attempt, options) {
|
|
2611
|
+
const max = options.maxBackoffMs ?? DEFAULTS2.maxBackoffMs;
|
|
2612
|
+
const retryAfterMs = parseRetryAfter(o.retryAfter);
|
|
2613
|
+
if (retryAfterMs !== void 0) return Math.min(retryAfterMs, max);
|
|
2614
|
+
const base = options.backoffBaseMs ?? DEFAULTS2.backoffBaseMs;
|
|
2615
|
+
const exp = base * 2 ** attempt;
|
|
2616
|
+
const jitter = Math.floor(Math.random() * base);
|
|
2617
|
+
return Math.min(exp + jitter, max);
|
|
2618
|
+
}
|
|
2619
|
+
function parseRetryAfter(value) {
|
|
2620
|
+
if (!value) return void 0;
|
|
2621
|
+
const seconds = Number(value);
|
|
2622
|
+
if (Number.isFinite(seconds)) return Math.max(0, seconds * 1e3);
|
|
2623
|
+
const date = Date.parse(value);
|
|
2624
|
+
if (!Number.isNaN(date)) return Math.max(0, date - Date.now());
|
|
2625
|
+
return void 0;
|
|
2626
|
+
}
|
|
2627
|
+
function describeReason(o, softNotFound) {
|
|
2628
|
+
if (o.reason) return o.reason;
|
|
2629
|
+
if (o.loop) return "redirect loop";
|
|
2630
|
+
if (softNotFound) return "soft-404";
|
|
2631
|
+
if (o.status === 429) return "429 rate-limited";
|
|
2632
|
+
if (o.status === 403) return "403 forbidden";
|
|
2633
|
+
if (o.status >= 400) return `${o.status}`;
|
|
2634
|
+
return void 0;
|
|
2635
|
+
}
|
|
2636
|
+
function schemeOf(url) {
|
|
2637
|
+
try {
|
|
2638
|
+
return new URL(url).protocol.replace(/:$/, "").toLowerCase();
|
|
2639
|
+
} catch {
|
|
2640
|
+
return null;
|
|
2641
|
+
}
|
|
2642
|
+
}
|
|
2643
|
+
function defaultSleep(ms) {
|
|
2644
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2645
|
+
}
|
|
2646
|
+
|
|
2647
|
+
// src/lib/core/links/types.ts
|
|
2648
|
+
function emptyVerdictSummary() {
|
|
2649
|
+
return { ok: 0, redirect: 0, broken: 0, blocked: 0, warning: 0, skipped: 0 };
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
// src/lib/core/links/audit.ts
|
|
2653
|
+
var DEFAULTS3 = {
|
|
2654
|
+
scanConcurrency: 4,
|
|
2655
|
+
checkConcurrency: 8,
|
|
2656
|
+
maxPerHost: 3,
|
|
2657
|
+
timeoutMs: 8e3,
|
|
2658
|
+
maxPages: 500,
|
|
2659
|
+
maxLinks: 1e4,
|
|
2660
|
+
maxScanBytes: 3 * 1024 * 1024
|
|
2661
|
+
};
|
|
2662
|
+
async function runLinkAudit(discovery, options = {}) {
|
|
2663
|
+
const maxPages = options.maxPages ?? DEFAULTS3.maxPages;
|
|
2664
|
+
const maxLinks = options.maxLinks ?? DEFAULTS3.maxLinks;
|
|
2665
|
+
const checkExternal = options.checkExternal !== false;
|
|
2666
|
+
const warnings = [];
|
|
2667
|
+
let truncated = false;
|
|
2668
|
+
const allPages = discovery.urls.map((u) => u.loc);
|
|
2669
|
+
const pageUrls = allPages.slice(0, maxPages);
|
|
2670
|
+
if (allPages.length > maxPages) {
|
|
2671
|
+
truncated = true;
|
|
2672
|
+
warnings.push(`Site has ${allPages.length} pages; only the first ${maxPages} were scanned.`);
|
|
2673
|
+
}
|
|
2674
|
+
const occurrences = [];
|
|
2675
|
+
let pagesScanned = 0;
|
|
2676
|
+
let pagesFailed = 0;
|
|
2677
|
+
let scanDone = 0;
|
|
2678
|
+
await mapWithHostCaps(
|
|
2679
|
+
pageUrls,
|
|
2680
|
+
{
|
|
2681
|
+
concurrency: options.scanConcurrency ?? DEFAULTS3.scanConcurrency,
|
|
2682
|
+
perHost: Infinity,
|
|
2683
|
+
signal: options.signal
|
|
2684
|
+
},
|
|
2685
|
+
async (pageUrl) => {
|
|
2686
|
+
const res = await fetchUrl(pageUrl, {
|
|
2687
|
+
method: "GET",
|
|
2688
|
+
timeoutMs: options.timeoutMs ?? DEFAULTS3.timeoutMs,
|
|
2689
|
+
allowInsecureTls: options.allowInsecureTls,
|
|
2690
|
+
userAgent: options.userAgent,
|
|
2691
|
+
maxBytes: options.maxScanBytes ?? DEFAULTS3.maxScanBytes,
|
|
2692
|
+
signal: options.signal
|
|
2693
|
+
});
|
|
2694
|
+
if (res.status === 0 || res.status >= 400 || res.body === void 0) {
|
|
2695
|
+
pagesFailed += 1;
|
|
2696
|
+
} else {
|
|
2697
|
+
pagesScanned += 1;
|
|
2698
|
+
const refs = extractLinks(res.body, {
|
|
2699
|
+
baseUrl: res.finalUrl,
|
|
2700
|
+
includeAssets: options.includeAssets
|
|
2701
|
+
});
|
|
2702
|
+
for (const ref of refs) occurrences.push({ pageUrl, ref });
|
|
2703
|
+
}
|
|
2704
|
+
scanDone += 1;
|
|
2705
|
+
options.onProgress?.({ phase: "scan", done: scanDone, total: pageUrls.length });
|
|
2706
|
+
},
|
|
2707
|
+
() => "scan"
|
|
2708
|
+
);
|
|
2709
|
+
const uniqueUrls = [];
|
|
2710
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2711
|
+
for (const occ of occurrences) {
|
|
2712
|
+
const url = occ.ref.url;
|
|
2713
|
+
if (seen.has(url)) continue;
|
|
2714
|
+
if (!checkExternal && occ.ref.kind === "external") continue;
|
|
2715
|
+
if (uniqueUrls.length >= maxLinks) {
|
|
2716
|
+
truncated = true;
|
|
2717
|
+
warnings.push(`More than ${maxLinks} unique links found; the remainder were not checked.`);
|
|
2718
|
+
break;
|
|
2719
|
+
}
|
|
2720
|
+
seen.add(url);
|
|
2721
|
+
uniqueUrls.push(url);
|
|
2722
|
+
}
|
|
2723
|
+
const checks = {};
|
|
2724
|
+
let checkDone = 0;
|
|
2725
|
+
await mapWithHostCaps(
|
|
2726
|
+
uniqueUrls,
|
|
2727
|
+
{
|
|
2728
|
+
concurrency: options.checkConcurrency ?? DEFAULTS3.checkConcurrency,
|
|
2729
|
+
perHost: options.maxPerHost ?? DEFAULTS3.maxPerHost,
|
|
2730
|
+
signal: options.signal
|
|
2731
|
+
},
|
|
2732
|
+
async (url) => {
|
|
2733
|
+
const check = await checkLink(url, {
|
|
2734
|
+
timeoutMs: options.timeoutMs ?? DEFAULTS3.timeoutMs,
|
|
2735
|
+
allowInsecureTls: options.allowInsecureTls,
|
|
2736
|
+
userAgent: options.userAgent,
|
|
2737
|
+
signal: options.signal,
|
|
2738
|
+
sleep: options.sleep
|
|
2739
|
+
});
|
|
2740
|
+
checks[url] = check;
|
|
2741
|
+
checkDone += 1;
|
|
2742
|
+
options.onProgress?.({ phase: "check", done: checkDone, total: uniqueUrls.length });
|
|
2743
|
+
},
|
|
2744
|
+
(url) => hostOf(url)
|
|
2745
|
+
);
|
|
2746
|
+
const summary = emptyVerdictSummary();
|
|
2747
|
+
for (const check of Object.values(checks)) summary[check.verdict] += 1;
|
|
2748
|
+
const brokenByPage = buildBrokenByPage(occurrences, checks);
|
|
2749
|
+
return {
|
|
2750
|
+
origin: discovery.origin,
|
|
2751
|
+
baseUrl: discovery.baseUrl,
|
|
2752
|
+
pagesScanned,
|
|
2753
|
+
occurrences,
|
|
2754
|
+
checks,
|
|
2755
|
+
summary,
|
|
2756
|
+
brokenByPage,
|
|
2757
|
+
truncated,
|
|
2758
|
+
diagnostics: { pagesFailed, warnings }
|
|
2759
|
+
};
|
|
2760
|
+
}
|
|
2761
|
+
function buildBrokenByPage(occurrences, checks) {
|
|
2762
|
+
const byPage = /* @__PURE__ */ new Map();
|
|
2763
|
+
for (const occ of occurrences) {
|
|
2764
|
+
const check = checks[occ.ref.url];
|
|
2765
|
+
if (!check) continue;
|
|
2766
|
+
if (check.verdict === "ok" || check.verdict === "redirect" || check.verdict === "skipped") {
|
|
2767
|
+
continue;
|
|
2768
|
+
}
|
|
2769
|
+
let page = byPage.get(occ.pageUrl);
|
|
2770
|
+
if (!page) {
|
|
2771
|
+
page = /* @__PURE__ */ new Map();
|
|
2772
|
+
byPage.set(occ.pageUrl, page);
|
|
2773
|
+
}
|
|
2774
|
+
page.set(check.url, check);
|
|
2775
|
+
}
|
|
2776
|
+
return Array.from(byPage.entries()).map(([pageUrl, map]) => ({ pageUrl, broken: Array.from(map.values()) })).sort((a, b) => b.broken.length - a.broken.length);
|
|
2777
|
+
}
|
|
2778
|
+
function hostOf(url) {
|
|
2779
|
+
try {
|
|
2780
|
+
return new URL(url).host;
|
|
2781
|
+
} catch {
|
|
2782
|
+
return url;
|
|
2783
|
+
}
|
|
2784
|
+
}
|
|
2785
|
+
function mapWithHostCaps(items, opts, worker, hostOf2) {
|
|
2786
|
+
const concurrency = Math.max(1, opts.concurrency);
|
|
2787
|
+
const perHost = opts.perHost;
|
|
2788
|
+
return new Promise((resolve) => {
|
|
2789
|
+
if (items.length === 0) {
|
|
2790
|
+
resolve();
|
|
2791
|
+
return;
|
|
2792
|
+
}
|
|
2793
|
+
const pending = items.map((item) => ({ item, host: hostOf2(item) }));
|
|
2794
|
+
const hostActive = /* @__PURE__ */ new Map();
|
|
2795
|
+
let inflight = 0;
|
|
2796
|
+
let settled = false;
|
|
2797
|
+
const finish = () => {
|
|
2798
|
+
if (!settled) {
|
|
2799
|
+
settled = true;
|
|
2800
|
+
resolve();
|
|
2801
|
+
}
|
|
2802
|
+
};
|
|
2803
|
+
const pump = () => {
|
|
2804
|
+
if (settled) return;
|
|
2805
|
+
if (opts.signal?.aborted) {
|
|
2806
|
+
if (inflight === 0) finish();
|
|
2807
|
+
return;
|
|
2808
|
+
}
|
|
2809
|
+
while (inflight < concurrency) {
|
|
2810
|
+
let pickIndex = -1;
|
|
2811
|
+
for (let k = 0; k < pending.length; k++) {
|
|
2812
|
+
const entry = pending[k];
|
|
2813
|
+
if ((hostActive.get(entry.host) ?? 0) < perHost) {
|
|
2814
|
+
pickIndex = k;
|
|
2815
|
+
break;
|
|
2816
|
+
}
|
|
2817
|
+
}
|
|
2818
|
+
if (pickIndex === -1) break;
|
|
2819
|
+
const [picked] = pending.splice(pickIndex, 1);
|
|
2820
|
+
const host = picked.host;
|
|
2821
|
+
hostActive.set(host, (hostActive.get(host) ?? 0) + 1);
|
|
2822
|
+
inflight += 1;
|
|
2823
|
+
void worker(picked.item).catch(() => void 0).finally(() => {
|
|
2824
|
+
inflight -= 1;
|
|
2825
|
+
hostActive.set(host, (hostActive.get(host) ?? 1) - 1);
|
|
2826
|
+
if (pending.length === 0 && inflight === 0) finish();
|
|
2827
|
+
else pump();
|
|
2828
|
+
});
|
|
2829
|
+
}
|
|
2830
|
+
if (pending.length === 0 && inflight === 0) finish();
|
|
2831
|
+
};
|
|
2832
|
+
pump();
|
|
2833
|
+
});
|
|
2834
|
+
}
|
|
2835
|
+
|
|
2836
|
+
// src/lib/core/net/normalize-url.ts
|
|
2837
|
+
var SCHEME_PATTERN = /^([a-zA-Z][a-zA-Z0-9+.-]*):\/\//;
|
|
2838
|
+
function normalizeInputUrl(input) {
|
|
2839
|
+
const trimmed = input.trim();
|
|
2840
|
+
if (!trimmed) return { ok: false };
|
|
2841
|
+
const schemeMatch = trimmed.match(SCHEME_PATTERN);
|
|
2842
|
+
let candidate;
|
|
2843
|
+
if (schemeMatch) {
|
|
2844
|
+
const scheme = (schemeMatch[1] ?? "").toLowerCase();
|
|
2845
|
+
if (scheme !== "http" && scheme !== "https") return { ok: false };
|
|
2846
|
+
candidate = trimmed;
|
|
2847
|
+
} else {
|
|
2848
|
+
candidate = `https://${trimmed}`;
|
|
2849
|
+
}
|
|
2850
|
+
let parsed;
|
|
2851
|
+
try {
|
|
2852
|
+
parsed = new URL(candidate);
|
|
2853
|
+
} catch {
|
|
2854
|
+
return { ok: false };
|
|
2855
|
+
}
|
|
2856
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return { ok: false };
|
|
2857
|
+
if (!parsed.hostname) return { ok: false };
|
|
2858
|
+
return { ok: true, url: candidate };
|
|
2859
|
+
}
|
|
2860
|
+
|
|
2861
|
+
// src/report/build.ts
|
|
2862
|
+
function isOkStatus(status) {
|
|
2863
|
+
return status >= 200 && status < 300;
|
|
2864
|
+
}
|
|
2865
|
+
function isHtmlPage(page) {
|
|
2866
|
+
const type = page.fetch.contentType?.trim().toLowerCase();
|
|
2867
|
+
if (!type) return true;
|
|
2868
|
+
return type === "text/html" || type === "application/xhtml+xml";
|
|
2869
|
+
}
|
|
2870
|
+
function localeOfUrl(url) {
|
|
2871
|
+
try {
|
|
2872
|
+
const { pathname } = new URL(url);
|
|
2873
|
+
const first = pathname.split("/").filter(Boolean)[0];
|
|
2874
|
+
if (first && looksLikeLocaleSegment(first)) return first;
|
|
2875
|
+
} catch {
|
|
2876
|
+
}
|
|
2877
|
+
return null;
|
|
2878
|
+
}
|
|
2879
|
+
function deriveTranslationHoles(matrix, ignoreRoutes = []) {
|
|
2880
|
+
const realLocales = matrix.locales.filter((l) => l !== "x-default");
|
|
2881
|
+
if (realLocales.length < 2) return [];
|
|
2882
|
+
const holes = [];
|
|
2883
|
+
for (const route of matrix.routes) {
|
|
2884
|
+
if (ignoreRoutes.length > 0 && matchesAny(route, [...ignoreRoutes])) continue;
|
|
2885
|
+
const present = [];
|
|
2886
|
+
const missing = [];
|
|
2887
|
+
for (const locale of realLocales) {
|
|
2888
|
+
const cell = matrix.cells[route]?.[locale];
|
|
2889
|
+
if (cell && cell.url) present.push(locale);
|
|
2890
|
+
else missing.push(locale);
|
|
2891
|
+
}
|
|
2892
|
+
if (present.length > 0 && missing.length > 0) {
|
|
2893
|
+
holes.push({
|
|
2894
|
+
id: fingerprint("i18n", "hole", route),
|
|
2895
|
+
route,
|
|
2896
|
+
presentLocales: present,
|
|
2897
|
+
missingLocales: missing
|
|
2898
|
+
});
|
|
2899
|
+
}
|
|
2900
|
+
}
|
|
2901
|
+
return holes;
|
|
2902
|
+
}
|
|
2903
|
+
function exitCode(report, failOn = "warning") {
|
|
2904
|
+
if (failOn === "never") return 0;
|
|
2905
|
+
if (report.summary.verdict === "green") return 0;
|
|
2906
|
+
if (failOn === "error") return report.summary.verdict === "red" ? 1 : 0;
|
|
2907
|
+
return 1;
|
|
2908
|
+
}
|
|
2909
|
+
function dropCanonicalDuplicates(pages) {
|
|
2910
|
+
const crawled = new Set(pages.map((p) => routeKey(p.fetch.finalUrl)));
|
|
2911
|
+
const kept = pages.filter((page) => {
|
|
2912
|
+
const canonical = page.links.canonical;
|
|
2913
|
+
if (!canonical) return true;
|
|
2914
|
+
const self = routeKey(page.fetch.finalUrl);
|
|
2915
|
+
let target;
|
|
2916
|
+
try {
|
|
2917
|
+
target = routeKey(new URL(canonical, page.fetch.finalUrl).toString());
|
|
2918
|
+
} catch {
|
|
2919
|
+
return true;
|
|
2920
|
+
}
|
|
2921
|
+
if (target === self) return true;
|
|
2922
|
+
return !crawled.has(target);
|
|
2923
|
+
});
|
|
2924
|
+
return { kept, dropped: pages.length - kept.length };
|
|
2925
|
+
}
|
|
2926
|
+
async function runAudit(inputUrl, options = {}) {
|
|
2927
|
+
const normalized = normalizeInputUrl(inputUrl);
|
|
2928
|
+
if (!normalized.ok) {
|
|
2929
|
+
throw new Error(`"${inputUrl}" is not a valid http(s) URL.`);
|
|
2930
|
+
}
|
|
2931
|
+
const entry = normalized.url;
|
|
2932
|
+
const origin = new URL(entry).origin;
|
|
2933
|
+
const depth = options.depth ?? 2;
|
|
2934
|
+
const maxPages = options.maxPages ?? 200;
|
|
2935
|
+
const checkExternal = options.checkExternal !== false;
|
|
2936
|
+
const discovery = options.noSitemap ? void 0 : await discoverSitemap(entry, {
|
|
2937
|
+
signal: options.signal,
|
|
2938
|
+
timeoutMs: options.timeoutMs,
|
|
2939
|
+
allowInsecureTls: options.allowInsecureTls,
|
|
2940
|
+
crawlFallback: false
|
|
2941
|
+
}).catch(() => void 0);
|
|
2942
|
+
const sitemapUrls = (discovery?.urls ?? []).map((u) => u.loc);
|
|
2943
|
+
const robots = await probeRobots(origin, {
|
|
2944
|
+
signal: options.signal,
|
|
2945
|
+
timeoutMs: options.timeoutMs
|
|
2946
|
+
}).catch(() => void 0);
|
|
2947
|
+
const crawlResult = await crawl({
|
|
2948
|
+
entryUrl: entry,
|
|
2949
|
+
seedUrls: sitemapUrls,
|
|
2950
|
+
depth,
|
|
2951
|
+
maxPages,
|
|
2952
|
+
include: options.include,
|
|
2953
|
+
exclude: options.exclude,
|
|
2954
|
+
inspectOptions: {
|
|
2955
|
+
mode: options.static ? "static" : "auto",
|
|
2956
|
+
probes: false,
|
|
2957
|
+
allowInsecureTls: options.allowInsecureTls,
|
|
2958
|
+
timeoutMs: options.timeoutMs,
|
|
2959
|
+
signal: options.signal
|
|
2960
|
+
},
|
|
2961
|
+
onPage: (page, ctx) => options.onProgress?.({
|
|
2962
|
+
phase: "crawl",
|
|
2963
|
+
done: ctx.visited,
|
|
2964
|
+
total: ctx.visited + ctx.queued,
|
|
2965
|
+
url: page.fetch.finalUrl,
|
|
2966
|
+
status: page.fetch.status
|
|
2967
|
+
})
|
|
2968
|
+
});
|
|
2969
|
+
const pages = crawlResult.pages;
|
|
2970
|
+
const warnings = crawlResult.errors.map((e) => `crawl: ${e.url} \u2014 ${e.message}`);
|
|
2971
|
+
const okPages = pages.filter((p) => isOkStatus(p.fetch.status));
|
|
2972
|
+
const documents = okPages.filter(isHtmlPage);
|
|
2973
|
+
const { kept: htmlPages, dropped: duplicatePages } = dropCanonicalDuplicates(documents);
|
|
2974
|
+
const unreachablePages = pages.filter((p) => !isOkStatus(p.fetch.status)).map((p) => ({
|
|
2975
|
+
id: fingerprint("page", routeKey(p.fetch.finalUrl)),
|
|
2976
|
+
url: p.fetch.finalUrl,
|
|
2977
|
+
status: p.fetch.status
|
|
2978
|
+
}));
|
|
2979
|
+
const seoIssues = [];
|
|
2980
|
+
for (const page of htmlPages) {
|
|
2981
|
+
const pageUrl = page.fetch.finalUrl;
|
|
2982
|
+
const occurrence = /* @__PURE__ */ new Map();
|
|
2983
|
+
for (const issue of lint(page)) {
|
|
2984
|
+
const n = occurrence.get(issue.ruleId) ?? 0;
|
|
2985
|
+
occurrence.set(issue.ruleId, n + 1);
|
|
2986
|
+
const rule = getRule(issue.ruleId);
|
|
2987
|
+
seoIssues.push({
|
|
2988
|
+
id: fingerprint("seo", issue.ruleId, routeKey(pageUrl), String(n)),
|
|
2989
|
+
pageUrl,
|
|
2990
|
+
ruleId: issue.ruleId,
|
|
2991
|
+
severity: issue.severity,
|
|
2992
|
+
message: issue.message,
|
|
2993
|
+
why: rule?.summary,
|
|
2994
|
+
fix: issue.fix?.snippet
|
|
2995
|
+
});
|
|
2996
|
+
}
|
|
2997
|
+
}
|
|
2998
|
+
const localeAxis = deriveLocaleAxis({
|
|
2999
|
+
explicit: options.locales,
|
|
3000
|
+
sitemapUrls,
|
|
3001
|
+
pages: htmlPages
|
|
3002
|
+
});
|
|
3003
|
+
if (localeAxis.source === "none" && localeAxis.candidates.length > 0) {
|
|
3004
|
+
const suggestion = suggestedLocales(localeAxis);
|
|
3005
|
+
warnings.push(
|
|
3006
|
+
suggestion ? `No sitemap and no --locales: i18n checks are off. Locale-looking prefixes found \u2014 re-run with \`--locales ${suggestion}\` to enable them.` : `No sitemap and no --locales: i18n checks are off. Prefixes seen (${localeAxis.candidates.map((c) => c.tag).join(", ")}) do not look like locales; pass --locales if any of them is one.`
|
|
3007
|
+
);
|
|
3008
|
+
}
|
|
3009
|
+
const matrix = buildI18nMatrix(htmlPages, {
|
|
3010
|
+
declaredUrls: sitemapUrls,
|
|
3011
|
+
locales: localeAxis.locales
|
|
3012
|
+
});
|
|
3013
|
+
const holes = localeAxis.multilingual ? deriveTranslationHoles(matrix, options.ignoreHoles) : [];
|
|
3014
|
+
const ignoredHoles = localeAxis.multilingual && (options.ignoreHoles?.length ?? 0) > 0 ? deriveTranslationHoles(matrix).length - holes.length : 0;
|
|
3015
|
+
const reciprocity = reciprocityIssues(htmlPages).map((issue) => ({
|
|
3016
|
+
id: fingerprint(
|
|
3017
|
+
"i18n",
|
|
3018
|
+
issue.code,
|
|
3019
|
+
routeKey(issue.url),
|
|
3020
|
+
issue.peerUrl ? routeKey(issue.peerUrl) : "",
|
|
3021
|
+
issue.locale ?? ""
|
|
3022
|
+
),
|
|
3023
|
+
...issue
|
|
3024
|
+
}));
|
|
3025
|
+
const siteContext = {
|
|
3026
|
+
origin,
|
|
3027
|
+
pages: htmlPages,
|
|
3028
|
+
matrix,
|
|
3029
|
+
localeAxis,
|
|
3030
|
+
discovery,
|
|
3031
|
+
robots
|
|
3032
|
+
};
|
|
3033
|
+
const siteOccurrence = /* @__PURE__ */ new Map();
|
|
3034
|
+
const siteIssues = lintSite(siteContext).map((draft) => {
|
|
3035
|
+
const key = `${draft.ruleId}\0${draft.pageUrl}`;
|
|
3036
|
+
const n = siteOccurrence.get(key) ?? 0;
|
|
3037
|
+
siteOccurrence.set(key, n + 1);
|
|
3038
|
+
const rule = getSiteRule(draft.ruleId);
|
|
3039
|
+
return {
|
|
3040
|
+
id: fingerprint("site", draft.ruleId, routeKey(draft.pageUrl), String(n)),
|
|
3041
|
+
pageUrl: draft.pageUrl,
|
|
3042
|
+
ruleId: draft.ruleId,
|
|
3043
|
+
severity: draft.severity,
|
|
3044
|
+
message: draft.message,
|
|
3045
|
+
why: rule?.summary,
|
|
3046
|
+
fix: draft.fix?.snippet
|
|
3047
|
+
};
|
|
3048
|
+
});
|
|
3049
|
+
const linkDiscovery = syntheticDiscovery(origin, entry, okPages, crawlResult.truncated);
|
|
3050
|
+
const linkReport = await runLinkAudit(linkDiscovery, {
|
|
3051
|
+
allowInsecureTls: options.allowInsecureTls,
|
|
3052
|
+
checkExternal,
|
|
3053
|
+
timeoutMs: options.timeoutMs,
|
|
3054
|
+
maxPages,
|
|
3055
|
+
signal: options.signal,
|
|
3056
|
+
onProgress: (p) => options.onProgress?.({
|
|
3057
|
+
phase: p.phase === "scan" ? "scan" : "links",
|
|
3058
|
+
done: p.done,
|
|
3059
|
+
total: p.total
|
|
3060
|
+
})
|
|
3061
|
+
});
|
|
3062
|
+
warnings.push(...linkReport.diagnostics.warnings);
|
|
3063
|
+
const brokenLinks = [];
|
|
3064
|
+
for (const { pageUrl, broken } of linkReport.brokenByPage) {
|
|
3065
|
+
for (const check of broken) {
|
|
3066
|
+
brokenLinks.push({
|
|
3067
|
+
id: fingerprint("link", routeKey(pageUrl), targetKey(check.url)),
|
|
3068
|
+
pageUrl,
|
|
3069
|
+
href: check.url,
|
|
3070
|
+
status: check.status,
|
|
3071
|
+
verdict: check.verdict,
|
|
3072
|
+
reason: check.reason
|
|
3073
|
+
});
|
|
3074
|
+
}
|
|
3075
|
+
}
|
|
3076
|
+
const scanTargets = okPages.length;
|
|
3077
|
+
const scanFailed = linkReport.diagnostics.pagesFailed;
|
|
3078
|
+
const scanBlind = scanTargets > 0 && linkReport.pagesScanned === 0;
|
|
3079
|
+
if (scanBlind) {
|
|
3080
|
+
warnings.push(
|
|
3081
|
+
`Link scan failed on all ${scanFailed} page(s) \u2014 "0 broken links" is unverified, not clean.`
|
|
3082
|
+
);
|
|
3083
|
+
} else if (scanFailed > 0 && scanFailed >= scanTargets / 2) {
|
|
3084
|
+
warnings.push(
|
|
3085
|
+
`Link scan failed on ${scanFailed} of ${scanTargets} page(s); link results are partial.`
|
|
3086
|
+
);
|
|
3087
|
+
}
|
|
3088
|
+
const brokenCount = linkReport.summary.broken;
|
|
3089
|
+
const errorCount = seoIssues.filter((i) => i.severity === "error").length + siteIssues.filter((i) => i.severity === "error").length;
|
|
3090
|
+
const missingTranslations = holes.length + reciprocity.length;
|
|
3091
|
+
const anyFinding = missingTranslations > 0 || seoIssues.length > 0 || siteIssues.length > 0 || brokenLinks.length > 0;
|
|
3092
|
+
const crawlBlind = pages.length === 0;
|
|
3093
|
+
if (crawlBlind) {
|
|
3094
|
+
warnings.push(
|
|
3095
|
+
`Crawled 0 pages from ${entry} \u2014 the audit reached nothing, so "no findings" is unverified, not clean.`
|
|
3096
|
+
);
|
|
3097
|
+
}
|
|
3098
|
+
let verdict = "green";
|
|
3099
|
+
if (crawlBlind || brokenCount > 0 || errorCount > 0 || unreachablePages.length > 0 || scanBlind) {
|
|
3100
|
+
verdict = "red";
|
|
3101
|
+
} else if (anyFinding) {
|
|
3102
|
+
verdict = "yellow";
|
|
3103
|
+
}
|
|
3104
|
+
return {
|
|
3105
|
+
url: entry,
|
|
3106
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3107
|
+
summary: {
|
|
3108
|
+
brokenLinks: brokenCount,
|
|
3109
|
+
missingTranslations,
|
|
3110
|
+
seoIssues: seoIssues.length,
|
|
3111
|
+
siteIssues: siteIssues.length,
|
|
3112
|
+
unreachablePages: unreachablePages.length,
|
|
3113
|
+
verdict
|
|
3114
|
+
},
|
|
3115
|
+
localeAxis: {
|
|
3116
|
+
locales: localeAxis.locales,
|
|
3117
|
+
source: localeAxis.source,
|
|
3118
|
+
multilingual: localeAxis.multilingual,
|
|
3119
|
+
...localeAxis.candidates.length > 0 ? { candidates: localeAxis.candidates } : {}
|
|
3120
|
+
},
|
|
3121
|
+
pages: pages.map((p) => ({
|
|
3122
|
+
url: p.fetch.finalUrl,
|
|
3123
|
+
status: p.fetch.status,
|
|
3124
|
+
locale: localeOfUrl(p.fetch.finalUrl)
|
|
3125
|
+
})),
|
|
3126
|
+
unreachablePages,
|
|
3127
|
+
brokenLinks,
|
|
3128
|
+
missingTranslations: { holes, reciprocity },
|
|
3129
|
+
seoIssues,
|
|
3130
|
+
siteIssues,
|
|
3131
|
+
diagnostics: {
|
|
3132
|
+
pagesCrawled: pages.length,
|
|
3133
|
+
pagesScanned: linkReport.pagesScanned,
|
|
3134
|
+
pagesFailed: linkReport.diagnostics.pagesFailed,
|
|
3135
|
+
truncated: crawlResult.truncated || linkReport.truncated,
|
|
3136
|
+
warnings,
|
|
3137
|
+
...ignoredHoles > 0 ? { ignoredHoles } : {},
|
|
3138
|
+
...duplicatePages > 0 ? { duplicatePages } : {},
|
|
3139
|
+
...discovery ? {
|
|
3140
|
+
sitemap: {
|
|
3141
|
+
found: discovery.diagnostics.found,
|
|
3142
|
+
sitemapUrl: discovery.diagnostics.sitemapUrl,
|
|
3143
|
+
urlCount: sitemapUrls.length,
|
|
3144
|
+
uncrawled: countUncrawled(sitemapUrls, pages)
|
|
3145
|
+
}
|
|
3146
|
+
} : {}
|
|
3147
|
+
}
|
|
3148
|
+
};
|
|
3149
|
+
}
|
|
3150
|
+
function countUncrawled(sitemapUrls, pages) {
|
|
3151
|
+
const crawled = new Set(pages.map((p) => p.fetch.finalUrl));
|
|
3152
|
+
let n = 0;
|
|
3153
|
+
for (const url of sitemapUrls) if (!crawled.has(url)) n += 1;
|
|
3154
|
+
return n;
|
|
3155
|
+
}
|
|
3156
|
+
function syntheticDiscovery(origin, baseUrl, pages, truncated) {
|
|
3157
|
+
return {
|
|
3158
|
+
origin,
|
|
3159
|
+
baseUrl,
|
|
3160
|
+
source: "crawl",
|
|
3161
|
+
urls: pages.map((p) => ({ loc: p.fetch.finalUrl })),
|
|
3162
|
+
diagnostics: {
|
|
3163
|
+
found: false,
|
|
3164
|
+
status: 0,
|
|
3165
|
+
declaredInRobots: false,
|
|
3166
|
+
robotsFound: false,
|
|
3167
|
+
atWellKnownPath: false,
|
|
3168
|
+
wellFormed: false,
|
|
3169
|
+
isIndex: false,
|
|
3170
|
+
childSitemapCount: 0,
|
|
3171
|
+
childSitemapErrors: 0,
|
|
3172
|
+
urlCount: pages.length,
|
|
3173
|
+
warnings: []
|
|
3174
|
+
},
|
|
3175
|
+
truncated
|
|
3176
|
+
};
|
|
3177
|
+
}
|
|
3178
|
+
|
|
3179
|
+
// src/report/render-terminal.ts
|
|
3180
|
+
var ANSI = {
|
|
3181
|
+
reset: "\x1B[0m",
|
|
3182
|
+
bold: "\x1B[1m",
|
|
3183
|
+
dim: "\x1B[2m",
|
|
3184
|
+
red: "\x1B[31m",
|
|
3185
|
+
green: "\x1B[32m",
|
|
3186
|
+
yellow: "\x1B[33m",
|
|
3187
|
+
cyan: "\x1B[36m"
|
|
3188
|
+
};
|
|
3189
|
+
function paint(enabled) {
|
|
3190
|
+
return (code, text) => enabled ? `${code}${text}${ANSI.reset}` : text;
|
|
3191
|
+
}
|
|
3192
|
+
var VERDICT_FLAG = {
|
|
3193
|
+
green: "green flag",
|
|
3194
|
+
yellow: "yellow flag",
|
|
3195
|
+
red: "red flag"
|
|
3196
|
+
};
|
|
3197
|
+
function verdictColor(verdict) {
|
|
3198
|
+
return verdict === "green" ? ANSI.green : verdict === "yellow" ? ANSI.yellow : ANSI.red;
|
|
3199
|
+
}
|
|
3200
|
+
function renderTerminal(report, options = {}) {
|
|
3201
|
+
const color = options.color ?? false;
|
|
3202
|
+
const c = paint(color);
|
|
3203
|
+
const lines = [];
|
|
3204
|
+
lines.push("");
|
|
3205
|
+
lines.push(`${c(ANSI.bold, "goflag")} ${c(ANSI.dim, report.url)}`);
|
|
3206
|
+
lines.push(
|
|
3207
|
+
`${c(verdictColor(report.summary.verdict), VERDICT_FLAG[report.summary.verdict].toUpperCase())} ` + c(
|
|
3208
|
+
ANSI.dim,
|
|
3209
|
+
`${report.diagnostics.pagesCrawled} pages crawled, ${report.diagnostics.pagesScanned} scanned`
|
|
3210
|
+
)
|
|
3211
|
+
);
|
|
3212
|
+
lines.push("");
|
|
3213
|
+
lines.push(
|
|
3214
|
+
[
|
|
3215
|
+
count(c, report.summary.brokenLinks, "broken link", report.summary.brokenLinks > 0),
|
|
3216
|
+
count(
|
|
3217
|
+
c,
|
|
3218
|
+
report.summary.missingTranslations,
|
|
3219
|
+
"missing translation",
|
|
3220
|
+
report.summary.missingTranslations > 0
|
|
3221
|
+
),
|
|
3222
|
+
count(c, report.summary.seoIssues, "SEO issue", report.summary.seoIssues > 0),
|
|
3223
|
+
count(c, report.summary.siteIssues, "site issue", report.summary.siteIssues > 0),
|
|
3224
|
+
count(
|
|
3225
|
+
c,
|
|
3226
|
+
report.summary.unreachablePages,
|
|
3227
|
+
"unreachable page",
|
|
3228
|
+
report.summary.unreachablePages > 0
|
|
3229
|
+
)
|
|
3230
|
+
].join(" ")
|
|
3231
|
+
);
|
|
3232
|
+
lines.push("");
|
|
3233
|
+
if (report.unreachablePages.length > 0) {
|
|
3234
|
+
lines.push(c(ANSI.bold, "Unreachable pages"));
|
|
3235
|
+
for (const page of report.unreachablePages) {
|
|
3236
|
+
lines.push(` ${c(ANSI.red, `[${page.status || "network error"}]`)} ${page.url}`);
|
|
3237
|
+
}
|
|
3238
|
+
lines.push("");
|
|
3239
|
+
}
|
|
3240
|
+
if (report.brokenLinks.length > 0) {
|
|
3241
|
+
lines.push(c(ANSI.bold, "Broken links"));
|
|
3242
|
+
const byPage = groupBy(report.brokenLinks, (b) => b.pageUrl);
|
|
3243
|
+
for (const [pageUrl, links] of byPage) {
|
|
3244
|
+
lines.push(` ${c(ANSI.cyan, pageUrl)}`);
|
|
3245
|
+
for (const link of links) {
|
|
3246
|
+
const tag = link.verdict === "broken" ? c(ANSI.red, `[${link.status || link.reason || "broken"}]`) : c(ANSI.yellow, `[${link.verdict}${link.reason ? ` ${link.reason}` : ""}]`);
|
|
3247
|
+
lines.push(` ${tag} ${link.href}`);
|
|
3248
|
+
}
|
|
3249
|
+
}
|
|
3250
|
+
lines.push("");
|
|
3251
|
+
}
|
|
3252
|
+
const { holes, reciprocity } = report.missingTranslations;
|
|
3253
|
+
if (holes.length > 0 || reciprocity.length > 0) {
|
|
3254
|
+
lines.push(c(ANSI.bold, "Missing translations"));
|
|
3255
|
+
for (const hole of holes) {
|
|
3256
|
+
lines.push(
|
|
3257
|
+
` ${c(ANSI.cyan, hole.route)} \u2014 missing ${c(ANSI.yellow, hole.missingLocales.join(", "))}` + c(ANSI.dim, ` (have ${hole.presentLocales.join(", ")})`)
|
|
3258
|
+
);
|
|
3259
|
+
}
|
|
3260
|
+
for (const issue of reciprocity) {
|
|
3261
|
+
lines.push(` ${c(ANSI.yellow, issue.code)} ${c(ANSI.dim, issue.url)}`);
|
|
3262
|
+
lines.push(` ${issue.message}`);
|
|
3263
|
+
}
|
|
3264
|
+
lines.push("");
|
|
3265
|
+
}
|
|
3266
|
+
if (report.seoIssues.length > 0) {
|
|
3267
|
+
lines.push(c(ANSI.bold, "SEO issues"));
|
|
3268
|
+
lines.push(...renderIssuesByPage(report.seoIssues, c));
|
|
3269
|
+
lines.push("");
|
|
3270
|
+
}
|
|
3271
|
+
if (report.siteIssues.length > 0) {
|
|
3272
|
+
lines.push(c(ANSI.bold, "Site-wide issues"));
|
|
3273
|
+
lines.push(
|
|
3274
|
+
` ${c(ANSI.dim, `locales: ${report.localeAxis.locales.join(", ") || "none"} (via ${report.localeAxis.source})`)}`
|
|
3275
|
+
);
|
|
3276
|
+
lines.push(...renderIssuesByPage(report.siteIssues, c));
|
|
3277
|
+
lines.push("");
|
|
3278
|
+
}
|
|
3279
|
+
if (report.summary.verdict === "green") {
|
|
3280
|
+
lines.push(c(ANSI.green, "No problems found."));
|
|
3281
|
+
lines.push("");
|
|
3282
|
+
}
|
|
3283
|
+
for (const warning of report.diagnostics.warnings) {
|
|
3284
|
+
lines.push(c(ANSI.dim, `note: ${warning}`));
|
|
3285
|
+
}
|
|
3286
|
+
if (report.diagnostics.truncated) {
|
|
3287
|
+
lines.push(c(ANSI.dim, "note: results truncated by a page/link cap."));
|
|
3288
|
+
}
|
|
3289
|
+
return lines.join("\n");
|
|
3290
|
+
}
|
|
3291
|
+
function count(c, n, noun, bad) {
|
|
3292
|
+
const label = `${n} ${noun}${n === 1 ? "" : "s"}`;
|
|
3293
|
+
return bad ? c(ANSI.bold, c(ANSI.yellow, label)) : c(ANSI.dim, label);
|
|
3294
|
+
}
|
|
3295
|
+
function renderIssuesByPage(issues, c) {
|
|
3296
|
+
const lines = [];
|
|
3297
|
+
for (const [pageUrl, group] of groupBy([...issues], (i) => i.pageUrl)) {
|
|
3298
|
+
lines.push(` ${c(ANSI.cyan, pageUrl)}`);
|
|
3299
|
+
for (const issue of group) {
|
|
3300
|
+
const sev = issue.severity === "error" ? c(ANSI.red, "error") : issue.severity === "warning" ? c(ANSI.yellow, "warn ") : c(ANSI.dim, "info ");
|
|
3301
|
+
lines.push(` ${sev} ${c(ANSI.dim, issue.ruleId)} ${stripBackticks(issue.message)}`);
|
|
3302
|
+
}
|
|
3303
|
+
}
|
|
3304
|
+
return lines;
|
|
3305
|
+
}
|
|
3306
|
+
function groupBy(items, key) {
|
|
3307
|
+
const map = /* @__PURE__ */ new Map();
|
|
3308
|
+
for (const item of items) {
|
|
3309
|
+
const k = key(item);
|
|
3310
|
+
const arr = map.get(k);
|
|
3311
|
+
if (arr) arr.push(item);
|
|
3312
|
+
else map.set(k, [item]);
|
|
3313
|
+
}
|
|
3314
|
+
return map;
|
|
3315
|
+
}
|
|
3316
|
+
function stripBackticks(s) {
|
|
3317
|
+
return s.replace(/`/g, "");
|
|
3318
|
+
}
|
|
3319
|
+
|
|
3320
|
+
// src/report/render-summary.ts
|
|
3321
|
+
var ANSI2 = {
|
|
3322
|
+
reset: "\x1B[0m",
|
|
3323
|
+
bold: "\x1B[1m",
|
|
3324
|
+
dim: "\x1B[2m",
|
|
3325
|
+
red: "\x1B[31m",
|
|
3326
|
+
green: "\x1B[32m",
|
|
3327
|
+
yellow: "\x1B[33m",
|
|
3328
|
+
cyan: "\x1B[36m"
|
|
3329
|
+
};
|
|
3330
|
+
function paint2(enabled) {
|
|
3331
|
+
return (code, text) => enabled ? `${code}${text}${ANSI2.reset}` : text;
|
|
3332
|
+
}
|
|
3333
|
+
var VERDICT_FLAG2 = {
|
|
3334
|
+
green: "GREEN FLAG",
|
|
3335
|
+
yellow: "YELLOW FLAG",
|
|
3336
|
+
red: "RED FLAG"
|
|
3337
|
+
};
|
|
3338
|
+
function verdictColor2(verdict) {
|
|
3339
|
+
return verdict === "green" ? ANSI2.green : verdict === "yellow" ? ANSI2.yellow : ANSI2.red;
|
|
3340
|
+
}
|
|
3341
|
+
function renderSummaryTerminal(summary, options = {}) {
|
|
3342
|
+
const c = paint2(options.color ?? false);
|
|
3343
|
+
const lines = [];
|
|
3344
|
+
lines.push("");
|
|
3345
|
+
lines.push(`${c(ANSI2.bold, "goflag")} ${c(ANSI2.dim, summary.url)} ${c(ANSI2.dim, "(summary)")}`);
|
|
3346
|
+
lines.push(
|
|
3347
|
+
`${c(verdictColor2(summary.verdict), VERDICT_FLAG2[summary.verdict])} ` + c(
|
|
3348
|
+
ANSI2.dim,
|
|
3349
|
+
`${summary.totals.pagesCrawled} pages crawled, ${summary.totals.pagesScanned} scanned`
|
|
3350
|
+
)
|
|
3351
|
+
);
|
|
3352
|
+
lines.push("");
|
|
3353
|
+
lines.push(
|
|
3354
|
+
[
|
|
3355
|
+
metric(c, summary.totals.brokenLinks, "broken link"),
|
|
3356
|
+
metric(c, summary.totals.missingTranslations, "missing translation"),
|
|
3357
|
+
metric(c, summary.totals.seoIssues, "SEO issue"),
|
|
3358
|
+
metric(c, summary.totals.siteIssues, "site issue"),
|
|
3359
|
+
metric(c, summary.totals.unreachablePages, "unreachable page")
|
|
3360
|
+
].join(" ")
|
|
3361
|
+
);
|
|
3362
|
+
lines.push("");
|
|
3363
|
+
if (summary.unreachablePages.length > 0) {
|
|
3364
|
+
lines.push(c(ANSI2.bold, "Unreachable pages"));
|
|
3365
|
+
for (const page of summary.unreachablePages) {
|
|
3366
|
+
lines.push(` ${c(ANSI2.red, `[${page.status || "network error"}]`)} ${page.url}`);
|
|
3367
|
+
}
|
|
3368
|
+
lines.push("");
|
|
3369
|
+
}
|
|
3370
|
+
if (summary.brokenLinks.length > 0) {
|
|
3371
|
+
lines.push(c(ANSI2.bold, "Broken links"));
|
|
3372
|
+
for (const link of summary.brokenLinks) {
|
|
3373
|
+
const tag = link.verdict === "broken" ? c(ANSI2.red, `[${link.status || link.reason || "broken"}]`) : c(ANSI2.yellow, `[${link.verdict}${link.reason ? ` ${link.reason}` : ""}]`);
|
|
3374
|
+
lines.push(` ${tag} ${link.href} ${c(ANSI2.dim, `\xD7${link.count}`)}`);
|
|
3375
|
+
lines.push(` ${c(ANSI2.dim, `on ${samplePages(link.pages, link.morePages)}`)}`);
|
|
3376
|
+
}
|
|
3377
|
+
lines.push("");
|
|
3378
|
+
}
|
|
3379
|
+
const { holes, reciprocity } = summary.translations;
|
|
3380
|
+
if (holes.length > 0 || reciprocity.length > 0) {
|
|
3381
|
+
lines.push(c(ANSI2.bold, "Missing translations"));
|
|
3382
|
+
for (const hole of holes) {
|
|
3383
|
+
lines.push(
|
|
3384
|
+
` ${c(ANSI2.cyan, hole.route)} \u2014 missing ${c(ANSI2.yellow, hole.missingLocales.join(", "))}` + c(ANSI2.dim, ` (have ${hole.presentLocales.join(", ")})`)
|
|
3385
|
+
);
|
|
3386
|
+
}
|
|
3387
|
+
for (const issue of reciprocity) {
|
|
3388
|
+
lines.push(` ${c(ANSI2.yellow, issue.code)} ${c(ANSI2.dim, `\xD7${issue.count}`)}`);
|
|
3389
|
+
lines.push(` ${stripBackticks2(issue.sample)}`);
|
|
3390
|
+
lines.push(` ${c(ANSI2.dim, `on ${samplePages(issue.pages, issue.morePages)}`)}`);
|
|
3391
|
+
}
|
|
3392
|
+
lines.push("");
|
|
3393
|
+
}
|
|
3394
|
+
if (summary.seoIssues.length > 0) {
|
|
3395
|
+
lines.push(c(ANSI2.bold, "SEO issues"));
|
|
3396
|
+
lines.push(...rollupRows(summary.seoIssues, c));
|
|
3397
|
+
lines.push("");
|
|
3398
|
+
}
|
|
3399
|
+
if (summary.siteIssues.length > 0) {
|
|
3400
|
+
lines.push(c(ANSI2.bold, "Site-wide issues"));
|
|
3401
|
+
lines.push(...rollupRows(summary.siteIssues, c));
|
|
3402
|
+
lines.push("");
|
|
3403
|
+
}
|
|
3404
|
+
if (summary.verdict === "green") {
|
|
3405
|
+
lines.push(c(ANSI2.green, "No problems found."));
|
|
3406
|
+
lines.push("");
|
|
3407
|
+
}
|
|
3408
|
+
for (const warning of summary.warnings) {
|
|
3409
|
+
lines.push(c(ANSI2.dim, `note: ${warning}`));
|
|
3410
|
+
}
|
|
3411
|
+
if (summary.truncated) {
|
|
3412
|
+
lines.push(c(ANSI2.dim, "note: results truncated by a page/link cap."));
|
|
3413
|
+
}
|
|
3414
|
+
return lines.join("\n");
|
|
3415
|
+
}
|
|
3416
|
+
function metric(c, n, noun) {
|
|
3417
|
+
const label = `${n} ${noun}${n === 1 ? "" : "s"}`;
|
|
3418
|
+
return n > 0 ? c(ANSI2.bold, c(ANSI2.yellow, label)) : c(ANSI2.dim, label);
|
|
3419
|
+
}
|
|
3420
|
+
function samplePages(pages, more) {
|
|
3421
|
+
const shown = pages.join(", ");
|
|
3422
|
+
return more > 0 ? `${shown} (+${more} more)` : shown;
|
|
3423
|
+
}
|
|
3424
|
+
function stripBackticks2(s) {
|
|
3425
|
+
return s.replace(/`/g, "");
|
|
3426
|
+
}
|
|
3427
|
+
function rollupRows(issues, c) {
|
|
3428
|
+
const lines = [];
|
|
3429
|
+
for (const issue of issues) {
|
|
3430
|
+
const sev = issue.severity === "error" ? c(ANSI2.red, "error") : issue.severity === "warning" ? c(ANSI2.yellow, "warn ") : c(ANSI2.dim, "info ");
|
|
3431
|
+
lines.push(` ${sev} ${c(ANSI2.cyan, issue.ruleId)} ${c(ANSI2.dim, `\xD7${issue.count}`)}`);
|
|
3432
|
+
if (issue.why) lines.push(` ${c(ANSI2.dim, stripBackticks2(issue.why))}`);
|
|
3433
|
+
if (issue.fix) {
|
|
3434
|
+
const [head, ...rest] = issue.fix.split("\n");
|
|
3435
|
+
lines.push(` ${c(ANSI2.dim, "fix:")} ${head}`);
|
|
3436
|
+
for (const line of rest) lines.push(` ${c(ANSI2.dim, line)}`);
|
|
3437
|
+
}
|
|
3438
|
+
lines.push(` ${c(ANSI2.dim, `on ${samplePages(issue.pages, issue.morePages)}`)}`);
|
|
3439
|
+
}
|
|
3440
|
+
return lines;
|
|
3441
|
+
}
|
|
3442
|
+
|
|
3443
|
+
// src/report/summarize.ts
|
|
3444
|
+
var SAMPLE_LIMIT = 5;
|
|
3445
|
+
function summarize(report) {
|
|
3446
|
+
return {
|
|
3447
|
+
url: report.url,
|
|
3448
|
+
finishedAt: report.finishedAt,
|
|
3449
|
+
verdict: report.summary.verdict,
|
|
3450
|
+
totals: {
|
|
3451
|
+
brokenLinks: report.summary.brokenLinks,
|
|
3452
|
+
missingTranslations: report.summary.missingTranslations,
|
|
3453
|
+
seoIssues: report.summary.seoIssues,
|
|
3454
|
+
siteIssues: report.summary.siteIssues,
|
|
3455
|
+
unreachablePages: report.summary.unreachablePages,
|
|
3456
|
+
pagesCrawled: report.diagnostics.pagesCrawled,
|
|
3457
|
+
pagesScanned: report.diagnostics.pagesScanned,
|
|
3458
|
+
pagesFailed: report.diagnostics.pagesFailed
|
|
3459
|
+
},
|
|
3460
|
+
unreachablePages: report.unreachablePages,
|
|
3461
|
+
brokenLinks: rollupLinks(report),
|
|
3462
|
+
translations: {
|
|
3463
|
+
holes: report.missingTranslations.holes,
|
|
3464
|
+
reciprocity: rollupReciprocity(report)
|
|
3465
|
+
},
|
|
3466
|
+
seoIssues: rollupByRule(report.seoIssues),
|
|
3467
|
+
siteIssues: rollupByRule(report.siteIssues),
|
|
3468
|
+
truncated: report.diagnostics.truncated,
|
|
3469
|
+
warnings: report.diagnostics.warnings
|
|
3470
|
+
};
|
|
3471
|
+
}
|
|
3472
|
+
function rollupLinks(report) {
|
|
3473
|
+
const groups = /* @__PURE__ */ new Map();
|
|
3474
|
+
for (const link of report.brokenLinks) {
|
|
3475
|
+
const key = `${link.href}\0${link.verdict}`;
|
|
3476
|
+
const existing = groups.get(key);
|
|
3477
|
+
if (existing) {
|
|
3478
|
+
existing.count += 1;
|
|
3479
|
+
if (existing.pages.length < SAMPLE_LIMIT) existing.pages.push(link.pageUrl);
|
|
3480
|
+
else existing.morePages += 1;
|
|
3481
|
+
} else {
|
|
3482
|
+
groups.set(key, {
|
|
3483
|
+
href: link.href,
|
|
3484
|
+
verdict: link.verdict,
|
|
3485
|
+
status: link.status,
|
|
3486
|
+
reason: link.reason,
|
|
3487
|
+
count: 1,
|
|
3488
|
+
pages: [link.pageUrl],
|
|
3489
|
+
morePages: 0
|
|
3490
|
+
});
|
|
3491
|
+
}
|
|
3492
|
+
}
|
|
3493
|
+
return [...groups.values()].sort((a, b) => b.count - a.count || a.href.localeCompare(b.href));
|
|
3494
|
+
}
|
|
3495
|
+
function rollupByRule(issues) {
|
|
3496
|
+
const groups = /* @__PURE__ */ new Map();
|
|
3497
|
+
for (const issue of issues) {
|
|
3498
|
+
const existing = groups.get(issue.ruleId);
|
|
3499
|
+
if (existing) {
|
|
3500
|
+
existing.count += 1;
|
|
3501
|
+
if (existing.pages.length < SAMPLE_LIMIT) existing.pages.push(issue.pageUrl);
|
|
3502
|
+
else existing.morePages += 1;
|
|
3503
|
+
} else {
|
|
3504
|
+
groups.set(issue.ruleId, {
|
|
3505
|
+
ruleId: issue.ruleId,
|
|
3506
|
+
severity: issue.severity,
|
|
3507
|
+
why: issue.why,
|
|
3508
|
+
fix: issue.fix,
|
|
3509
|
+
sample: issue.message,
|
|
3510
|
+
count: 1,
|
|
3511
|
+
pages: [issue.pageUrl],
|
|
3512
|
+
morePages: 0
|
|
3513
|
+
});
|
|
3514
|
+
}
|
|
3515
|
+
}
|
|
3516
|
+
const sevRank = { error: 0, warning: 1, info: 2 };
|
|
3517
|
+
return [...groups.values()].sort(
|
|
3518
|
+
(a, b) => sevRank[a.severity] - sevRank[b.severity] || a.ruleId.localeCompare(b.ruleId)
|
|
3519
|
+
);
|
|
3520
|
+
}
|
|
3521
|
+
function rollupReciprocity(report) {
|
|
3522
|
+
const groups = /* @__PURE__ */ new Map();
|
|
3523
|
+
for (const issue of report.missingTranslations.reciprocity) {
|
|
3524
|
+
const existing = groups.get(issue.code);
|
|
3525
|
+
if (existing) {
|
|
3526
|
+
existing.count += 1;
|
|
3527
|
+
if (existing.pages.length < SAMPLE_LIMIT) existing.pages.push(issue.url);
|
|
3528
|
+
else existing.morePages += 1;
|
|
3529
|
+
} else {
|
|
3530
|
+
groups.set(issue.code, {
|
|
3531
|
+
code: issue.code,
|
|
3532
|
+
count: 1,
|
|
3533
|
+
sample: issue.message,
|
|
3534
|
+
pages: [issue.url],
|
|
3535
|
+
morePages: 0
|
|
3536
|
+
});
|
|
3537
|
+
}
|
|
3538
|
+
}
|
|
3539
|
+
return [...groups.values()].sort((a, b) => b.count - a.count || a.code.localeCompare(b.code));
|
|
3540
|
+
}
|
|
3541
|
+
|
|
3542
|
+
// src/report/logger.ts
|
|
3543
|
+
var FRAMES = ["\u280B", "\u2819", "\u2839", "\u2838", "\u283C", "\u2834", "\u2826", "\u2827", "\u2807", "\u280F"];
|
|
3544
|
+
var BAR_WIDTH = 18;
|
|
3545
|
+
var ANSI3 = {
|
|
3546
|
+
reset: "\x1B[0m",
|
|
3547
|
+
dim: "\x1B[2m",
|
|
3548
|
+
bold: "\x1B[1m",
|
|
3549
|
+
red: "\x1B[31m",
|
|
3550
|
+
green: "\x1B[32m",
|
|
3551
|
+
yellow: "\x1B[33m",
|
|
3552
|
+
cyan: "\x1B[36m"
|
|
3553
|
+
};
|
|
3554
|
+
var PHASE_LABEL = {
|
|
3555
|
+
crawl: "crawl",
|
|
3556
|
+
scan: "scan",
|
|
3557
|
+
links: "links"
|
|
3558
|
+
};
|
|
3559
|
+
var PHASE_HEADER = {
|
|
3560
|
+
crawl: "Crawling pages",
|
|
3561
|
+
scan: "Scanning pages for links",
|
|
3562
|
+
links: "Checking links"
|
|
3563
|
+
};
|
|
3564
|
+
var PHASE_DONE_NOUN = {
|
|
3565
|
+
crawl: "pages",
|
|
3566
|
+
scan: "pages scanned",
|
|
3567
|
+
links: "links"
|
|
3568
|
+
};
|
|
3569
|
+
var Logger = class {
|
|
3570
|
+
stream;
|
|
3571
|
+
color;
|
|
3572
|
+
mode;
|
|
3573
|
+
isTTY;
|
|
3574
|
+
spinnerIntervalMs;
|
|
3575
|
+
throttleMs;
|
|
3576
|
+
now;
|
|
3577
|
+
timer;
|
|
3578
|
+
frame = 0;
|
|
3579
|
+
last;
|
|
3580
|
+
lineOpen = false;
|
|
3581
|
+
lastPlainAt = 0;
|
|
3582
|
+
headerShown = {};
|
|
3583
|
+
constructor(options = {}) {
|
|
3584
|
+
this.stream = options.stream ?? process.stderr;
|
|
3585
|
+
this.isTTY = this.stream.isTTY === true;
|
|
3586
|
+
this.color = options.color ?? this.isTTY;
|
|
3587
|
+
this.mode = options.mode ?? "compact";
|
|
3588
|
+
this.spinnerIntervalMs = options.spinnerIntervalMs ?? 80;
|
|
3589
|
+
this.throttleMs = options.throttleMs ?? 250;
|
|
3590
|
+
this.now = options.now ?? Date.now;
|
|
3591
|
+
}
|
|
3592
|
+
paint(code, text) {
|
|
3593
|
+
return this.color ? `${code}${text}${ANSI3.reset}` : text;
|
|
3594
|
+
}
|
|
3595
|
+
/** Print a standalone line (banner, warning). Survives around the spinner. */
|
|
3596
|
+
note(message) {
|
|
3597
|
+
if (this.mode === "quiet") return;
|
|
3598
|
+
this.clearLine();
|
|
3599
|
+
this.stream.write(`${this.paint(ANSI3.dim, message)}
|
|
3600
|
+
`);
|
|
3601
|
+
if (this.mode === "compact" && this.last) this.render();
|
|
3602
|
+
}
|
|
3603
|
+
/** Feed one progress tick. */
|
|
3604
|
+
onProgress = (event) => {
|
|
3605
|
+
if (this.mode === "quiet") return;
|
|
3606
|
+
this.last = event;
|
|
3607
|
+
if (this.mode === "verbose") {
|
|
3608
|
+
this.verboseLine(event);
|
|
3609
|
+
return;
|
|
3610
|
+
}
|
|
3611
|
+
if (this.isTTY) {
|
|
3612
|
+
this.ensureTimer();
|
|
3613
|
+
this.render();
|
|
3614
|
+
} else {
|
|
3615
|
+
this.plainThrottled(event);
|
|
3616
|
+
}
|
|
3617
|
+
};
|
|
3618
|
+
/** Stop the spinner and clear the status line so the report starts clean. */
|
|
3619
|
+
stop() {
|
|
3620
|
+
if (this.timer) {
|
|
3621
|
+
clearInterval(this.timer);
|
|
3622
|
+
this.timer = void 0;
|
|
3623
|
+
}
|
|
3624
|
+
this.clearLine();
|
|
3625
|
+
}
|
|
3626
|
+
// --- rendering ---------------------------------------------------------
|
|
3627
|
+
ensureTimer() {
|
|
3628
|
+
if (this.timer || this.spinnerIntervalMs <= 0) return;
|
|
3629
|
+
this.timer = setInterval(() => {
|
|
3630
|
+
this.frame = (this.frame + 1) % FRAMES.length;
|
|
3631
|
+
this.render();
|
|
3632
|
+
}, this.spinnerIntervalMs);
|
|
3633
|
+
this.timer.unref?.();
|
|
3634
|
+
}
|
|
3635
|
+
render() {
|
|
3636
|
+
if (!this.last) return;
|
|
3637
|
+
const line = this.composeLine(this.last);
|
|
3638
|
+
if (this.isTTY) {
|
|
3639
|
+
this.stream.write(`\r\x1B[2K${line}`);
|
|
3640
|
+
this.lineOpen = true;
|
|
3641
|
+
} else {
|
|
3642
|
+
this.stream.write(`${line}
|
|
3643
|
+
`);
|
|
3644
|
+
}
|
|
3645
|
+
}
|
|
3646
|
+
composeLine(e) {
|
|
3647
|
+
const spinner = this.paint(ANSI3.cyan, FRAMES[this.frame]);
|
|
3648
|
+
const phase = this.paint(ANSI3.bold, PHASE_LABEL[e.phase]);
|
|
3649
|
+
const counts = `${e.done}/${e.total}`;
|
|
3650
|
+
const bar = this.bar(e.done, e.total);
|
|
3651
|
+
const subject = e.url ? this.paint(ANSI3.dim, this.truncate(e.url)) : "";
|
|
3652
|
+
return [spinner, phase, bar, counts, subject].filter(Boolean).join(" ");
|
|
3653
|
+
}
|
|
3654
|
+
bar(done, total) {
|
|
3655
|
+
if (total <= 0) return "";
|
|
3656
|
+
const ratio = Math.max(0, Math.min(1, done / total));
|
|
3657
|
+
const filled = Math.round(ratio * BAR_WIDTH);
|
|
3658
|
+
const bar = "\u2588".repeat(filled) + "\u2591".repeat(BAR_WIDTH - filled);
|
|
3659
|
+
return this.paint(ANSI3.dim, `[${bar}]`);
|
|
3660
|
+
}
|
|
3661
|
+
verboseLine(e) {
|
|
3662
|
+
if (!this.headerShown[e.phase]) {
|
|
3663
|
+
this.headerShown[e.phase] = true;
|
|
3664
|
+
this.stream.write(`${this.paint(ANSI3.bold, PHASE_HEADER[e.phase])}
|
|
3665
|
+
`);
|
|
3666
|
+
}
|
|
3667
|
+
if (e.phase === "crawl" && e.url) {
|
|
3668
|
+
const counts = this.paint(ANSI3.dim, `(${e.done}/${e.total})`);
|
|
3669
|
+
this.stream.write(` ${this.statusTag(e.status)} ${e.url} ${counts}
|
|
3670
|
+
`);
|
|
3671
|
+
} else if (e.phase === "scan" || e.phase === "links") {
|
|
3672
|
+
if (e.done === e.total && e.total > 0) {
|
|
3673
|
+
this.stream.write(` ${this.paint(ANSI3.dim, `${e.total} ${PHASE_DONE_NOUN[e.phase]}`)}
|
|
3674
|
+
`);
|
|
3675
|
+
}
|
|
3676
|
+
}
|
|
3677
|
+
}
|
|
3678
|
+
statusTag(status) {
|
|
3679
|
+
if (status === void 0) return this.paint(ANSI3.dim, "\xB7\xB7\xB7");
|
|
3680
|
+
if (status >= 200 && status < 300) return this.paint(ANSI3.green, String(status));
|
|
3681
|
+
if (status >= 300 && status < 400) return this.paint(ANSI3.yellow, String(status));
|
|
3682
|
+
return this.paint(ANSI3.red, String(status));
|
|
3683
|
+
}
|
|
3684
|
+
plainThrottled(e) {
|
|
3685
|
+
const t = this.now();
|
|
3686
|
+
const done = e.done === e.total;
|
|
3687
|
+
if (!done && t - this.lastPlainAt < this.throttleMs) return;
|
|
3688
|
+
this.lastPlainAt = t;
|
|
3689
|
+
this.stream.write(`goflag: ${PHASE_LABEL[e.phase]} ${e.done}/${e.total}
|
|
3690
|
+
`);
|
|
3691
|
+
}
|
|
3692
|
+
clearLine() {
|
|
3693
|
+
if (this.isTTY && this.lineOpen) {
|
|
3694
|
+
this.stream.write("\r\x1B[2K");
|
|
3695
|
+
this.lineOpen = false;
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3698
|
+
truncate(url) {
|
|
3699
|
+
const width = this.stream.columns ?? 80;
|
|
3700
|
+
const budget = Math.max(16, width - 44);
|
|
3701
|
+
if (url.length <= budget) return url;
|
|
3702
|
+
return `\u2026${url.slice(url.length - budget + 1)}`;
|
|
3703
|
+
}
|
|
3704
|
+
};
|
|
3705
|
+
|
|
3706
|
+
export {
|
|
3707
|
+
routeKey,
|
|
3708
|
+
targetKey,
|
|
3709
|
+
fingerprint,
|
|
3710
|
+
deriveTranslationHoles,
|
|
3711
|
+
exitCode,
|
|
3712
|
+
runAudit,
|
|
3713
|
+
renderTerminal,
|
|
3714
|
+
renderSummaryTerminal,
|
|
3715
|
+
SAMPLE_LIMIT,
|
|
3716
|
+
summarize,
|
|
3717
|
+
Logger
|
|
3718
|
+
};
|
|
3719
|
+
//# sourceMappingURL=chunk-75OFMTOC.js.map
|