@eightmedia/flightcheck 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/analyze/structuredReport.d.ts +2 -0
- package/dist/analyze/structuredReport.d.ts.map +1 -1
- package/dist/analyze/structuredReport.js +33 -15
- package/dist/checks/agent-ready.d.ts.map +1 -1
- package/dist/checks/agent-ready.js +8 -8
- package/dist/checks/crawl.d.ts.map +1 -1
- package/dist/checks/crawl.js +11 -0
- package/dist/checks/dns.d.ts.map +1 -1
- package/dist/checks/dns.js +2 -2
- package/dist/checks/issues.d.ts +3 -1
- package/dist/checks/issues.d.ts.map +1 -1
- package/dist/checks/issues.js +2 -2
- package/dist/checks/redirect.d.ts.map +1 -1
- package/dist/checks/redirect.js +12 -12
- package/dist/checks/robots-txt.d.ts.map +1 -1
- package/dist/checks/robots-txt.js +6 -6
- package/dist/checks/security-headers.d.ts.map +1 -1
- package/dist/checks/security-headers.js +6 -6
- package/dist/checks/sitemap.d.ts.map +1 -1
- package/dist/checks/sitemap.js +6 -6
- package/dist/cli.js +1 -1
- package/dist/crawler/crawleeStorage.d.ts +18 -0
- package/dist/crawler/crawleeStorage.d.ts.map +1 -0
- package/dist/crawler/crawleeStorage.js +24 -0
- package/dist/crawler/defaults.d.ts +2 -1
- package/dist/crawler/defaults.d.ts.map +1 -1
- package/dist/crawler/defaults.js +2 -1
- package/dist/crawler/runCrawler.d.ts.map +1 -1
- package/dist/crawler/runCrawler.js +112 -84
- package/dist/crawler/smartPageSelection.d.ts +77 -0
- package/dist/crawler/smartPageSelection.d.ts.map +1 -0
- package/dist/crawler/smartPageSelection.js +308 -0
- package/docs/agent-ready.md +2 -2
- package/docs/analyze.md +5 -3
- package/docs/crawler.md +18 -2
- package/docs/index.md +1 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [1.1.0] — 2026-10-28
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- Findings include an optional `fix` suggestion in `*-report.json` and Markdown (`- Fix: …`)
|
|
8
|
+
- Crawlee storage uses `.flightcheckStorage/<run-hash>/` and is cleaned up after each crawl
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
|
|
12
|
+
- Default `--max-pages` is now **100** (was 1000)
|
|
13
|
+
- Within the page budget, crawl seeds are selected from **URL-tree shape** (not slug
|
|
14
|
+
names): home/structural pages and list indexes first, then a fair sample of leaf
|
|
15
|
+
URLs under parents that have many leaf children
|
|
16
|
+
- Discovered structural links jump the queue; leaf URLs under list parents are capped
|
|
17
|
+
|
|
3
18
|
## [1.0.1] — 2026-10-28
|
|
4
19
|
|
|
5
20
|
### Added
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"structuredReport.d.ts","sourceRoot":"","sources":["../../src/analyze/structuredReport.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAG1E,OAAO,EAEL,KAAK,oBAAoB,EAEzB,KAAK,YAAY,EACjB,KAAK,eAAe,EACrB,MAAM,aAAa,CAAC;AAErB,eAAO,MAAM,0BAA0B,EAAG,CAAU,CAAC;AAErD,MAAM,MAAM,eAAe,GAAG,kBAAkB,CAAC;AACjD,MAAM,MAAM,eAAe,GACvB,OAAO,GACP,KAAK,GACL,MAAM,GACN,gBAAgB,GAChB,UAAU,GACV,aAAa,CAAC;AAClB,MAAM,MAAM,YAAY,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAC;AAExE,MAAM,MAAM,gBAAgB,GAAG;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,eAAe,CAAC;IAC1B,QAAQ,EAAE,eAAe,CAAC;IAC1B,KAAK,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC/C,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,GAAG,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC9C,QAAQ,CAAC,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,WAAW,EAAE,gBAAgB,EAAE,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,gEAAgE;IAChE,OAAO,EAAE,OAAO,0BAA0B,CAAC;IAC3C,oEAAoE;IACpE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mGAAmG;IACnG,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,QAAQ,EAAE,aAAa,EAAE,CAAC;CAC3B,CAAC;
|
|
1
|
+
{"version":3,"file":"structuredReport.d.ts","sourceRoot":"","sources":["../../src/analyze/structuredReport.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAG1E,OAAO,EAEL,KAAK,oBAAoB,EAEzB,KAAK,YAAY,EACjB,KAAK,eAAe,EACrB,MAAM,aAAa,CAAC;AAErB,eAAO,MAAM,0BAA0B,EAAG,CAAU,CAAC;AAErD,MAAM,MAAM,eAAe,GAAG,kBAAkB,CAAC;AACjD,MAAM,MAAM,eAAe,GACvB,OAAO,GACP,KAAK,GACL,MAAM,GACN,gBAAgB,GAChB,UAAU,GACV,aAAa,CAAC;AAClB,MAAM,MAAM,YAAY,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAC;AAExE,MAAM,MAAM,gBAAgB,GAAG;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,eAAe,CAAC;IAC1B,QAAQ,EAAE,eAAe,CAAC;IAC1B,KAAK,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,sDAAsD;IACtD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC/C,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,GAAG,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC9C,QAAQ,CAAC,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,WAAW,EAAE,gBAAgB,EAAE,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,gEAAgE;IAChE,OAAO,EAAE,OAAO,0BAA0B,CAAC;IAC3C,oEAAoE;IACpE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mGAAmG;IACnG,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,QAAQ,EAAE,aAAa,EAAE,CAAC;CAC3B,CAAC;AAieF,MAAM,MAAM,4BAA4B,GAAG,oBAAoB,GAAG;IAChE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,wBAAsB,qBAAqB,CACzC,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,YAAY,EAAE,EACvB,KAAK,GAAE,eAAoB,EAC3B,OAAO,GAAE,4BAAiC,GACzC,OAAO,CAAC,iBAAiB,CAAC,CAa5B;AAiCD,4FAA4F;AAC5F,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,CAuFtE"}
|
|
@@ -170,6 +170,7 @@ function findingFromCheck(issue, category, scope, extras = {}) {
|
|
|
170
170
|
scope,
|
|
171
171
|
title: humanizeId(issue.id),
|
|
172
172
|
message: issue.message,
|
|
173
|
+
...(issue.fix ? { fix: issue.fix } : {}),
|
|
173
174
|
...extras,
|
|
174
175
|
};
|
|
175
176
|
}
|
|
@@ -182,29 +183,35 @@ function pushUnique(findings, finding) {
|
|
|
182
183
|
findings.push(finding);
|
|
183
184
|
}
|
|
184
185
|
function collectSeoFindings(fields, scope, count, extras = {}) {
|
|
185
|
-
return fields.map((field) =>
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
186
|
+
return fields.map((field) => {
|
|
187
|
+
const message = fieldMessage(field, scope, extras.template);
|
|
188
|
+
return {
|
|
189
|
+
id: field,
|
|
190
|
+
severity: "error",
|
|
191
|
+
category: "seo",
|
|
192
|
+
scope,
|
|
193
|
+
title: fieldTitle(field),
|
|
194
|
+
message,
|
|
195
|
+
fix: message,
|
|
196
|
+
count,
|
|
197
|
+
...extras,
|
|
198
|
+
};
|
|
199
|
+
});
|
|
195
200
|
}
|
|
196
201
|
function collectBestPracticeFindings(keys, scope, count, extras = {}) {
|
|
197
202
|
return keys
|
|
198
203
|
.filter((key) => BEST_PRACTICE_IDS.has(key))
|
|
199
204
|
.map((key) => {
|
|
200
205
|
const copy = BEST_PRACTICE_COPY[key];
|
|
206
|
+
const message = copy?.message ?? `Fix ${key}.`;
|
|
201
207
|
return {
|
|
202
208
|
id: key,
|
|
203
209
|
severity: copy?.severity ?? "review",
|
|
204
210
|
category: "best-practices",
|
|
205
211
|
scope,
|
|
206
212
|
title: copy?.title ?? humanizeId(key),
|
|
207
|
-
message
|
|
213
|
+
message,
|
|
214
|
+
fix: message,
|
|
208
215
|
count,
|
|
209
216
|
...extras,
|
|
210
217
|
};
|
|
@@ -215,13 +222,15 @@ function collectA11yFindings(keys, scope, count, extras = {}) {
|
|
|
215
222
|
.filter((key) => A11Y_IDS.has(key))
|
|
216
223
|
.map((key) => {
|
|
217
224
|
const copy = A11Y_COPY[key];
|
|
225
|
+
const message = copy?.message ?? `Fix accessibility issue: ${key}.`;
|
|
218
226
|
return {
|
|
219
227
|
id: key,
|
|
220
228
|
severity: "warning",
|
|
221
229
|
category: "a11y",
|
|
222
230
|
scope,
|
|
223
231
|
title: copy?.title ?? humanizeId(key),
|
|
224
|
-
message
|
|
232
|
+
message,
|
|
233
|
+
fix: message,
|
|
225
234
|
count,
|
|
226
235
|
...extras,
|
|
227
236
|
};
|
|
@@ -284,6 +293,7 @@ function buildFindings(analysis, tier1) {
|
|
|
284
293
|
scope: "page",
|
|
285
294
|
title: `Exposed ${secret.provider} ${secret.type}`,
|
|
286
295
|
message: `Rotate or remove this secret. Masked value: ${secret.masked} (source: ${secret.source}).`,
|
|
296
|
+
fix: "Remove the secret from client-side HTML/JS, rotate the credential, and load secrets only from a server-side env.",
|
|
287
297
|
urls: [page.url],
|
|
288
298
|
count: 1,
|
|
289
299
|
});
|
|
@@ -313,13 +323,15 @@ function buildFindings(analysis, tier1) {
|
|
|
313
323
|
if (!analysis.redirectOnly) {
|
|
314
324
|
const seoById = aggregatePageFindings(analysis.pageFixes, (fix) => fix.remainingFields.map((id) => ({ id })));
|
|
315
325
|
for (const [id, { urls }] of seoById) {
|
|
326
|
+
const message = fieldMessage(id, "page");
|
|
316
327
|
findings.push({
|
|
317
328
|
id,
|
|
318
329
|
severity: "error",
|
|
319
330
|
category: "seo",
|
|
320
331
|
scope: "page",
|
|
321
332
|
title: fieldTitle(id),
|
|
322
|
-
message
|
|
333
|
+
message,
|
|
334
|
+
fix: message,
|
|
323
335
|
urls,
|
|
324
336
|
count: urls.length,
|
|
325
337
|
});
|
|
@@ -327,13 +339,15 @@ function buildFindings(analysis, tier1) {
|
|
|
327
339
|
const bestById = aggregatePageFindings(analysis.pageFixes, (fix) => fix.remainingWarningKeys.filter((key) => BEST_PRACTICE_IDS.has(key)).map((id) => ({ id })));
|
|
328
340
|
for (const [id, { urls }] of bestById) {
|
|
329
341
|
const copy = BEST_PRACTICE_COPY[id];
|
|
342
|
+
const message = copy?.message ?? `Fix ${id}.`;
|
|
330
343
|
findings.push({
|
|
331
344
|
id,
|
|
332
345
|
severity: copy?.severity ?? "review",
|
|
333
346
|
category: "best-practices",
|
|
334
347
|
scope: "page",
|
|
335
348
|
title: copy?.title ?? humanizeId(id),
|
|
336
|
-
message
|
|
349
|
+
message,
|
|
350
|
+
fix: message,
|
|
337
351
|
urls,
|
|
338
352
|
count: urls.length,
|
|
339
353
|
});
|
|
@@ -341,13 +355,15 @@ function buildFindings(analysis, tier1) {
|
|
|
341
355
|
const a11yById = aggregatePageFindings(analysis.pageFixes, (fix) => fix.remainingA11yIssues.map((issue) => ({ id: issue.id, message: issue.message })));
|
|
342
356
|
for (const [id, { urls, message }] of a11yById) {
|
|
343
357
|
const copy = A11Y_COPY[id];
|
|
358
|
+
const resolved = message ?? copy?.message ?? `Fix accessibility issue: ${id}.`;
|
|
344
359
|
findings.push({
|
|
345
360
|
id,
|
|
346
361
|
severity: "warning",
|
|
347
362
|
category: "a11y",
|
|
348
363
|
scope: "page",
|
|
349
364
|
title: copy?.title ?? humanizeId(id),
|
|
350
|
-
message:
|
|
365
|
+
message: resolved,
|
|
366
|
+
fix: copy?.message ?? resolved,
|
|
351
367
|
urls,
|
|
352
368
|
count: urls.length,
|
|
353
369
|
});
|
|
@@ -439,6 +455,8 @@ function formatFindingMarkdown(finding) {
|
|
|
439
455
|
const lines = [`#### ${finding.title}`, "", finding.message, "", `- Where: ${scopeLabel(finding)}`];
|
|
440
456
|
if (finding.category)
|
|
441
457
|
lines.push(`- Category: ${finding.category}`);
|
|
458
|
+
if (finding.fix)
|
|
459
|
+
lines.push(`- Fix: ${finding.fix}`);
|
|
442
460
|
if (finding.urls && finding.urls.length > 1) {
|
|
443
461
|
lines.push("- Pages:");
|
|
444
462
|
for (const url of finding.urls.slice(0, 20)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent-ready.d.ts","sourceRoot":"","sources":["../../src/checks/agent-ready.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAGrD,6FAA6F;AAC7F,eAAO,MAAM,kBAAkB,8LAarB,CAAC;AAEX,4EAA4E;AAC5E,eAAO,MAAM,eAAe,wEAAyE,CAAC;AAEtG,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE;QACT,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,iBAAiB,EAAE,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,eAAe,EAAE,MAAM,EAAE,CAAC;KAC3B,CAAC;IACF,OAAO,EAAE;QACP,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,mBAAmB,EAAE;QACnB,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,OAAO,CAAC;KAC3B,CAAC;IACF,WAAW,EAAE;QACX,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;IACF,SAAS,EAAE;QACT,aAAa,EAAE,OAAO,CAAC;QACvB,WAAW,EAAE,OAAO,CAAC;QACrB,UAAU,EAAE,OAAO,CAAC;QACpB,GAAG,EAAE,OAAO,CAAC;QACb,YAAY,EAAE,OAAO,CAAC;KACvB,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,qBAAqB,CAAC;IAClC,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,SAAS,CAAC,EAAE;QACV,aAAa,CAAC,EAAE,qBAAqB,CAAC;QACtC,WAAW,CAAC,EAAE,qBAAqB,CAAC;QACpC,UAAU,CAAC,EAAE,qBAAqB,CAAC;QACnC,GAAG,CAAC,EAAE,qBAAqB,CAAC;QAC5B,YAAY,CAAC,EAAE,qBAAqB,CAAC;KACtC,CAAC;CACH,CAAC;AAoDF,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,sBAAsB,GAAG,qBAAqB,
|
|
1
|
+
{"version":3,"file":"agent-ready.d.ts","sourceRoot":"","sources":["../../src/checks/agent-ready.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAGrD,6FAA6F;AAC7F,eAAO,MAAM,kBAAkB,8LAarB,CAAC;AAEX,4EAA4E;AAC5E,eAAO,MAAM,eAAe,wEAAyE,CAAC;AAEtG,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE;QACT,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,iBAAiB,EAAE,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,eAAe,EAAE,MAAM,EAAE,CAAC;KAC3B,CAAC;IACF,OAAO,EAAE;QACP,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,mBAAmB,EAAE;QACnB,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,OAAO,CAAC;KAC3B,CAAC;IACF,WAAW,EAAE;QACX,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;IACF,SAAS,EAAE;QACT,aAAa,EAAE,OAAO,CAAC;QACvB,WAAW,EAAE,OAAO,CAAC;QACrB,UAAU,EAAE,OAAO,CAAC;QACpB,GAAG,EAAE,OAAO,CAAC;QACb,YAAY,EAAE,OAAO,CAAC;KACvB,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,qBAAqB,CAAC;IAClC,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,SAAS,CAAC,EAAE;QACV,aAAa,CAAC,EAAE,qBAAqB,CAAC;QACtC,WAAW,CAAC,EAAE,qBAAqB,CAAC;QACpC,UAAU,CAAC,EAAE,qBAAqB,CAAC;QACnC,GAAG,CAAC,EAAE,qBAAqB,CAAC;QAC5B,YAAY,CAAC,EAAE,qBAAqB,CAAC;KACtC,CAAC;CACH,CAAC;AAoDF,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,sBAAsB,GAAG,qBAAqB,CAoJtF;AAmCD,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,SAAS,GAAG,OAAO,CAAC,qBAAqB,CAAC,CA+C7G"}
|
|
@@ -96,31 +96,31 @@ export function analyzeAgentReady(input) {
|
|
|
96
96
|
};
|
|
97
97
|
const robotsStatus = input.robotsTxt?.status ?? null;
|
|
98
98
|
if (!isSuccessfulStatus(robotsStatus) || !robotsBody.trim()) {
|
|
99
|
-
issues.push(issue("agent-robots-missing", "warning", "robots.txt is missing or empty; agents cannot discover crawl policy or Content Signals."));
|
|
99
|
+
issues.push(issue("agent-robots-missing", "warning", "robots.txt is missing or empty; agents cannot discover crawl policy or Content Signals.", "Add a robots.txt at the site root (for example public/robots.txt) with at least `User-agent: *` and a Sitemap line."));
|
|
100
100
|
}
|
|
101
101
|
else {
|
|
102
102
|
if (contentSignals.length === 0) {
|
|
103
|
-
issues.push(issue("agent-missing-content-signals", "warning", "robots.txt has no Content-Signal directives (ai-train / search / ai-input)."));
|
|
103
|
+
issues.push(issue("agent-missing-content-signals", "warning", "robots.txt has no Content-Signal directives (ai-train / search / ai-input).", "Add a Content-Signal under `User-agent: *` in robots.txt, for example `Content-Signal: search=yes,ai-train=no`."));
|
|
104
104
|
}
|
|
105
105
|
if (aiBotUserAgents.length === 0) {
|
|
106
|
-
issues.push(issue("agent-missing-ai-bot-rules", "warning", "robots.txt has no explicit AI bot User-agent rules (e.g. GPTBot, ClaudeBot, Google-Extended)."));
|
|
106
|
+
issues.push(issue("agent-missing-ai-bot-rules", "warning", "robots.txt has no explicit AI bot User-agent rules (e.g. GPTBot, ClaudeBot, Google-Extended).", "Add explicit `User-agent` blocks for AI crawlers (GPTBot, ClaudeBot, Google-Extended, …) with Allow/Disallow as intended."));
|
|
107
107
|
}
|
|
108
108
|
}
|
|
109
109
|
if (!llmsOk) {
|
|
110
|
-
issues.push(issue("agent-missing-llms-txt", "warning", "llms.txt is missing or not a plain-text overview at /llms.txt."));
|
|
110
|
+
issues.push(issue("agent-missing-llms-txt", "warning", "llms.txt is missing or not a plain-text overview at /llms.txt.", "Add /llms.txt as plain text with an H1 title (`# Site Name`) and links to the most important content."));
|
|
111
111
|
}
|
|
112
112
|
else if (!llmsHasTitle) {
|
|
113
|
-
issues.push(issue("agent-llms-txt-incomplete", "review", "llms.txt exists but has no H1 title line (# Site Name)."));
|
|
113
|
+
issues.push(issue("agent-llms-txt-incomplete", "review", "llms.txt exists but has no H1 title line (# Site Name).", "Start llms.txt with an H1 line, for example `# Site Name`."));
|
|
114
114
|
}
|
|
115
115
|
if (!markdownOk) {
|
|
116
|
-
issues.push(issue("agent-missing-markdown-negotiation", "warning", "Homepage does not negotiate Markdown (Accept: text/markdown → Content-Type: text/markdown)."));
|
|
116
|
+
issues.push(issue("agent-missing-markdown-negotiation", "warning", "Homepage does not negotiate Markdown (Accept: text/markdown → Content-Type: text/markdown).", "When the request has `Accept: text/markdown`, return the page body as Markdown with `Content-Type: text/markdown` (middleware or route)."));
|
|
117
117
|
}
|
|
118
118
|
if (agentLinkRelations.length === 0) {
|
|
119
|
-
issues.push(issue("agent-missing-link-headers", "review", 'Homepage has no agent-useful Link headers (rel="api-catalog", "service-doc", "service-desc", or "describedby").'));
|
|
119
|
+
issues.push(issue("agent-missing-link-headers", "review", 'Homepage has no agent-useful Link headers (rel="api-catalog", "service-doc", "service-desc", or "describedby").', 'Add Link response headers on the homepage, for example `Link: </.well-known/api-catalog>; rel="api-catalog"`.'));
|
|
120
120
|
}
|
|
121
121
|
const wellKnownPresent = Object.values(wellKnown).some(Boolean);
|
|
122
122
|
if (!wellKnownPresent) {
|
|
123
|
-
issues.push(issue("agent-missing-protocol-discovery", "review", "No agent protocol discovery endpoints found (MCP Server Card, Agent Skills, API Catalog, ARD, or A2A Agent Card)."));
|
|
123
|
+
issues.push(issue("agent-missing-protocol-discovery", "review", "No agent protocol discovery endpoints found (MCP Server Card, Agent Skills, API Catalog, ARD, or A2A Agent Card).", "Add /.well-known/agent-skills/index.json. Use an empty array if no skills are available (`skills: []`)."));
|
|
124
124
|
}
|
|
125
125
|
const findings = {
|
|
126
126
|
robotsTxt: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/checks/crawl.ts"],"names":[],"mappings":"AAOA,OAAO,EAAuB,KAAK,UAAU,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/checks/crawl.ts"],"names":[],"mappings":"AAOA,OAAO,EAAuB,KAAK,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAMjF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,oBAAoB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC9E,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;CAC/B,CAAC,CAuED;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAiCzF"}
|
package/dist/checks/crawl.js
CHANGED
|
@@ -2,6 +2,7 @@ import { DEFAULT_BURST_DELAY_MS, DEFAULT_CONCURRENCY, DEFAULT_DELAY_MS, DEFAULT_
|
|
|
2
2
|
import { resolveActiveGroups } from "../crawler/checkGroups.js";
|
|
3
3
|
import { buildDefaultOutputPaths } from "../crawler/outputPaths.js";
|
|
4
4
|
import { runCrawler } from "../crawler/runCrawler.js";
|
|
5
|
+
import { selectSmartCrawlUrls } from "../crawler/smartPageSelection.js";
|
|
5
6
|
import { collectSitemapPageUrls } from "./sitemap.js";
|
|
6
7
|
import { isHttpUrl, matchesExcludePattern, normalizeUrl } from "../crawler/urlUtils.js";
|
|
7
8
|
export async function prepareCrawlSeedUrls(options) {
|
|
@@ -51,6 +52,16 @@ export async function prepareCrawlSeedUrls(options) {
|
|
|
51
52
|
}
|
|
52
53
|
}
|
|
53
54
|
seedUrls = seedUrls.filter((url) => !matchesExcludePattern(url, excludePatterns));
|
|
55
|
+
const unlimitedSeedCount = seedUrls.length;
|
|
56
|
+
const smart = selectSmartCrawlUrls(seedUrls, Math.floor(maxPages), {
|
|
57
|
+
startUrl: normalizedStartUrl,
|
|
58
|
+
});
|
|
59
|
+
seedUrls = smart.urls;
|
|
60
|
+
if (unlimitedSeedCount > seedUrls.length) {
|
|
61
|
+
// Intentionally concise: full URL lists stay in sitemap compare / crawl JSON.
|
|
62
|
+
const listCount = smart.tree.listParents.size;
|
|
63
|
+
console.info(`[crawl] smart max-pages: selected ${seedUrls.length}/${unlimitedSeedCount} seed URLs (limit=${Math.floor(maxPages)}, listParents=${listCount})`);
|
|
64
|
+
}
|
|
54
65
|
return { normalizedStartUrl, seedUrls, outputPath, activeGroups };
|
|
55
66
|
}
|
|
56
67
|
export async function runCrawlCheck(options) {
|
package/dist/checks/dns.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dns.d.ts","sourceRoot":"","sources":["../../src/checks/dns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,MAAM,UAAU,GAAG;IACvB,CAAC,EAAE,MAAM,EAAE,CAAC;IACZ,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,EAAE,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClD,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,EAAE,EAAE,MAAM,EAAE,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC;IACpB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AA2CF,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,
|
|
1
|
+
{"version":3,"file":"dns.d.ts","sourceRoot":"","sources":["../../src/checks/dns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,MAAM,UAAU,GAAG;IACvB,CAAC,EAAE,MAAM,EAAE,CAAC;IACZ,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,EAAE,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClD,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,EAAE,EAAE,MAAM,EAAE,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC;IACpB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AA2CF,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAqD9E;AAED,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAE3E"}
|
package/dist/checks/dns.js
CHANGED
|
@@ -56,10 +56,10 @@ export async function analyzeDnsRecords(input) {
|
|
|
56
56
|
const dmarc = (await resolveDmarc(hostname)) ?? findDmarc(hostname, flatTxt);
|
|
57
57
|
const skipMailAuthIssues = isPlatformPreviewHostname(hostname);
|
|
58
58
|
if (!skipMailAuthIssues && !spf) {
|
|
59
|
-
issues.push(issue("missing-spf", "warning", "No SPF TXT record found."));
|
|
59
|
+
issues.push(issue("missing-spf", "warning", "No SPF TXT record found.", `Add a TXT record on ${hostname} starting with \`v=spf1\` that lists your mail senders.`));
|
|
60
60
|
}
|
|
61
61
|
if (!skipMailAuthIssues && !dmarc) {
|
|
62
|
-
issues.push(issue("missing-dmarc", "warning", "No DMARC TXT record found."));
|
|
62
|
+
issues.push(issue("missing-dmarc", "warning", "No DMARC TXT record found.", `Add a TXT record on \`_dmarc.${hostname}\` starting with \`v=DMARC1;\` (for example \`p=none; rua=mailto:dmarc@${hostname}\`).`));
|
|
63
63
|
}
|
|
64
64
|
return {
|
|
65
65
|
hostname,
|
package/dist/checks/issues.d.ts
CHANGED
|
@@ -3,6 +3,8 @@ export type CheckIssue = {
|
|
|
3
3
|
id: string;
|
|
4
4
|
severity: CheckIssueSeverity;
|
|
5
5
|
message: string;
|
|
6
|
+
/** Concrete remediation suggestion for reports / agents. */
|
|
7
|
+
fix?: string;
|
|
6
8
|
};
|
|
7
|
-
export declare function issue(id: string, severity: CheckIssueSeverity, message: string): CheckIssue;
|
|
9
|
+
export declare function issue(id: string, severity: CheckIssueSeverity, message: string, fix?: string): CheckIssue;
|
|
8
10
|
//# sourceMappingURL=issues.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"issues.d.ts","sourceRoot":"","sources":["../../src/checks/issues.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE7E,MAAM,MAAM,UAAU,GAAG;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"issues.d.ts","sourceRoot":"","sources":["../../src/checks/issues.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE7E,MAAM,MAAM,UAAU,GAAG;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,wBAAgB,KAAK,CACnB,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,EAAE,MAAM,EACf,GAAG,CAAC,EAAE,MAAM,GACX,UAAU,CAEZ"}
|
package/dist/checks/issues.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export function issue(id, severity, message) {
|
|
2
|
-
return { id, severity, message };
|
|
1
|
+
export function issue(id, severity, message, fix) {
|
|
2
|
+
return fix === undefined ? { id, severity, message } : { id, severity, message, fix };
|
|
3
3
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redirect.d.ts","sourceRoot":"","sources":["../../src/checks/redirect.ts"],"names":[],"mappings":"AAEA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,eAAO,MAAM,qBAAqB,aAAqC,CAAC;AACxE,eAAO,MAAM,+BAA+B,aAAsB,CAAC;AAEnE,MAAM,MAAM,kBAAkB,GAAG;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,kBAAkB,GAAG;IACrD,UAAU,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC;IACtB,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CAClC,CAAC;AAEF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAG1D;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAIzD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,GAAG,MAAM,CAE/G;AAMD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAM7D;AAED,eAAO,MAAM,uBAAuB,EAAE,OAAO,EAAuB,CAAC;AAmBrE,wBAAgB,eAAe,CAAC,KAAK,EAAE,kBAAkB,GAAG,mBAAmB,
|
|
1
|
+
{"version":3,"file":"redirect.d.ts","sourceRoot":"","sources":["../../src/checks/redirect.ts"],"names":[],"mappings":"AAEA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,eAAO,MAAM,qBAAqB,aAAqC,CAAC;AACxE,eAAO,MAAM,+BAA+B,aAAsB,CAAC;AAEnE,MAAM,MAAM,kBAAkB,GAAG;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,kBAAkB,GAAG;IACrD,UAAU,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC;IACtB,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CAClC,CAAC;AAEF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAG1D;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAIzD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,GAAG,MAAM,CAE/G;AAMD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAM7D;AAED,eAAO,MAAM,uBAAuB,EAAE,OAAO,EAAuB,CAAC;AAmBrE,wBAAgB,eAAe,CAAC,KAAK,EAAE,kBAAkB,GAAG,mBAAmB,CA0E9E;AAED,wBAAgB,uBAAuB,CACrC,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,IAAI,CAAC,kBAAkB,EAAE,KAAK,GAAG,QAAQ,GAAG,UAAU,GAAG,gBAAgB,GAAG,cAAc,CAAC,GACjG,UAAU,EAAE,CAgFd;AAED,wBAAgB,iBAAiB,CAAC,eAAe,EAAE,OAAO,EAAE,GAAG,OAAO,CAKrE;AAED,wBAAgB,wBAAwB,CACtC,eAAe,EAAE,OAAO,EAAE,EAC1B,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,GAClD,OAAO,EAAE,CAWX;AAED,wBAAgB,uBAAuB,CAAC,eAAe,EAAE,OAAO,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,CAGzG;AA0DD,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,SAAqB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CA8BrH"}
|
package/dist/checks/redirect.js
CHANGED
|
@@ -62,19 +62,19 @@ export function analyzeRedirect(input) {
|
|
|
62
62
|
};
|
|
63
63
|
}
|
|
64
64
|
if (!input.location) {
|
|
65
|
-
issues.push(issue("redirect-missing-location", "error", "Redirect response is missing a Location header."));
|
|
65
|
+
issues.push(issue("redirect-missing-location", "error", "Redirect response is missing a Location header.", "Include a valid Location header on redirect responses pointing at the canonical URL."));
|
|
66
66
|
}
|
|
67
67
|
else if (!input.resolvedTarget) {
|
|
68
|
-
issues.push(issue("redirect-invalid-location", "error", "Redirect Location header is not a valid URL."));
|
|
68
|
+
issues.push(issue("redirect-invalid-location", "error", "Redirect Location header is not a valid URL.", "Set Location to an absolute http(s) URL for the redirect target."));
|
|
69
69
|
}
|
|
70
70
|
if (input.status !== null && !PERMANENT_REDIRECT_STATUS_CODES.has(input.status)) {
|
|
71
|
-
issues.push(issue("redirect-temporary-status", "warning", `Redirect uses status ${input.status}; use 301 or 308 for permanent domain redirects
|
|
71
|
+
issues.push(issue("redirect-temporary-status", "warning", `Redirect uses status ${input.status}; use 301 or 308 for permanent domain redirects.`, "Change the redirect status to 301 or 308 for permanent domain redirects."));
|
|
72
72
|
}
|
|
73
73
|
if (isHttps && !input.headers["strict-transport-security"]) {
|
|
74
|
-
issues.push(issue("redirect-missing-hsts", "warning", "Strict-Transport-Security header is missing on the redirect response."));
|
|
74
|
+
issues.push(issue("redirect-missing-hsts", "warning", "Strict-Transport-Security header is missing on the redirect response.", "Send Strict-Transport-Security on the HTTPS redirect response as well as on final pages."));
|
|
75
75
|
}
|
|
76
76
|
if (input.resolvedTarget && input.targetStatus !== null && input.targetStatus >= 400) {
|
|
77
|
-
issues.push(issue("redirect-target-unreachable", "error", `Redirect target returned status ${input.targetStatus}: ${input.resolvedTarget}
|
|
77
|
+
issues.push(issue("redirect-target-unreachable", "error", `Redirect target returned status ${input.targetStatus}: ${input.resolvedTarget}`, "Fix the redirect target so it returns a successful response, or point Location at a working URL."));
|
|
78
78
|
}
|
|
79
79
|
return {
|
|
80
80
|
...input,
|
|
@@ -87,27 +87,27 @@ export function analyzeWwwAliasRedirect(canonicalUrl, probe) {
|
|
|
87
87
|
const issues = [];
|
|
88
88
|
const canonicalHost = hostnameFromUrl(canonicalUrl);
|
|
89
89
|
if (probe.status === null) {
|
|
90
|
-
issues.push(issue("redirect-www-alias-unreachable", "error", `Alternate host is unreachable and should redirect to ${canonicalHost}: ${probe.url}`));
|
|
90
|
+
issues.push(issue("redirect-www-alias-unreachable", "error", `Alternate host is unreachable and should redirect to ${canonicalHost}: ${probe.url}`, `Configure DNS and hosting so the alternate host responds and permanently redirects to ${canonicalHost}.`));
|
|
91
91
|
return issues;
|
|
92
92
|
}
|
|
93
93
|
if (!REDIRECT_STATUS_CODES.has(probe.status)) {
|
|
94
|
-
issues.push(issue("redirect-www-alias-no-redirect", "error", `Alternate host should redirect to ${canonicalHost} but returned status ${probe.status}: ${probe.url}`));
|
|
94
|
+
issues.push(issue("redirect-www-alias-no-redirect", "error", `Alternate host should redirect to ${canonicalHost} but returned status ${probe.status}: ${probe.url}`, `Configure the alternate host to respond with 301/308 Location pointing at ${canonicalHost}.`));
|
|
95
95
|
return issues;
|
|
96
96
|
}
|
|
97
97
|
if (!probe.location) {
|
|
98
|
-
issues.push(issue("redirect-www-alias-missing-location", "error", `Alternate host redirect is missing a Location header: ${probe.url}`));
|
|
98
|
+
issues.push(issue("redirect-www-alias-missing-location", "error", `Alternate host redirect is missing a Location header: ${probe.url}`, `Add a Location header on the alternate-host redirect targeting ${canonicalHost}.`));
|
|
99
99
|
}
|
|
100
100
|
else if (!probe.resolvedTarget) {
|
|
101
|
-
issues.push(issue("redirect-www-alias-invalid-location", "error", `Alternate host redirect Location header is not a valid URL: ${probe.url}`));
|
|
101
|
+
issues.push(issue("redirect-www-alias-invalid-location", "error", `Alternate host redirect Location header is not a valid URL: ${probe.url}`, `Set Location to an absolute http(s) URL on ${canonicalHost}.`));
|
|
102
102
|
}
|
|
103
103
|
else if (hostnameFromUrl(probe.resolvedTarget) !== canonicalHost) {
|
|
104
|
-
issues.push(issue("redirect-www-alias-wrong-target", "error", `Alternate host redirects to ${probe.resolvedTarget} instead of ${canonicalHost}: ${probe.url}`));
|
|
104
|
+
issues.push(issue("redirect-www-alias-wrong-target", "error", `Alternate host redirects to ${probe.resolvedTarget} instead of ${canonicalHost}: ${probe.url}`, `Point the alternate-host redirect at the canonical hostname (${canonicalHost}).`));
|
|
105
105
|
}
|
|
106
106
|
if (probe.status !== null && !PERMANENT_REDIRECT_STATUS_CODES.has(probe.status)) {
|
|
107
|
-
issues.push(issue("redirect-www-alias-temporary-status", "warning", `Alternate host redirect uses status ${probe.status}; use 301 or 308 for permanent domain redirects: ${probe.url}
|
|
107
|
+
issues.push(issue("redirect-www-alias-temporary-status", "warning", `Alternate host redirect uses status ${probe.status}; use 301 or 308 for permanent domain redirects: ${probe.url}`, "Change the alternate-host redirect status to 301 or 308."));
|
|
108
108
|
}
|
|
109
109
|
if (probe.resolvedTarget && probe.targetStatus !== null && probe.targetStatus >= 400) {
|
|
110
|
-
issues.push(issue("redirect-www-alias-target-unreachable", "error", `Alternate host redirect target returned status ${probe.targetStatus}: ${probe.resolvedTarget}
|
|
110
|
+
issues.push(issue("redirect-www-alias-target-unreachable", "error", `Alternate host redirect target returned status ${probe.targetStatus}: ${probe.resolvedTarget}`, "Fix the canonical target URL so the alternate-host redirect lands on a successful page."));
|
|
111
111
|
}
|
|
112
112
|
return issues;
|
|
113
113
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robots-txt.d.ts","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAGrD,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAgDF,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,qBAAqB,GAAG,oBAAoB,
|
|
1
|
+
{"version":3,"file":"robots-txt.d.ts","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAGrD,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAgDF,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,qBAAqB,GAAG,oBAAoB,CAqFnF;AAED,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,SAAS,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAmB3G"}
|
|
@@ -47,15 +47,15 @@ export function analyzeRobotsTxt(input) {
|
|
|
47
47
|
const issues = [];
|
|
48
48
|
const { url, status, body } = input;
|
|
49
49
|
if (status === null || status === 404) {
|
|
50
|
-
issues.push(issue("robots-missing", "warning", "robots.txt is missing."));
|
|
50
|
+
issues.push(issue("robots-missing", "warning", "robots.txt is missing.", "Publish /robots.txt at the site origin with at least `User-agent: *` and a Sitemap directive."));
|
|
51
51
|
return { url, status, body, sitemapUrls: [], issues };
|
|
52
52
|
}
|
|
53
53
|
if (!body) {
|
|
54
|
-
issues.push(issue("robots-unparseable", "error", "robots.txt response body is empty."));
|
|
54
|
+
issues.push(issue("robots-unparseable", "error", "robots.txt response body is empty.", "Return a non-empty robots.txt body with valid User-agent / Allow / Disallow / Sitemap lines."));
|
|
55
55
|
return { url, status, body, sitemapUrls: [], issues };
|
|
56
56
|
}
|
|
57
57
|
if (hasMalformedRobotsLines(body)) {
|
|
58
|
-
issues.push(issue("robots-unparseable", "error", "robots.txt contains malformed directives."));
|
|
58
|
+
issues.push(issue("robots-unparseable", "error", "robots.txt contains malformed directives.", "Fix robots.txt syntax: one directive per line (`User-agent:`, `Allow:`, `Disallow:`, `Sitemap:`)."));
|
|
59
59
|
}
|
|
60
60
|
let sitemapUrls = [];
|
|
61
61
|
try {
|
|
@@ -64,17 +64,17 @@ export function analyzeRobotsTxt(input) {
|
|
|
64
64
|
sitemapUrls = parsed.getSitemaps();
|
|
65
65
|
}
|
|
66
66
|
catch {
|
|
67
|
-
issues.push(issue("robots-unparseable", "error", "robots.txt could not be parsed."));
|
|
67
|
+
issues.push(issue("robots-unparseable", "error", "robots.txt could not be parsed.", "Validate robots.txt content and encoding; ensure it is plain text with standard directives."));
|
|
68
68
|
return { url, status, body, sitemapUrls: [], issues };
|
|
69
69
|
}
|
|
70
70
|
if (sitemapUrls.length === 0) {
|
|
71
71
|
sitemapUrls = extractSitemapUrls(body);
|
|
72
72
|
}
|
|
73
73
|
if (blocksAllUserAgents(body, url)) {
|
|
74
|
-
issues.push(issue("robots-blocks-all", "critical", "robots.txt disallows all crawling for User-agent: *."));
|
|
74
|
+
issues.push(issue("robots-blocks-all", "critical", "robots.txt disallows all crawling for User-agent: *.", "Remove or narrow `Disallow: /` under `User-agent: *` so search engines and agents can crawl public pages."));
|
|
75
75
|
}
|
|
76
76
|
if (sitemapUrls.length === 0) {
|
|
77
|
-
issues.push(issue("robots-missing-sitemap", "warning", "robots.txt has no Sitemap directive."));
|
|
77
|
+
issues.push(issue("robots-missing-sitemap", "warning", "robots.txt has no Sitemap directive.", "Add a Sitemap line pointing at your sitemap, for example `Sitemap: https://example.com/sitemap.xml`."));
|
|
78
78
|
}
|
|
79
79
|
return { url, status, body, sitemapUrls, issues };
|
|
80
80
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"security-headers.d.ts","sourceRoot":"","sources":["../../src/checks/security-headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,MAAM,6BAA6B,GAAG;IAC1C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAWF,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,6BAAkC,GAC1C,UAAU,EAAE,
|
|
1
|
+
{"version":3,"file":"security-headers.d.ts","sourceRoot":"","sources":["../../src/checks/security-headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,MAAM,6BAA6B,GAAG;IAC1C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAWF,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,6BAAkC,GAC1C,UAAU,EAAE,CA8Ed;AAED,MAAM,MAAM,0BAA0B,GAAG;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF,wBAAsB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAgBnG"}
|
|
@@ -17,22 +17,22 @@ export function analyzeSecurityHeaders(headers, options = {}) {
|
|
|
17
17
|
const referrerPolicy = headerValue(headers, "referrer-policy");
|
|
18
18
|
const permissionsPolicy = headerValue(headers, "permissions-policy");
|
|
19
19
|
if (isHttps && !hsts) {
|
|
20
|
-
issues.push(issue("missing-hsts", "error", "Strict-Transport-Security header is missing on HTTPS."));
|
|
20
|
+
issues.push(issue("missing-hsts", "error", "Strict-Transport-Security header is missing on HTTPS.", "Send `Strict-Transport-Security: max-age=31536000; includeSubDomains` on HTTPS responses (hosting or app headers)."));
|
|
21
21
|
}
|
|
22
22
|
if (!csp) {
|
|
23
|
-
issues.push(issue("missing-csp", "warning", "Content-Security-Policy header is missing."));
|
|
23
|
+
issues.push(issue("missing-csp", "warning", "Content-Security-Policy header is missing.", "Add a Content-Security-Policy header tailored to your app (start with report-only if needed)."));
|
|
24
24
|
}
|
|
25
25
|
if (!xFrameOptions && !hasFrameAncestorsCsp(csp)) {
|
|
26
|
-
issues.push(issue("missing-clickjacking-protection", "warning", "Neither X-Frame-Options nor CSP frame-ancestors is set."));
|
|
26
|
+
issues.push(issue("missing-clickjacking-protection", "warning", "Neither X-Frame-Options nor CSP frame-ancestors is set.", "Set `X-Frame-Options: DENY` (or SAMEORIGIN) or add `frame-ancestors` to your Content-Security-Policy."));
|
|
27
27
|
}
|
|
28
28
|
if (!nosniff || nosniff.toLowerCase() !== "nosniff") {
|
|
29
|
-
issues.push(issue("missing-nosniff", "warning", "X-Content-Type-Options: nosniff is missing."));
|
|
29
|
+
issues.push(issue("missing-nosniff", "warning", "X-Content-Type-Options: nosniff is missing.", "Send `X-Content-Type-Options: nosniff` on all responses."));
|
|
30
30
|
}
|
|
31
31
|
if (!referrerPolicy) {
|
|
32
|
-
issues.push(issue("missing-referrer-policy", "review", "Referrer-Policy header is missing."));
|
|
32
|
+
issues.push(issue("missing-referrer-policy", "review", "Referrer-Policy header is missing.", "Send a Referrer-Policy such as `strict-origin-when-cross-origin`."));
|
|
33
33
|
}
|
|
34
34
|
if (!permissionsPolicy) {
|
|
35
|
-
issues.push(issue("missing-permissions-policy", "review", "Permissions-Policy header is missing."));
|
|
35
|
+
issues.push(issue("missing-permissions-policy", "review", "Permissions-Policy header is missing.", "Add a Permissions-Policy header that disables unused browser features (camera, mic, geolocation, …)."));
|
|
36
36
|
}
|
|
37
37
|
return issues;
|
|
38
38
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/checks/sitemap.ts"],"names":[],"mappings":"AAGA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAMrD,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B,CAAC;AAQF,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,0BAA0B,GAAG,kBAAkB,
|
|
1
|
+
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/checks/sitemap.ts"],"names":[],"mappings":"AAGA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAMrD,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B,CAAC;AAQF,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,0BAA0B,GAAG,kBAAkB,CAyH3F;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAE,MAAM,EAAO,GAAG,MAAM,EAAE,CAIpG;AAkBD,wBAAsB,kBAAkB,CACtC,aAAa,EAAE,MAAM,EAAE,EACvB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAchC;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,IAAI,CACpC,sBAAsB,EACtB,WAAW,GAAG,0BAA0B,GAAG,iBAAiB,CAC7D,CAAC;AAEF,wBAAsB,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAYlG;AAED,wBAAsB,sBAAsB,CAC1C,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,iBAAiB,GAAE,MAAM,EAAO,EAChC,gBAAgB,CAAC,EAAE,MAAM,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC,CAmBnB"}
|
package/dist/checks/sitemap.js
CHANGED
|
@@ -16,12 +16,12 @@ export function analyzeSitemapCompare(input) {
|
|
|
16
16
|
const issues = [];
|
|
17
17
|
for (const source of input.sources) {
|
|
18
18
|
if (source.status === null || source.status >= 400) {
|
|
19
|
-
issues.push(issue("sitemap-unreachable", "error", `Sitemap not reachable: ${source.url} (${source.status ?? "unreachable"}).`));
|
|
19
|
+
issues.push(issue("sitemap-unreachable", "error", `Sitemap not reachable: ${source.url} (${source.status ?? "unreachable"}).`, `Ensure ${source.url} returns HTTP 200 with a valid XML sitemap or sitemap index.`));
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
const successfulSources = input.sources.filter((source) => source.status !== null && source.status >= 200 && source.status < 300);
|
|
23
23
|
if (successfulSources.length === 0) {
|
|
24
|
-
issues.push(issue("sitemap-missing", "error", "No sitemap found."));
|
|
24
|
+
issues.push(issue("sitemap-missing", "error", "No sitemap found.", "Publish /sitemap.xml (or reference it from robots.txt) and make sure it returns HTTP 200."));
|
|
25
25
|
return {
|
|
26
26
|
sources: input.sources,
|
|
27
27
|
sitemapPageUrls: [],
|
|
@@ -35,7 +35,7 @@ export function analyzeSitemapCompare(input) {
|
|
|
35
35
|
const sitemapPageUrls = filterComparableUrls(successfulSources.flatMap((source) => source.pageUrls), excludePatterns);
|
|
36
36
|
const crawledUrls = filterComparableUrls(input.crawledUrls, excludePatterns);
|
|
37
37
|
if (sitemapPageUrls.length === 0) {
|
|
38
|
-
issues.push(issue("sitemap-empty", "error", "Sitemap was reachable but contained no page URLs (check sitemap index nesting and host aliases such as apex vs www)."));
|
|
38
|
+
issues.push(issue("sitemap-empty", "error", "Sitemap was reachable but contained no page URLs (check sitemap index nesting and host aliases such as apex vs www).", "Populate the sitemap with page `<loc>` entries, or fix nested sitemap indexes and apex/www host mismatches."));
|
|
39
39
|
return {
|
|
40
40
|
sources: input.sources,
|
|
41
41
|
sitemapPageUrls,
|
|
@@ -51,17 +51,17 @@ export function analyzeSitemapCompare(input) {
|
|
|
51
51
|
const inSitemapNotCrawled = sitemapPageUrls.filter((url) => !crawledSet.has(url));
|
|
52
52
|
const crawledNotInSitemap = crawledUrls.filter((url) => !sitemapSet.has(url));
|
|
53
53
|
if (inSitemapNotCrawled.length > 0) {
|
|
54
|
-
issues.push(issue("sitemap-url-not-crawled", "critical", `${inSitemapNotCrawled.length} sitemap URL(s) were not crawled. ${SITEMAP_URL_NOT_CRAWLED_TIP}
|
|
54
|
+
issues.push(issue("sitemap-url-not-crawled", "critical", `${inSitemapNotCrawled.length} sitemap URL(s) were not crawled. ${SITEMAP_URL_NOT_CRAWLED_TIP}`, "Remove stale sitemap entries, fix robots blocks, or raise --max-pages so listed URLs can be crawled."));
|
|
55
55
|
}
|
|
56
56
|
if (crawledNotInSitemap.length > 0) {
|
|
57
|
-
issues.push(issue("sitemap-missing-crawled-url", "critical", `${crawledNotInSitemap.length} crawled URL(s) are missing from the sitemap
|
|
57
|
+
issues.push(issue("sitemap-missing-crawled-url", "critical", `${crawledNotInSitemap.length} crawled URL(s) are missing from the sitemap.`, "Add the crawled URLs to the sitemap (or regenerate the sitemap from the live route set)."));
|
|
58
58
|
}
|
|
59
59
|
const total = new Set([...sitemapPageUrls, ...crawledUrls]).size;
|
|
60
60
|
const mismatchCount = new Set([...inSitemapNotCrawled, ...crawledNotInSitemap]).size;
|
|
61
61
|
const mismatchPercent = total === 0 ? 0 : (mismatchCount / total) * 100;
|
|
62
62
|
if (mismatchPercent > threshold) {
|
|
63
63
|
issues.push(issue("sitemap-mismatch-threshold", "critical", `Sitemap mismatch is ${mismatchPercent.toFixed(1)}% (threshold ${threshold}%). ` +
|
|
64
|
-
"Sitemap URLs must match crawled URLs exactly, including hostname."));
|
|
64
|
+
"Sitemap URLs must match crawled URLs exactly, including hostname.", "Align sitemap hostnames and paths with the live site (apex vs www) and regenerate the sitemap."));
|
|
65
65
|
}
|
|
66
66
|
return {
|
|
67
67
|
sources: input.sources,
|
package/dist/cli.js
CHANGED
|
@@ -26,7 +26,7 @@ async function main() {
|
|
|
26
26
|
.option("--errors-only", "Include only critical/error findings in the Markdown report", false)
|
|
27
27
|
.option("--parallel", "Run independent checks in parallel", false)
|
|
28
28
|
.option("--checks <ids>", "Comma-separated checks to run (crawl,analyze,lighthouse,security-headers,dns,robots-txt,sitemap,agent-ready)")
|
|
29
|
-
.option("--max-pages <n>", "Maximum
|
|
29
|
+
.option("--max-pages <n>", "Maximum pages to crawl (default 100; prefers structural pages, samples leaf URLs under large list branches)", String(DEFAULT_MAX_PAGES))
|
|
30
30
|
.option("--delay-ms <n>", "Delay between requests in ms", String(DEFAULT_DELAY_MS))
|
|
31
31
|
.option("--timeout-ms <n>", "Request timeout in ms", String(DEFAULT_TIMEOUT_MS))
|
|
32
32
|
.option("--concurrency <n>", "Maximum concurrent requests", String(DEFAULT_CONCURRENCY))
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** Root directory for ephemeral Crawlee request-queue storage (cwd-relative). */
|
|
2
|
+
export declare const FLIGHTCHECK_STORAGE_ROOT = ".flightcheckStorage";
|
|
3
|
+
/**
|
|
4
|
+
* Create a unique per-run storage path: `.flightcheckStorage/<hash>`.
|
|
5
|
+
* Concurrent runs each get their own subdirectory so cleanup of one does not
|
|
6
|
+
* wipe another run's queue files.
|
|
7
|
+
*/
|
|
8
|
+
export declare function createRunStoragePath(options?: {
|
|
9
|
+
cwd?: string;
|
|
10
|
+
seed?: string;
|
|
11
|
+
}): {
|
|
12
|
+
rootDir: string;
|
|
13
|
+
runDir: string;
|
|
14
|
+
runId: string;
|
|
15
|
+
};
|
|
16
|
+
/** Remove a single run subdirectory. Ignores missing paths. */
|
|
17
|
+
export declare function cleanupRunStorageDir(runDir: string): Promise<void>;
|
|
18
|
+
//# sourceMappingURL=crawleeStorage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawleeStorage.d.ts","sourceRoot":"","sources":["../../src/crawler/crawleeStorage.ts"],"names":[],"mappings":"AAIA,iFAAiF;AACjF,eAAO,MAAM,wBAAwB,wBAAwB,CAAC;AAE9D;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,CAAC,EAAE;IAC7C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CASrD;AAED,+DAA+D;AAC/D,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAExE"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
2
|
+
import { rm } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
/** Root directory for ephemeral Crawlee request-queue storage (cwd-relative). */
|
|
5
|
+
export const FLIGHTCHECK_STORAGE_ROOT = ".flightcheckStorage";
|
|
6
|
+
/**
|
|
7
|
+
* Create a unique per-run storage path: `.flightcheckStorage/<hash>`.
|
|
8
|
+
* Concurrent runs each get their own subdirectory so cleanup of one does not
|
|
9
|
+
* wipe another run's queue files.
|
|
10
|
+
*/
|
|
11
|
+
export function createRunStoragePath(options) {
|
|
12
|
+
const cwd = options?.cwd ?? process.cwd();
|
|
13
|
+
const runId = createHash("sha256")
|
|
14
|
+
.update(`${options?.seed ?? ""}:${Date.now()}:${process.pid}:${randomBytes(8).toString("hex")}`)
|
|
15
|
+
.digest("hex")
|
|
16
|
+
.slice(0, 16);
|
|
17
|
+
const rootDir = path.join(cwd, FLIGHTCHECK_STORAGE_ROOT);
|
|
18
|
+
const runDir = path.join(rootDir, runId);
|
|
19
|
+
return { rootDir, runDir, runId };
|
|
20
|
+
}
|
|
21
|
+
/** Remove a single run subdirectory. Ignores missing paths. */
|
|
22
|
+
export async function cleanupRunStorageDir(runDir) {
|
|
23
|
+
await rm(runDir, { recursive: true, force: true });
|
|
24
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
/** Default crawl budget; smart selection prefers root/list pages within this cap. */
|
|
2
|
+
export declare const DEFAULT_MAX_PAGES = 100;
|
|
2
3
|
export declare const DEFAULT_DELAY_MS = 500;
|
|
3
4
|
export declare const DEFAULT_TIMEOUT_MS = 10000;
|
|
4
5
|
export declare const DEFAULT_CONCURRENCY = 1;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"defaults.d.ts","sourceRoot":"","sources":["../../src/crawler/defaults.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,iBAAiB,
|
|
1
|
+
{"version":3,"file":"defaults.d.ts","sourceRoot":"","sources":["../../src/crawler/defaults.ts"],"names":[],"mappings":"AAAA,qFAAqF;AACrF,eAAO,MAAM,iBAAiB,MAAM,CAAC;AACrC,eAAO,MAAM,gBAAgB,MAAM,CAAC;AACpC,eAAO,MAAM,kBAAkB,QAAQ,CAAC;AACxC,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,sBAAsB,IAAI,CAAC;AACxC,eAAO,MAAM,kBAAkB,kBAAkB,CAAC;AAClD,eAAO,MAAM,0BAA0B,iBAAiB,CAAC;AACzD,eAAO,MAAM,oBAAoB,KAAK,CAAC;AACvC,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAC9C,eAAO,MAAM,kBAAkB,2CAA2C,CAAC"}
|