@eightmedia/flightcheck 1.0.1 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,19 @@
1
1
  # Changelog
2
2
 
3
+ ## [1.1.0] — 2026-10-28
4
+
5
+ ### Added
6
+
7
+ - Findings include an optional `fix` suggestion in `*-report.json` and Markdown (`- Fix: …`)
8
+
9
+ ### Changed
10
+
11
+ - Default `--max-pages` is now **100** (was 1000)
12
+ - Within the page budget, crawl seeds are selected from **URL-tree shape** (not slug
13
+ names): home/structural pages and list indexes first, then a fair sample of leaf
14
+ URLs under parents that have many leaf children
15
+ - Discovered structural links jump the queue; leaf URLs under list parents are capped
16
+
3
17
  ## [1.0.1] — 2026-10-28
4
18
 
5
19
  ### Added
@@ -18,6 +18,8 @@ export type ReportFinding = {
18
18
  scope: FindingScope;
19
19
  title: string;
20
20
  message: string;
21
+ /** Concrete remediation suggestion when available. */
22
+ fix?: string;
21
23
  template?: string;
22
24
  urls?: string[];
23
25
  count?: number;
@@ -1 +1 @@
1
- {"version":3,"file":"structuredReport.d.ts","sourceRoot":"","sources":["../../src/analyze/structuredReport.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAG1E,OAAO,EAEL,KAAK,oBAAoB,EAEzB,KAAK,YAAY,EACjB,KAAK,eAAe,EACrB,MAAM,aAAa,CAAC;AAErB,eAAO,MAAM,0BAA0B,EAAG,CAAU,CAAC;AAErD,MAAM,MAAM,eAAe,GAAG,kBAAkB,CAAC;AACjD,MAAM,MAAM,eAAe,GACvB,OAAO,GACP,KAAK,GACL,MAAM,GACN,gBAAgB,GAChB,UAAU,GACV,aAAa,CAAC;AAClB,MAAM,MAAM,YAAY,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAC;AAExE,MAAM,MAAM,gBAAgB,GAAG;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,eAAe,CAAC;IAC1B,QAAQ,EAAE,eAAe,CAAC;IAC1B,KAAK,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC/C,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,GAAG,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC9C,QAAQ,CAAC,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,WAAW,EAAE,gBAAgB,EAAE,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,gEAAgE;IAChE,OAAO,EAAE,OAAO,0BAA0B,CAAC;IAC3C,oEAAoE;IACpE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mGAAmG;IACnG,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,QAAQ,EAAE,aAAa,EAAE,CAAC;CAC3B,CAAC;AAidF,MAAM,MAAM,4BAA4B,GAAG,oBAAoB,GAAG;IAChE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,wBAAsB,qBAAqB,CACzC,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,YAAY,EAAE,EACvB,KAAK,GAAE,eAAoB,EAC3B,OAAO,GAAE,4BAAiC,GACzC,OAAO,CAAC,iBAAiB,CAAC,CAa5B;AAgCD,4FAA4F;AAC5F,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,CAuFtE"}
1
+ {"version":3,"file":"structuredReport.d.ts","sourceRoot":"","sources":["../../src/analyze/structuredReport.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAG1E,OAAO,EAEL,KAAK,oBAAoB,EAEzB,KAAK,YAAY,EACjB,KAAK,eAAe,EACrB,MAAM,aAAa,CAAC;AAErB,eAAO,MAAM,0BAA0B,EAAG,CAAU,CAAC;AAErD,MAAM,MAAM,eAAe,GAAG,kBAAkB,CAAC;AACjD,MAAM,MAAM,eAAe,GACvB,OAAO,GACP,KAAK,GACL,MAAM,GACN,gBAAgB,GAChB,UAAU,GACV,aAAa,CAAC;AAClB,MAAM,MAAM,YAAY,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAC;AAExE,MAAM,MAAM,gBAAgB,GAAG;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,eAAe,CAAC;IAC1B,QAAQ,EAAE,eAAe,CAAC;IAC1B,KAAK,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,sDAAsD;IACtD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC/C,KAAK,EAAE,gBAAgB,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,GAAG,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC9C,QAAQ,CAAC,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,WAAW,EAAE,gBAAgB,EAAE,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,gEAAgE;IAChE,OAAO,EAAE,OAAO,0BAA0B,CAAC;IAC3C,oEAAoE;IACpE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mGAAmG;IACnG,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,QAAQ,EAAE,aAAa,EAAE,CAAC;CAC3B,CAAC;AAieF,MAAM,MAAM,4BAA4B,GAAG,oBAAoB,GAAG;IAChE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,wBAAsB,qBAAqB,CACzC,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,YAAY,EAAE,EACvB,KAAK,GAAE,eAAoB,EAC3B,OAAO,GAAE,4BAAiC,GACzC,OAAO,CAAC,iBAAiB,CAAC,CAa5B;AAiCD,4FAA4F;AAC5F,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,CAuFtE"}
@@ -170,6 +170,7 @@ function findingFromCheck(issue, category, scope, extras = {}) {
170
170
  scope,
171
171
  title: humanizeId(issue.id),
172
172
  message: issue.message,
173
+ ...(issue.fix ? { fix: issue.fix } : {}),
173
174
  ...extras,
174
175
  };
175
176
  }
@@ -182,29 +183,35 @@ function pushUnique(findings, finding) {
182
183
  findings.push(finding);
183
184
  }
184
185
  function collectSeoFindings(fields, scope, count, extras = {}) {
185
- return fields.map((field) => ({
186
- id: field,
187
- severity: "error",
188
- category: "seo",
189
- scope,
190
- title: fieldTitle(field),
191
- message: fieldMessage(field, scope, extras.template),
192
- count,
193
- ...extras,
194
- }));
186
+ return fields.map((field) => {
187
+ const message = fieldMessage(field, scope, extras.template);
188
+ return {
189
+ id: field,
190
+ severity: "error",
191
+ category: "seo",
192
+ scope,
193
+ title: fieldTitle(field),
194
+ message,
195
+ fix: message,
196
+ count,
197
+ ...extras,
198
+ };
199
+ });
195
200
  }
196
201
  function collectBestPracticeFindings(keys, scope, count, extras = {}) {
197
202
  return keys
198
203
  .filter((key) => BEST_PRACTICE_IDS.has(key))
199
204
  .map((key) => {
200
205
  const copy = BEST_PRACTICE_COPY[key];
206
+ const message = copy?.message ?? `Fix ${key}.`;
201
207
  return {
202
208
  id: key,
203
209
  severity: copy?.severity ?? "review",
204
210
  category: "best-practices",
205
211
  scope,
206
212
  title: copy?.title ?? humanizeId(key),
207
- message: copy?.message ?? `Fix ${key}.`,
213
+ message,
214
+ fix: message,
208
215
  count,
209
216
  ...extras,
210
217
  };
@@ -215,13 +222,15 @@ function collectA11yFindings(keys, scope, count, extras = {}) {
215
222
  .filter((key) => A11Y_IDS.has(key))
216
223
  .map((key) => {
217
224
  const copy = A11Y_COPY[key];
225
+ const message = copy?.message ?? `Fix accessibility issue: ${key}.`;
218
226
  return {
219
227
  id: key,
220
228
  severity: "warning",
221
229
  category: "a11y",
222
230
  scope,
223
231
  title: copy?.title ?? humanizeId(key),
224
- message: copy?.message ?? `Fix accessibility issue: ${key}.`,
232
+ message,
233
+ fix: message,
225
234
  count,
226
235
  ...extras,
227
236
  };
@@ -284,6 +293,7 @@ function buildFindings(analysis, tier1) {
284
293
  scope: "page",
285
294
  title: `Exposed ${secret.provider} ${secret.type}`,
286
295
  message: `Rotate or remove this secret. Masked value: ${secret.masked} (source: ${secret.source}).`,
296
+ fix: "Remove the secret from client-side HTML/JS, rotate the credential, and load secrets only from a server-side env.",
287
297
  urls: [page.url],
288
298
  count: 1,
289
299
  });
@@ -313,13 +323,15 @@ function buildFindings(analysis, tier1) {
313
323
  if (!analysis.redirectOnly) {
314
324
  const seoById = aggregatePageFindings(analysis.pageFixes, (fix) => fix.remainingFields.map((id) => ({ id })));
315
325
  for (const [id, { urls }] of seoById) {
326
+ const message = fieldMessage(id, "page");
316
327
  findings.push({
317
328
  id,
318
329
  severity: "error",
319
330
  category: "seo",
320
331
  scope: "page",
321
332
  title: fieldTitle(id),
322
- message: fieldMessage(id, "page"),
333
+ message,
334
+ fix: message,
323
335
  urls,
324
336
  count: urls.length,
325
337
  });
@@ -327,13 +339,15 @@ function buildFindings(analysis, tier1) {
327
339
  const bestById = aggregatePageFindings(analysis.pageFixes, (fix) => fix.remainingWarningKeys.filter((key) => BEST_PRACTICE_IDS.has(key)).map((id) => ({ id })));
328
340
  for (const [id, { urls }] of bestById) {
329
341
  const copy = BEST_PRACTICE_COPY[id];
342
+ const message = copy?.message ?? `Fix ${id}.`;
330
343
  findings.push({
331
344
  id,
332
345
  severity: copy?.severity ?? "review",
333
346
  category: "best-practices",
334
347
  scope: "page",
335
348
  title: copy?.title ?? humanizeId(id),
336
- message: copy?.message ?? `Fix ${id}.`,
349
+ message,
350
+ fix: message,
337
351
  urls,
338
352
  count: urls.length,
339
353
  });
@@ -341,13 +355,15 @@ function buildFindings(analysis, tier1) {
341
355
  const a11yById = aggregatePageFindings(analysis.pageFixes, (fix) => fix.remainingA11yIssues.map((issue) => ({ id: issue.id, message: issue.message })));
342
356
  for (const [id, { urls, message }] of a11yById) {
343
357
  const copy = A11Y_COPY[id];
358
+ const resolved = message ?? copy?.message ?? `Fix accessibility issue: ${id}.`;
344
359
  findings.push({
345
360
  id,
346
361
  severity: "warning",
347
362
  category: "a11y",
348
363
  scope: "page",
349
364
  title: copy?.title ?? humanizeId(id),
350
- message: message ?? copy?.message ?? `Fix accessibility issue: ${id}.`,
365
+ message: resolved,
366
+ fix: copy?.message ?? resolved,
351
367
  urls,
352
368
  count: urls.length,
353
369
  });
@@ -439,6 +455,8 @@ function formatFindingMarkdown(finding) {
439
455
  const lines = [`#### ${finding.title}`, "", finding.message, "", `- Where: ${scopeLabel(finding)}`];
440
456
  if (finding.category)
441
457
  lines.push(`- Category: ${finding.category}`);
458
+ if (finding.fix)
459
+ lines.push(`- Fix: ${finding.fix}`);
442
460
  if (finding.urls && finding.urls.length > 1) {
443
461
  lines.push("- Pages:");
444
462
  for (const url of finding.urls.slice(0, 20)) {
@@ -1 +1 @@
1
- {"version":3,"file":"agent-ready.d.ts","sourceRoot":"","sources":["../../src/checks/agent-ready.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAGrD,6FAA6F;AAC7F,eAAO,MAAM,kBAAkB,8LAarB,CAAC;AAEX,4EAA4E;AAC5E,eAAO,MAAM,eAAe,wEAAyE,CAAC;AAEtG,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE;QACT,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,iBAAiB,EAAE,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,eAAe,EAAE,MAAM,EAAE,CAAC;KAC3B,CAAC;IACF,OAAO,EAAE;QACP,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,mBAAmB,EAAE;QACnB,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,OAAO,CAAC;KAC3B,CAAC;IACF,WAAW,EAAE;QACX,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;IACF,SAAS,EAAE;QACT,aAAa,EAAE,OAAO,CAAC;QACvB,WAAW,EAAE,OAAO,CAAC;QACrB,UAAU,EAAE,OAAO,CAAC;QACpB,GAAG,EAAE,OAAO,CAAC;QACb,YAAY,EAAE,OAAO,CAAC;KACvB,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,qBAAqB,CAAC;IAClC,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,SAAS,CAAC,EAAE;QACV,aAAa,CAAC,EAAE,qBAAqB,CAAC;QACtC,WAAW,CAAC,EAAE,qBAAqB,CAAC;QACpC,UAAU,CAAC,EAAE,qBAAqB,CAAC;QACnC,GAAG,CAAC,EAAE,qBAAqB,CAAC;QAC5B,YAAY,CAAC,EAAE,qBAAqB,CAAC;KACtC,CAAC;CACH,CAAC;AAoDF,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,sBAAsB,GAAG,qBAAqB,CAoItF;AAmCD,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,SAAS,GAAG,OAAO,CAAC,qBAAqB,CAAC,CA+C7G"}
1
+ {"version":3,"file":"agent-ready.d.ts","sourceRoot":"","sources":["../../src/checks/agent-ready.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAGrD,6FAA6F;AAC7F,eAAO,MAAM,kBAAkB,8LAarB,CAAC;AAEX,4EAA4E;AAC5E,eAAO,MAAM,eAAe,wEAAyE,CAAC;AAEtG,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE;QACT,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,iBAAiB,EAAE,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,eAAe,EAAE,MAAM,EAAE,CAAC;KAC3B,CAAC;IACF,OAAO,EAAE;QACP,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,mBAAmB,EAAE;QACnB,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,OAAO,CAAC;KAC3B,CAAC;IACF,WAAW,EAAE;QACX,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;IACF,SAAS,EAAE;QACT,aAAa,EAAE,OAAO,CAAC;QACvB,WAAW,EAAE,OAAO,CAAC;QACrB,UAAU,EAAE,OAAO,CAAC;QACpB,GAAG,EAAE,OAAO,CAAC;QACb,YAAY,EAAE,OAAO,CAAC;KACvB,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,qBAAqB,CAAC;IAClC,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,SAAS,CAAC,EAAE;QACV,aAAa,CAAC,EAAE,qBAAqB,CAAC;QACtC,WAAW,CAAC,EAAE,qBAAqB,CAAC;QACpC,UAAU,CAAC,EAAE,qBAAqB,CAAC;QACnC,GAAG,CAAC,EAAE,qBAAqB,CAAC;QAC5B,YAAY,CAAC,EAAE,qBAAqB,CAAC;KACtC,CAAC;CACH,CAAC;AAoDF,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,sBAAsB,GAAG,qBAAqB,CAoJtF;AAmCD,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,SAAS,GAAG,OAAO,CAAC,qBAAqB,CAAC,CA+C7G"}
@@ -96,31 +96,31 @@ export function analyzeAgentReady(input) {
96
96
  };
97
97
  const robotsStatus = input.robotsTxt?.status ?? null;
98
98
  if (!isSuccessfulStatus(robotsStatus) || !robotsBody.trim()) {
99
- issues.push(issue("agent-robots-missing", "warning", "robots.txt is missing or empty; agents cannot discover crawl policy or Content Signals."));
99
+ issues.push(issue("agent-robots-missing", "warning", "robots.txt is missing or empty; agents cannot discover crawl policy or Content Signals.", "Add a robots.txt at the site root (for example public/robots.txt) with at least `User-agent: *` and a Sitemap line."));
100
100
  }
101
101
  else {
102
102
  if (contentSignals.length === 0) {
103
- issues.push(issue("agent-missing-content-signals", "warning", "robots.txt has no Content-Signal directives (ai-train / search / ai-input)."));
103
+ issues.push(issue("agent-missing-content-signals", "warning", "robots.txt has no Content-Signal directives (ai-train / search / ai-input).", "Add a Content-Signal under `User-agent: *` in robots.txt, for example `Content-Signal: search=yes,ai-train=no`."));
104
104
  }
105
105
  if (aiBotUserAgents.length === 0) {
106
- issues.push(issue("agent-missing-ai-bot-rules", "warning", "robots.txt has no explicit AI bot User-agent rules (e.g. GPTBot, ClaudeBot, Google-Extended)."));
106
+ issues.push(issue("agent-missing-ai-bot-rules", "warning", "robots.txt has no explicit AI bot User-agent rules (e.g. GPTBot, ClaudeBot, Google-Extended).", "Add explicit `User-agent` blocks for AI crawlers (GPTBot, ClaudeBot, Google-Extended, …) with Allow/Disallow as intended."));
107
107
  }
108
108
  }
109
109
  if (!llmsOk) {
110
- issues.push(issue("agent-missing-llms-txt", "warning", "llms.txt is missing or not a plain-text overview at /llms.txt."));
110
+ issues.push(issue("agent-missing-llms-txt", "warning", "llms.txt is missing or not a plain-text overview at /llms.txt.", "Add /llms.txt as plain text with an H1 title (`# Site Name`) and links to the most important content."));
111
111
  }
112
112
  else if (!llmsHasTitle) {
113
- issues.push(issue("agent-llms-txt-incomplete", "review", "llms.txt exists but has no H1 title line (# Site Name)."));
113
+ issues.push(issue("agent-llms-txt-incomplete", "review", "llms.txt exists but has no H1 title line (# Site Name).", "Start llms.txt with an H1 line, for example `# Site Name`."));
114
114
  }
115
115
  if (!markdownOk) {
116
- issues.push(issue("agent-missing-markdown-negotiation", "warning", "Homepage does not negotiate Markdown (Accept: text/markdown → Content-Type: text/markdown)."));
116
+ issues.push(issue("agent-missing-markdown-negotiation", "warning", "Homepage does not negotiate Markdown (Accept: text/markdown → Content-Type: text/markdown).", "When the request has `Accept: text/markdown`, return the page body as Markdown with `Content-Type: text/markdown` (middleware or route)."));
117
117
  }
118
118
  if (agentLinkRelations.length === 0) {
119
- issues.push(issue("agent-missing-link-headers", "review", 'Homepage has no agent-useful Link headers (rel="api-catalog", "service-doc", "service-desc", or "describedby").'));
119
+ issues.push(issue("agent-missing-link-headers", "review", 'Homepage has no agent-useful Link headers (rel="api-catalog", "service-doc", "service-desc", or "describedby").', 'Add Link response headers on the homepage, for example `Link: </.well-known/api-catalog>; rel="api-catalog"`.'));
120
120
  }
121
121
  const wellKnownPresent = Object.values(wellKnown).some(Boolean);
122
122
  if (!wellKnownPresent) {
123
- issues.push(issue("agent-missing-protocol-discovery", "review", "No agent protocol discovery endpoints found (MCP Server Card, Agent Skills, API Catalog, ARD, or A2A Agent Card)."));
123
+ issues.push(issue("agent-missing-protocol-discovery", "review", "No agent protocol discovery endpoints found (MCP Server Card, Agent Skills, API Catalog, ARD, or A2A Agent Card).", "Add /.well-known/agent-skills/index.json. Use an empty array if no skills are available (`skills: []`)."));
124
124
  }
125
125
  const findings = {
126
126
  robotsTxt: {
@@ -1 +1 @@
1
- {"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/checks/crawl.ts"],"names":[],"mappings":"AAOA,OAAO,EAAuB,KAAK,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAKjF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,oBAAoB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC9E,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;CAC/B,CAAC,CAyDD;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAiCzF"}
1
+ {"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/checks/crawl.ts"],"names":[],"mappings":"AAOA,OAAO,EAAuB,KAAK,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAMjF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,oBAAoB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC9E,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;CAC/B,CAAC,CAuED;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAiCzF"}
@@ -2,6 +2,7 @@ import { DEFAULT_BURST_DELAY_MS, DEFAULT_CONCURRENCY, DEFAULT_DELAY_MS, DEFAULT_
2
2
  import { resolveActiveGroups } from "../crawler/checkGroups.js";
3
3
  import { buildDefaultOutputPaths } from "../crawler/outputPaths.js";
4
4
  import { runCrawler } from "../crawler/runCrawler.js";
5
+ import { selectSmartCrawlUrls } from "../crawler/smartPageSelection.js";
5
6
  import { collectSitemapPageUrls } from "./sitemap.js";
6
7
  import { isHttpUrl, matchesExcludePattern, normalizeUrl } from "../crawler/urlUtils.js";
7
8
  export async function prepareCrawlSeedUrls(options) {
@@ -51,6 +52,16 @@ export async function prepareCrawlSeedUrls(options) {
51
52
  }
52
53
  }
53
54
  seedUrls = seedUrls.filter((url) => !matchesExcludePattern(url, excludePatterns));
55
+ const unlimitedSeedCount = seedUrls.length;
56
+ const smart = selectSmartCrawlUrls(seedUrls, Math.floor(maxPages), {
57
+ startUrl: normalizedStartUrl,
58
+ });
59
+ seedUrls = smart.urls;
60
+ if (unlimitedSeedCount > seedUrls.length) {
61
+ // Intentionally concise: full URL lists stay in sitemap compare / crawl JSON.
62
+ const listCount = smart.tree.listParents.size;
63
+ console.info(`[crawl] smart max-pages: selected ${seedUrls.length}/${unlimitedSeedCount} seed URLs (limit=${Math.floor(maxPages)}, listParents=${listCount})`);
64
+ }
54
65
  return { normalizedStartUrl, seedUrls, outputPath, activeGroups };
55
66
  }
56
67
  export async function runCrawlCheck(options) {
@@ -1 +1 @@
1
- {"version":3,"file":"dns.d.ts","sourceRoot":"","sources":["../../src/checks/dns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,MAAM,UAAU,GAAG;IACvB,CAAC,EAAE,MAAM,EAAE,CAAC;IACZ,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,EAAE,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClD,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,EAAE,EAAE,MAAM,EAAE,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC;IACpB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AA2CF,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAuC9E;AAED,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAE3E"}
1
+ {"version":3,"file":"dns.d.ts","sourceRoot":"","sources":["../../src/checks/dns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,MAAM,UAAU,GAAG;IACvB,CAAC,EAAE,MAAM,EAAE,CAAC;IACZ,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,EAAE,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClD,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,EAAE,EAAE,MAAM,EAAE,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC;IACpB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AA2CF,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAqD9E;AAED,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAE3E"}
@@ -56,10 +56,10 @@ export async function analyzeDnsRecords(input) {
56
56
  const dmarc = (await resolveDmarc(hostname)) ?? findDmarc(hostname, flatTxt);
57
57
  const skipMailAuthIssues = isPlatformPreviewHostname(hostname);
58
58
  if (!skipMailAuthIssues && !spf) {
59
- issues.push(issue("missing-spf", "warning", "No SPF TXT record found."));
59
+ issues.push(issue("missing-spf", "warning", "No SPF TXT record found.", `Add a TXT record on ${hostname} starting with \`v=spf1\` that lists your mail senders.`));
60
60
  }
61
61
  if (!skipMailAuthIssues && !dmarc) {
62
- issues.push(issue("missing-dmarc", "warning", "No DMARC TXT record found."));
62
+ issues.push(issue("missing-dmarc", "warning", "No DMARC TXT record found.", `Add a TXT record on \`_dmarc.${hostname}\` starting with \`v=DMARC1;\` (for example \`p=none; rua=mailto:dmarc@${hostname}\`).`));
63
63
  }
64
64
  return {
65
65
  hostname,
@@ -3,6 +3,8 @@ export type CheckIssue = {
3
3
  id: string;
4
4
  severity: CheckIssueSeverity;
5
5
  message: string;
6
+ /** Concrete remediation suggestion for reports / agents. */
7
+ fix?: string;
6
8
  };
7
- export declare function issue(id: string, severity: CheckIssueSeverity, message: string): CheckIssue;
9
+ export declare function issue(id: string, severity: CheckIssueSeverity, message: string, fix?: string): CheckIssue;
8
10
  //# sourceMappingURL=issues.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"issues.d.ts","sourceRoot":"","sources":["../../src/checks/issues.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE7E,MAAM,MAAM,UAAU,GAAG;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,GAAG,UAAU,CAE3F"}
1
+ {"version":3,"file":"issues.d.ts","sourceRoot":"","sources":["../../src/checks/issues.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE7E,MAAM,MAAM,UAAU,GAAG;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,wBAAgB,KAAK,CACnB,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,EAAE,MAAM,EACf,GAAG,CAAC,EAAE,MAAM,GACX,UAAU,CAEZ"}
@@ -1,3 +1,3 @@
1
- export function issue(id, severity, message) {
2
- return { id, severity, message };
1
+ export function issue(id, severity, message, fix) {
2
+ return fix === undefined ? { id, severity, message } : { id, severity, message, fix };
3
3
  }
@@ -1 +1 @@
1
- {"version":3,"file":"redirect.d.ts","sourceRoot":"","sources":["../../src/checks/redirect.ts"],"names":[],"mappings":"AAEA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,eAAO,MAAM,qBAAqB,aAAqC,CAAC;AACxE,eAAO,MAAM,+BAA+B,aAAsB,CAAC;AAEnE,MAAM,MAAM,kBAAkB,GAAG;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,kBAAkB,GAAG;IACrD,UAAU,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC;IACtB,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CAClC,CAAC;AAEF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAG1D;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAIzD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,GAAG,MAAM,CAE/G;AAMD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAM7D;AAED,eAAO,MAAM,uBAAuB,EAAE,OAAO,EAAuB,CAAC;AAmBrE,wBAAgB,eAAe,CAAC,KAAK,EAAE,kBAAkB,GAAG,mBAAmB,CAyD9E;AAED,wBAAgB,uBAAuB,CACrC,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,IAAI,CAAC,kBAAkB,EAAE,KAAK,GAAG,QAAQ,GAAG,UAAU,GAAG,gBAAgB,GAAG,cAAc,CAAC,GACjG,UAAU,EAAE,CAyEd;AAED,wBAAgB,iBAAiB,CAAC,eAAe,EAAE,OAAO,EAAE,GAAG,OAAO,CAKrE;AAED,wBAAgB,wBAAwB,CACtC,eAAe,EAAE,OAAO,EAAE,EAC1B,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,GAClD,OAAO,EAAE,CAWX;AAED,wBAAgB,uBAAuB,CAAC,eAAe,EAAE,OAAO,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,CAGzG;AA0DD,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,SAAqB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CA8BrH"}
1
+ {"version":3,"file":"redirect.d.ts","sourceRoot":"","sources":["../../src/checks/redirect.ts"],"names":[],"mappings":"AAEA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,eAAO,MAAM,qBAAqB,aAAqC,CAAC;AACxE,eAAO,MAAM,+BAA+B,aAAsB,CAAC;AAEnE,MAAM,MAAM,kBAAkB,GAAG;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,kBAAkB,GAAG;IACrD,UAAU,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC;IACtB,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CAClC,CAAC;AAEF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAG1D;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAIzD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,GAAG,MAAM,CAE/G;AAMD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAM7D;AAED,eAAO,MAAM,uBAAuB,EAAE,OAAO,EAAuB,CAAC;AAmBrE,wBAAgB,eAAe,CAAC,KAAK,EAAE,kBAAkB,GAAG,mBAAmB,CA0E9E;AAED,wBAAgB,uBAAuB,CACrC,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,IAAI,CAAC,kBAAkB,EAAE,KAAK,GAAG,QAAQ,GAAG,UAAU,GAAG,gBAAgB,GAAG,cAAc,CAAC,GACjG,UAAU,EAAE,CAgFd;AAED,wBAAgB,iBAAiB,CAAC,eAAe,EAAE,OAAO,EAAE,GAAG,OAAO,CAKrE;AAED,wBAAgB,wBAAwB,CACtC,eAAe,EAAE,OAAO,EAAE,EAC1B,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,GAClD,OAAO,EAAE,CAWX;AAED,wBAAgB,uBAAuB,CAAC,eAAe,EAAE,OAAO,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,CAGzG;AA0DD,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,SAAqB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CA8BrH"}
@@ -62,19 +62,19 @@ export function analyzeRedirect(input) {
62
62
  };
63
63
  }
64
64
  if (!input.location) {
65
- issues.push(issue("redirect-missing-location", "error", "Redirect response is missing a Location header."));
65
+ issues.push(issue("redirect-missing-location", "error", "Redirect response is missing a Location header.", "Include a valid Location header on redirect responses pointing at the canonical URL."));
66
66
  }
67
67
  else if (!input.resolvedTarget) {
68
- issues.push(issue("redirect-invalid-location", "error", "Redirect Location header is not a valid URL."));
68
+ issues.push(issue("redirect-invalid-location", "error", "Redirect Location header is not a valid URL.", "Set Location to an absolute http(s) URL for the redirect target."));
69
69
  }
70
70
  if (input.status !== null && !PERMANENT_REDIRECT_STATUS_CODES.has(input.status)) {
71
- issues.push(issue("redirect-temporary-status", "warning", `Redirect uses status ${input.status}; use 301 or 308 for permanent domain redirects.`));
71
+ issues.push(issue("redirect-temporary-status", "warning", `Redirect uses status ${input.status}; use 301 or 308 for permanent domain redirects.`, "Change the redirect status to 301 or 308 for permanent domain redirects."));
72
72
  }
73
73
  if (isHttps && !input.headers["strict-transport-security"]) {
74
- issues.push(issue("redirect-missing-hsts", "warning", "Strict-Transport-Security header is missing on the redirect response."));
74
+ issues.push(issue("redirect-missing-hsts", "warning", "Strict-Transport-Security header is missing on the redirect response.", "Send Strict-Transport-Security on the HTTPS redirect response as well as on final pages."));
75
75
  }
76
76
  if (input.resolvedTarget && input.targetStatus !== null && input.targetStatus >= 400) {
77
- issues.push(issue("redirect-target-unreachable", "error", `Redirect target returned status ${input.targetStatus}: ${input.resolvedTarget}`));
77
+ issues.push(issue("redirect-target-unreachable", "error", `Redirect target returned status ${input.targetStatus}: ${input.resolvedTarget}`, "Fix the redirect target so it returns a successful response, or point Location at a working URL."));
78
78
  }
79
79
  return {
80
80
  ...input,
@@ -87,27 +87,27 @@ export function analyzeWwwAliasRedirect(canonicalUrl, probe) {
87
87
  const issues = [];
88
88
  const canonicalHost = hostnameFromUrl(canonicalUrl);
89
89
  if (probe.status === null) {
90
- issues.push(issue("redirect-www-alias-unreachable", "error", `Alternate host is unreachable and should redirect to ${canonicalHost}: ${probe.url}`));
90
+ issues.push(issue("redirect-www-alias-unreachable", "error", `Alternate host is unreachable and should redirect to ${canonicalHost}: ${probe.url}`, `Configure DNS and hosting so the alternate host responds and permanently redirects to ${canonicalHost}.`));
91
91
  return issues;
92
92
  }
93
93
  if (!REDIRECT_STATUS_CODES.has(probe.status)) {
94
- issues.push(issue("redirect-www-alias-no-redirect", "error", `Alternate host should redirect to ${canonicalHost} but returned status ${probe.status}: ${probe.url}`));
94
+ issues.push(issue("redirect-www-alias-no-redirect", "error", `Alternate host should redirect to ${canonicalHost} but returned status ${probe.status}: ${probe.url}`, `Configure the alternate host to respond with 301/308 Location pointing at ${canonicalHost}.`));
95
95
  return issues;
96
96
  }
97
97
  if (!probe.location) {
98
- issues.push(issue("redirect-www-alias-missing-location", "error", `Alternate host redirect is missing a Location header: ${probe.url}`));
98
+ issues.push(issue("redirect-www-alias-missing-location", "error", `Alternate host redirect is missing a Location header: ${probe.url}`, `Add a Location header on the alternate-host redirect targeting ${canonicalHost}.`));
99
99
  }
100
100
  else if (!probe.resolvedTarget) {
101
- issues.push(issue("redirect-www-alias-invalid-location", "error", `Alternate host redirect Location header is not a valid URL: ${probe.url}`));
101
+ issues.push(issue("redirect-www-alias-invalid-location", "error", `Alternate host redirect Location header is not a valid URL: ${probe.url}`, `Set Location to an absolute http(s) URL on ${canonicalHost}.`));
102
102
  }
103
103
  else if (hostnameFromUrl(probe.resolvedTarget) !== canonicalHost) {
104
- issues.push(issue("redirect-www-alias-wrong-target", "error", `Alternate host redirects to ${probe.resolvedTarget} instead of ${canonicalHost}: ${probe.url}`));
104
+ issues.push(issue("redirect-www-alias-wrong-target", "error", `Alternate host redirects to ${probe.resolvedTarget} instead of ${canonicalHost}: ${probe.url}`, `Point the alternate-host redirect at the canonical hostname (${canonicalHost}).`));
105
105
  }
106
106
  if (probe.status !== null && !PERMANENT_REDIRECT_STATUS_CODES.has(probe.status)) {
107
- issues.push(issue("redirect-www-alias-temporary-status", "warning", `Alternate host redirect uses status ${probe.status}; use 301 or 308 for permanent domain redirects: ${probe.url}`));
107
+ issues.push(issue("redirect-www-alias-temporary-status", "warning", `Alternate host redirect uses status ${probe.status}; use 301 or 308 for permanent domain redirects: ${probe.url}`, "Change the alternate-host redirect status to 301 or 308."));
108
108
  }
109
109
  if (probe.resolvedTarget && probe.targetStatus !== null && probe.targetStatus >= 400) {
110
- issues.push(issue("redirect-www-alias-target-unreachable", "error", `Alternate host redirect target returned status ${probe.targetStatus}: ${probe.resolvedTarget}`));
110
+ issues.push(issue("redirect-www-alias-target-unreachable", "error", `Alternate host redirect target returned status ${probe.targetStatus}: ${probe.resolvedTarget}`, "Fix the canonical target URL so the alternate-host redirect lands on a successful page."));
111
111
  }
112
112
  return issues;
113
113
  }
@@ -1 +1 @@
1
- {"version":3,"file":"robots-txt.d.ts","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAGrD,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAgDF,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,qBAAqB,GAAG,oBAAoB,CA2CnF;AAED,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,SAAS,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAmB3G"}
1
+ {"version":3,"file":"robots-txt.d.ts","sourceRoot":"","sources":["../../src/checks/robots-txt.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAGrD,MAAM,MAAM,qBAAqB,GAAG;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAgDF,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,qBAAqB,GAAG,oBAAoB,CAqFnF;AAED,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,SAAS,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAmB3G"}
@@ -47,15 +47,15 @@ export function analyzeRobotsTxt(input) {
47
47
  const issues = [];
48
48
  const { url, status, body } = input;
49
49
  if (status === null || status === 404) {
50
- issues.push(issue("robots-missing", "warning", "robots.txt is missing."));
50
+ issues.push(issue("robots-missing", "warning", "robots.txt is missing.", "Publish /robots.txt at the site origin with at least `User-agent: *` and a Sitemap directive."));
51
51
  return { url, status, body, sitemapUrls: [], issues };
52
52
  }
53
53
  if (!body) {
54
- issues.push(issue("robots-unparseable", "error", "robots.txt response body is empty."));
54
+ issues.push(issue("robots-unparseable", "error", "robots.txt response body is empty.", "Return a non-empty robots.txt body with valid User-agent / Allow / Disallow / Sitemap lines."));
55
55
  return { url, status, body, sitemapUrls: [], issues };
56
56
  }
57
57
  if (hasMalformedRobotsLines(body)) {
58
- issues.push(issue("robots-unparseable", "error", "robots.txt contains malformed directives."));
58
+ issues.push(issue("robots-unparseable", "error", "robots.txt contains malformed directives.", "Fix robots.txt syntax: one directive per line (`User-agent:`, `Allow:`, `Disallow:`, `Sitemap:`)."));
59
59
  }
60
60
  let sitemapUrls = [];
61
61
  try {
@@ -64,17 +64,17 @@ export function analyzeRobotsTxt(input) {
64
64
  sitemapUrls = parsed.getSitemaps();
65
65
  }
66
66
  catch {
67
- issues.push(issue("robots-unparseable", "error", "robots.txt could not be parsed."));
67
+ issues.push(issue("robots-unparseable", "error", "robots.txt could not be parsed.", "Validate robots.txt content and encoding; ensure it is plain text with standard directives."));
68
68
  return { url, status, body, sitemapUrls: [], issues };
69
69
  }
70
70
  if (sitemapUrls.length === 0) {
71
71
  sitemapUrls = extractSitemapUrls(body);
72
72
  }
73
73
  if (blocksAllUserAgents(body, url)) {
74
- issues.push(issue("robots-blocks-all", "critical", "robots.txt disallows all crawling for User-agent: *."));
74
+ issues.push(issue("robots-blocks-all", "critical", "robots.txt disallows all crawling for User-agent: *.", "Remove or narrow `Disallow: /` under `User-agent: *` so search engines and agents can crawl public pages."));
75
75
  }
76
76
  if (sitemapUrls.length === 0) {
77
- issues.push(issue("robots-missing-sitemap", "warning", "robots.txt has no Sitemap directive."));
77
+ issues.push(issue("robots-missing-sitemap", "warning", "robots.txt has no Sitemap directive.", "Add a Sitemap line pointing at your sitemap, for example `Sitemap: https://example.com/sitemap.xml`."));
78
78
  }
79
79
  return { url, status, body, sitemapUrls, issues };
80
80
  }
@@ -1 +1 @@
1
- {"version":3,"file":"security-headers.d.ts","sourceRoot":"","sources":["../../src/checks/security-headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,MAAM,6BAA6B,GAAG;IAC1C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAWF,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,6BAAkC,GAC1C,UAAU,EAAE,CA0Cd;AAED,MAAM,MAAM,0BAA0B,GAAG;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF,wBAAsB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAgBnG"}
1
+ {"version":3,"file":"security-headers.d.ts","sourceRoot":"","sources":["../../src/checks/security-headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAErD,MAAM,MAAM,6BAA6B,GAAG;IAC1C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAWF,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,6BAAkC,GAC1C,UAAU,EAAE,CA8Ed;AAED,MAAM,MAAM,0BAA0B,GAAG;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF,wBAAsB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAgBnG"}
@@ -17,22 +17,22 @@ export function analyzeSecurityHeaders(headers, options = {}) {
17
17
  const referrerPolicy = headerValue(headers, "referrer-policy");
18
18
  const permissionsPolicy = headerValue(headers, "permissions-policy");
19
19
  if (isHttps && !hsts) {
20
- issues.push(issue("missing-hsts", "error", "Strict-Transport-Security header is missing on HTTPS."));
20
+ issues.push(issue("missing-hsts", "error", "Strict-Transport-Security header is missing on HTTPS.", "Send `Strict-Transport-Security: max-age=31536000; includeSubDomains` on HTTPS responses (hosting or app headers)."));
21
21
  }
22
22
  if (!csp) {
23
- issues.push(issue("missing-csp", "warning", "Content-Security-Policy header is missing."));
23
+ issues.push(issue("missing-csp", "warning", "Content-Security-Policy header is missing.", "Add a Content-Security-Policy header tailored to your app (start with report-only if needed)."));
24
24
  }
25
25
  if (!xFrameOptions && !hasFrameAncestorsCsp(csp)) {
26
- issues.push(issue("missing-clickjacking-protection", "warning", "Neither X-Frame-Options nor CSP frame-ancestors is set."));
26
+ issues.push(issue("missing-clickjacking-protection", "warning", "Neither X-Frame-Options nor CSP frame-ancestors is set.", "Set `X-Frame-Options: DENY` (or SAMEORIGIN) or add `frame-ancestors` to your Content-Security-Policy."));
27
27
  }
28
28
  if (!nosniff || nosniff.toLowerCase() !== "nosniff") {
29
- issues.push(issue("missing-nosniff", "warning", "X-Content-Type-Options: nosniff is missing."));
29
+ issues.push(issue("missing-nosniff", "warning", "X-Content-Type-Options: nosniff is missing.", "Send `X-Content-Type-Options: nosniff` on all responses."));
30
30
  }
31
31
  if (!referrerPolicy) {
32
- issues.push(issue("missing-referrer-policy", "review", "Referrer-Policy header is missing."));
32
+ issues.push(issue("missing-referrer-policy", "review", "Referrer-Policy header is missing.", "Send a Referrer-Policy such as `strict-origin-when-cross-origin`."));
33
33
  }
34
34
  if (!permissionsPolicy) {
35
- issues.push(issue("missing-permissions-policy", "review", "Permissions-Policy header is missing."));
35
+ issues.push(issue("missing-permissions-policy", "review", "Permissions-Policy header is missing.", "Add a Permissions-Policy header that disables unused browser features (camera, mic, geolocation, …)."));
36
36
  }
37
37
  return issues;
38
38
  }
@@ -1 +1 @@
1
- {"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/checks/sitemap.ts"],"names":[],"mappings":"AAGA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAMrD,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B,CAAC;AAQF,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,0BAA0B,GAAG,kBAAkB,CA6G3F;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAE,MAAM,EAAO,GAAG,MAAM,EAAE,CAIpG;AAkBD,wBAAsB,kBAAkB,CACtC,aAAa,EAAE,MAAM,EAAE,EACvB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAchC;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,IAAI,CACpC,sBAAsB,EACtB,WAAW,GAAG,0BAA0B,GAAG,iBAAiB,CAC7D,CAAC;AAEF,wBAAsB,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAYlG;AAED,wBAAsB,sBAAsB,CAC1C,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,iBAAiB,GAAE,MAAM,EAAO,EAChC,gBAAgB,CAAC,EAAE,MAAM,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC,CAmBnB"}
1
+ {"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/checks/sitemap.ts"],"names":[],"mappings":"AAGA,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAMrD,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B,CAAC;AAQF,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,0BAA0B,GAAG,kBAAkB,CAyH3F;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAE,MAAM,EAAO,GAAG,MAAM,EAAE,CAIpG;AAkBD,wBAAsB,kBAAkB,CACtC,aAAa,EAAE,MAAM,EAAE,EACvB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAchC;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,IAAI,CACpC,sBAAsB,EACtB,WAAW,GAAG,0BAA0B,GAAG,iBAAiB,CAC7D,CAAC;AAEF,wBAAsB,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAYlG;AAED,wBAAsB,sBAAsB,CAC1C,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,iBAAiB,GAAE,MAAM,EAAO,EAChC,gBAAgB,CAAC,EAAE,MAAM,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC,CAmBnB"}
@@ -16,12 +16,12 @@ export function analyzeSitemapCompare(input) {
16
16
  const issues = [];
17
17
  for (const source of input.sources) {
18
18
  if (source.status === null || source.status >= 400) {
19
- issues.push(issue("sitemap-unreachable", "error", `Sitemap not reachable: ${source.url} (${source.status ?? "unreachable"}).`));
19
+ issues.push(issue("sitemap-unreachable", "error", `Sitemap not reachable: ${source.url} (${source.status ?? "unreachable"}).`, `Ensure ${source.url} returns HTTP 200 with a valid XML sitemap or sitemap index.`));
20
20
  }
21
21
  }
22
22
  const successfulSources = input.sources.filter((source) => source.status !== null && source.status >= 200 && source.status < 300);
23
23
  if (successfulSources.length === 0) {
24
- issues.push(issue("sitemap-missing", "error", "No sitemap found."));
24
+ issues.push(issue("sitemap-missing", "error", "No sitemap found.", "Publish /sitemap.xml (or reference it from robots.txt) and make sure it returns HTTP 200."));
25
25
  return {
26
26
  sources: input.sources,
27
27
  sitemapPageUrls: [],
@@ -35,7 +35,7 @@ export function analyzeSitemapCompare(input) {
35
35
  const sitemapPageUrls = filterComparableUrls(successfulSources.flatMap((source) => source.pageUrls), excludePatterns);
36
36
  const crawledUrls = filterComparableUrls(input.crawledUrls, excludePatterns);
37
37
  if (sitemapPageUrls.length === 0) {
38
- issues.push(issue("sitemap-empty", "error", "Sitemap was reachable but contained no page URLs (check sitemap index nesting and host aliases such as apex vs www)."));
38
+ issues.push(issue("sitemap-empty", "error", "Sitemap was reachable but contained no page URLs (check sitemap index nesting and host aliases such as apex vs www).", "Populate the sitemap with page `<loc>` entries, or fix nested sitemap indexes and apex/www host mismatches."));
39
39
  return {
40
40
  sources: input.sources,
41
41
  sitemapPageUrls,
@@ -51,17 +51,17 @@ export function analyzeSitemapCompare(input) {
51
51
  const inSitemapNotCrawled = sitemapPageUrls.filter((url) => !crawledSet.has(url));
52
52
  const crawledNotInSitemap = crawledUrls.filter((url) => !sitemapSet.has(url));
53
53
  if (inSitemapNotCrawled.length > 0) {
54
- issues.push(issue("sitemap-url-not-crawled", "critical", `${inSitemapNotCrawled.length} sitemap URL(s) were not crawled. ${SITEMAP_URL_NOT_CRAWLED_TIP}`));
54
+ issues.push(issue("sitemap-url-not-crawled", "critical", `${inSitemapNotCrawled.length} sitemap URL(s) were not crawled. ${SITEMAP_URL_NOT_CRAWLED_TIP}`, "Remove stale sitemap entries, fix robots blocks, or raise --max-pages so listed URLs can be crawled."));
55
55
  }
56
56
  if (crawledNotInSitemap.length > 0) {
57
- issues.push(issue("sitemap-missing-crawled-url", "critical", `${crawledNotInSitemap.length} crawled URL(s) are missing from the sitemap.`));
57
+ issues.push(issue("sitemap-missing-crawled-url", "critical", `${crawledNotInSitemap.length} crawled URL(s) are missing from the sitemap.`, "Add the crawled URLs to the sitemap (or regenerate the sitemap from the live route set)."));
58
58
  }
59
59
  const total = new Set([...sitemapPageUrls, ...crawledUrls]).size;
60
60
  const mismatchCount = new Set([...inSitemapNotCrawled, ...crawledNotInSitemap]).size;
61
61
  const mismatchPercent = total === 0 ? 0 : (mismatchCount / total) * 100;
62
62
  if (mismatchPercent > threshold) {
63
63
  issues.push(issue("sitemap-mismatch-threshold", "critical", `Sitemap mismatch is ${mismatchPercent.toFixed(1)}% (threshold ${threshold}%). ` +
64
- "Sitemap URLs must match crawled URLs exactly, including hostname."));
64
+ "Sitemap URLs must match crawled URLs exactly, including hostname.", "Align sitemap hostnames and paths with the live site (apex vs www) and regenerate the sitemap."));
65
65
  }
66
66
  return {
67
67
  sources: input.sources,
package/dist/cli.js CHANGED
@@ -26,7 +26,7 @@ async function main() {
26
26
  .option("--errors-only", "Include only critical/error findings in the Markdown report", false)
27
27
  .option("--parallel", "Run independent checks in parallel", false)
28
28
  .option("--checks <ids>", "Comma-separated checks to run (crawl,analyze,lighthouse,security-headers,dns,robots-txt,sitemap,agent-ready)")
29
- .option("--max-pages <n>", "Maximum number of pages", String(DEFAULT_MAX_PAGES))
29
+ .option("--max-pages <n>", "Maximum pages to crawl (default 100; prefers structural pages, samples leaf URLs under large list branches)", String(DEFAULT_MAX_PAGES))
30
30
  .option("--delay-ms <n>", "Delay between requests in ms", String(DEFAULT_DELAY_MS))
31
31
  .option("--timeout-ms <n>", "Request timeout in ms", String(DEFAULT_TIMEOUT_MS))
32
32
  .option("--concurrency <n>", "Maximum concurrent requests", String(DEFAULT_CONCURRENCY))
@@ -1,4 +1,5 @@
1
- export declare const DEFAULT_MAX_PAGES = 1000;
1
+ /** Default crawl budget; smart selection prefers root/list pages within this cap. */
2
+ export declare const DEFAULT_MAX_PAGES = 100;
2
3
  export declare const DEFAULT_DELAY_MS = 500;
3
4
  export declare const DEFAULT_TIMEOUT_MS = 10000;
4
5
  export declare const DEFAULT_CONCURRENCY = 1;
@@ -1 +1 @@
1
- {"version":3,"file":"defaults.d.ts","sourceRoot":"","sources":["../../src/crawler/defaults.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,iBAAiB,OAAO,CAAC;AACtC,eAAO,MAAM,gBAAgB,MAAM,CAAC;AACpC,eAAO,MAAM,kBAAkB,QAAQ,CAAC;AACxC,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,sBAAsB,IAAI,CAAC;AACxC,eAAO,MAAM,kBAAkB,kBAAkB,CAAC;AAClD,eAAO,MAAM,0BAA0B,iBAAiB,CAAC;AACzD,eAAO,MAAM,oBAAoB,KAAK,CAAC;AACvC,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAC9C,eAAO,MAAM,kBAAkB,2CAA2C,CAAC"}
1
+ {"version":3,"file":"defaults.d.ts","sourceRoot":"","sources":["../../src/crawler/defaults.ts"],"names":[],"mappings":"AAAA,qFAAqF;AACrF,eAAO,MAAM,iBAAiB,MAAM,CAAC;AACrC,eAAO,MAAM,gBAAgB,MAAM,CAAC;AACpC,eAAO,MAAM,kBAAkB,QAAQ,CAAC;AACxC,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,sBAAsB,IAAI,CAAC;AACxC,eAAO,MAAM,kBAAkB,kBAAkB,CAAC;AAClD,eAAO,MAAM,0BAA0B,iBAAiB,CAAC;AACzD,eAAO,MAAM,oBAAoB,KAAK,CAAC;AACvC,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAC9C,eAAO,MAAM,kBAAkB,2CAA2C,CAAC"}
@@ -1,4 +1,5 @@
1
- export const DEFAULT_MAX_PAGES = 1000;
1
+ /** Default crawl budget; smart selection prefers root/list pages within this cap. */
2
+ export const DEFAULT_MAX_PAGES = 100;
2
3
  export const DEFAULT_DELAY_MS = 500;
3
4
  export const DEFAULT_TIMEOUT_MS = 10000;
4
5
  export const DEFAULT_CONCURRENCY = 1;
@@ -1 +1 @@
1
- {"version":3,"file":"runCrawler.d.ts","sourceRoot":"","sources":["../../src/crawler/runCrawler.ts"],"names":[],"mappings":"AAUA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAOjE,OAAO,KAAK,EAAiB,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAKrE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAAC,CAqK7H"}
1
+ {"version":3,"file":"runCrawler.d.ts","sourceRoot":"","sources":["../../src/crawler/runCrawler.ts"],"names":[],"mappings":"AAUA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAOjE,OAAO,KAAK,EAAiB,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAMrE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAAC,CA4K7H"}
@@ -8,6 +8,7 @@ import { extractSameOriginScripts } from "./jsAssets.js";
8
8
  import { loadRobotsPolicy } from "./robots.js";
9
9
  import { writeReportToFile } from "./reporter.js";
10
10
  import { scanText } from "./secrets.js";
11
+ import { createDiscoveryGate } from "./smartPageSelection.js";
11
12
  import { matchesExcludePattern, normalizeUrl, sleep } from "./urlUtils.js";
12
13
  const MAX_JS_ASSET_BYTES = 5_000_000; // 5 MB — skip larger bundles to bound memory
13
14
  export async function runCrawler(options) {
@@ -84,10 +85,14 @@ export async function runCrawler(options) {
84
85
  const requestQueue = await RequestQueue.open();
85
86
  const records = [];
86
87
  const seen = new Set();
88
+ const discovery = createDiscoveryGate(seedUrls, maxPages);
89
+ // Seeds are already smart-ordered (home/root/list before details); FIFO preserves that.
87
90
  for (const seed of seedUrls) {
88
91
  if (!matchesExcludePattern(seed, excludePatterns)) {
89
- await requestQueue.addRequest({ url: seed, uniqueKey: normalizeUrl(seed) });
90
- seen.add(normalizeUrl(seed));
92
+ const normalized = normalizeUrl(seed);
93
+ await requestQueue.addRequest({ url: seed, uniqueKey: normalized });
94
+ seen.add(normalized);
95
+ discovery.remember(normalized);
91
96
  }
92
97
  }
93
98
  let processedSinceBurst = 0;
@@ -151,8 +156,12 @@ export async function runCrawler(options) {
151
156
  continue;
152
157
  if (robotsPolicy && !robotsPolicy.isAllowed(normalized))
153
158
  continue;
159
+ if (!discovery.allowEnqueue(normalized))
160
+ continue;
154
161
  seen.add(normalized);
155
- await requestQueue.addRequest({ url: normalized, uniqueKey: normalized });
162
+ discovery.remember(normalized);
163
+ const forefront = discovery.isStructural(normalized);
164
+ await requestQueue.addRequest({ url: normalized, uniqueKey: normalized }, { forefront });
156
165
  }
157
166
  }
158
167
  },
@@ -0,0 +1,77 @@
1
+ /**
2
+ * Smart crawl URL selection based on URL-tree shape (not slug names).
3
+ *
4
+ * A "list" parent is a path with many direct children that are mostly leaves
5
+ * (no further subpages) — typical for archives like news/events/blog, whatever
6
+ * they are named. Within --max-pages we keep structural pages and sample
7
+ * leaves under each list parent.
8
+ */
9
+ export type UrlKind = "home" | "root" | "list" | "detail" | "other";
10
+ export type ClassifiedUrl = {
11
+ url: string;
12
+ kind: UrlKind;
13
+ /** Pathname of the list parent when kind is list/detail, e.g. "/stories". */
14
+ section: string | null;
15
+ /** Sort priority (higher = sooner). */
16
+ priority: number;
17
+ };
18
+ export type UrlTreeAnalysis = {
19
+ /** Pathnames that look like list indexes (many leaf children). */
20
+ listParents: ReadonlySet<string>;
21
+ /** Origin used to rebuild URLs from pathnames when needed. */
22
+ origin: string;
23
+ };
24
+ /** Minimum direct leaf children before a path is treated as a list parent. */
25
+ export declare const DEFAULT_LIST_MIN_CHILDREN = 8;
26
+ /** Share of direct children that must be leaves to count as a list. */
27
+ export declare const DEFAULT_LIST_LEAF_RATIO = 0.7;
28
+ export declare function pathSegments(urlOrPath: string): string[];
29
+ export declare function pathnameOf(urlString: string): string;
30
+ /** Parent pathname: `/a/b` → `/a`, `/a` → `/`, `/` → null. */
31
+ export declare function parentPathname(pathname: string): string | null;
32
+ /**
33
+ * Build parent → direct children maps from a URL set and detect list parents:
34
+ * paths with many children that are mostly leaves (no further children in the set).
35
+ */
36
+ export declare function analyzeUrlTree(urls: string[], options?: {
37
+ startUrl?: string;
38
+ listMinChildren?: number;
39
+ listLeafRatio?: number;
40
+ }): UrlTreeAnalysis;
41
+ /** Closest list-parent pathname for this path, or null. */
42
+ export declare function listParentForPathname(pathname: string, listParents: ReadonlySet<string>): string | null;
43
+ export declare function classifyCrawlUrl(urlString: string, tree: UrlTreeAnalysis): ClassifiedUrl;
44
+ export declare function crawlRequestPriority(urlString: string, tree: UrlTreeAnalysis): number;
45
+ /**
46
+ * Soft per-list-parent cap for detail URLs discovered during a crawl.
47
+ * Structural pages (home/root/list) are never capped this way.
48
+ */
49
+ export declare function detailCapPerSection(maxPages: number, listParentCount?: number): number;
50
+ /**
51
+ * Pick up to `maxPages` URLs using URL-tree shape: keep home/root/list pages,
52
+ * then round-robin a sample of leaf details under each detected list parent.
53
+ */
54
+ export declare function selectSmartCrawlUrls(urls: string[], maxPages: number, options?: {
55
+ startUrl?: string;
56
+ listMinChildren?: number;
57
+ listLeafRatio?: number;
58
+ }): {
59
+ urls: string[];
60
+ tree: UrlTreeAnalysis;
61
+ };
62
+ /** Sort URLs by crawl priority (desc), stable for equal priority. */
63
+ export declare function sortByCrawlPriority(urls: string[], tree: UrlTreeAnalysis): string[];
64
+ /**
65
+ * Tracks list parents during a live crawl: uses seed analysis up front, and
66
+ * promotes parents to "list" when enough leaf-like children are observed.
67
+ */
68
+ export declare function createDiscoveryGate(seedUrls: string[], maxPages: number, options?: {
69
+ listMinChildren?: number;
70
+ listLeafRatio?: number;
71
+ }): {
72
+ tree: UrlTreeAnalysis;
73
+ allowEnqueue: (url: string) => boolean;
74
+ remember: (url: string) => void;
75
+ isStructural: (url: string) => boolean;
76
+ };
77
+ //# sourceMappingURL=smartPageSelection.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"smartPageSelection.d.ts","sourceRoot":"","sources":["../../src/crawler/smartPageSelection.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;AAEpE,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,OAAO,CAAC;IACd,6EAA6E;IAC7E,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,kEAAkE;IAClE,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACjC,8DAA8D;IAC9D,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,8EAA8E;AAC9E,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C,uEAAuE;AACvE,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAU3C,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAOxD;AAED,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAQpD;AAED,8DAA8D;AAC9D,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAK9D;AA2BD;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,CAAC,EAAE;IACR,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACA,eAAe,CA2CjB;AAED,2DAA2D;AAC3D,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,GAC/B,MAAM,GAAG,IAAI,CAOf;AAED,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,GAAG,aAAa,CAsBxF;AAED,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,GAAG,MAAM,CAErF;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,eAAe,SAAI,GAAG,MAAM,CAIjF;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,EAAE,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IACR,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACA;IAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,eAAe,CAAA;CAAE,CAmE3C;AAED,qEAAqE;AACrE,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,eAAe,GAAG,MAAM,EAAE,CAKnF;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,EAAE,EAClB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7D;IACD,IAAI,EAAE,eAAe,CAAC;IACtB,YAAY,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACvC,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,YAAY,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;CACxC,CAqEA"}
@@ -0,0 +1,308 @@
1
+ /**
2
+ * Smart crawl URL selection based on URL-tree shape (not slug names).
3
+ *
4
+ * A "list" parent is a path with many direct children that are mostly leaves
5
+ * (no further subpages) — typical for archives like news/events/blog, whatever
6
+ * they are named. Within --max-pages we keep structural pages and sample
7
+ * leaves under each list parent.
8
+ */
9
+ /** Minimum direct leaf children before a path is treated as a list parent. */
10
+ export const DEFAULT_LIST_MIN_CHILDREN = 8;
11
+ /** Share of direct children that must be leaves to count as a list. */
12
+ export const DEFAULT_LIST_LEAF_RATIO = 0.7;
13
+ const PRIORITY = {
14
+ home: 100,
15
+ root: 80,
16
+ list: 60,
17
+ detail: 40,
18
+ other: 20,
19
+ };
20
+ export function pathSegments(urlOrPath) {
21
+ try {
22
+ const pathname = urlOrPath.includes("://") ? new URL(urlOrPath).pathname : urlOrPath;
23
+ return pathname.split("/").filter(Boolean);
24
+ }
25
+ catch {
26
+ return [];
27
+ }
28
+ }
29
+ export function pathnameOf(urlString) {
30
+ try {
31
+ const pathname = new URL(urlString).pathname;
32
+ if (!pathname || pathname === "")
33
+ return "/";
34
+ return pathname.length > 1 && pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
35
+ }
36
+ catch {
37
+ return "/";
38
+ }
39
+ }
40
+ /** Parent pathname: `/a/b` → `/a`, `/a` → `/`, `/` → null. */
41
+ export function parentPathname(pathname) {
42
+ if (pathname === "/" || pathname === "")
43
+ return null;
44
+ const segments = pathname.split("/").filter(Boolean);
45
+ if (segments.length <= 1)
46
+ return "/";
47
+ return `/${segments.slice(0, -1).join("/")}`;
48
+ }
49
+ function originOf(urls, startUrl) {
50
+ const candidate = startUrl ?? urls[0];
51
+ if (!candidate)
52
+ return "https://example.com";
53
+ try {
54
+ return new URL(candidate).origin;
55
+ }
56
+ catch {
57
+ return "https://example.com";
58
+ }
59
+ }
60
+ function uniqueUrls(urls, startUrl) {
61
+ const seen = new Set();
62
+ const unique = [];
63
+ if (startUrl) {
64
+ unique.push(startUrl);
65
+ seen.add(startUrl);
66
+ }
67
+ for (const url of urls) {
68
+ if (seen.has(url))
69
+ continue;
70
+ seen.add(url);
71
+ unique.push(url);
72
+ }
73
+ return unique;
74
+ }
75
+ /**
76
+ * Build parent → direct children maps from a URL set and detect list parents:
77
+ * paths with many children that are mostly leaves (no further children in the set).
78
+ */
79
+ export function analyzeUrlTree(urls, options) {
80
+ const minChildren = options?.listMinChildren ?? DEFAULT_LIST_MIN_CHILDREN;
81
+ const leafRatio = options?.listLeafRatio ?? DEFAULT_LIST_LEAF_RATIO;
82
+ const unique = uniqueUrls(urls, options?.startUrl);
83
+ const origin = originOf(unique, options?.startUrl);
84
+ const childrenOf = new Map();
85
+ const pathnames = new Set();
86
+ for (const url of unique) {
87
+ const pathname = pathnameOf(url);
88
+ pathnames.add(pathname);
89
+ let child = pathname;
90
+ let parent = parentPathname(child);
91
+ while (parent !== null) {
92
+ let bucket = childrenOf.get(parent);
93
+ if (!bucket) {
94
+ bucket = new Set();
95
+ childrenOf.set(parent, bucket);
96
+ }
97
+ bucket.add(child);
98
+ child = parent;
99
+ parent = parentPathname(child);
100
+ }
101
+ }
102
+ const isLeaf = (pathname) => {
103
+ const kids = childrenOf.get(pathname);
104
+ return !kids || kids.size === 0;
105
+ };
106
+ const listParents = new Set();
107
+ for (const [parent, children] of childrenOf) {
108
+ if (parent === "/")
109
+ continue; // home fan-out is structural, not an archive list
110
+ const childList = [...children];
111
+ if (childList.length < minChildren)
112
+ continue;
113
+ const leafCount = childList.filter((child) => isLeaf(child)).length;
114
+ if (leafCount >= minChildren && leafCount / childList.length >= leafRatio) {
115
+ listParents.add(parent);
116
+ }
117
+ }
118
+ return { listParents, origin };
119
+ }
120
+ /** Closest list-parent pathname for this path, or null. */
121
+ export function listParentForPathname(pathname, listParents) {
122
+ let current = parentPathname(pathname);
123
+ while (current !== null) {
124
+ if (listParents.has(current))
125
+ return current;
126
+ current = parentPathname(current);
127
+ }
128
+ return null;
129
+ }
130
+ export function classifyCrawlUrl(urlString, tree) {
131
+ const pathname = pathnameOf(urlString);
132
+ if (pathname === "/") {
133
+ return { url: urlString, kind: "home", section: null, priority: PRIORITY.home };
134
+ }
135
+ if (tree.listParents.has(pathname)) {
136
+ return { url: urlString, kind: "list", section: pathname, priority: PRIORITY.list };
137
+ }
138
+ const listParent = listParentForPathname(pathname, tree.listParents);
139
+ if (listParent) {
140
+ return { url: urlString, kind: "detail", section: listParent, priority: PRIORITY.detail };
141
+ }
142
+ const depth = pathSegments(pathname).length;
143
+ if (depth <= 2) {
144
+ return { url: urlString, kind: "root", section: null, priority: PRIORITY.root - (depth - 1) * 5 };
145
+ }
146
+ return { url: urlString, kind: "other", section: null, priority: PRIORITY.other };
147
+ }
148
+ export function crawlRequestPriority(urlString, tree) {
149
+ return classifyCrawlUrl(urlString, tree).priority;
150
+ }
151
+ /**
152
+ * Soft per-list-parent cap for detail URLs discovered during a crawl.
153
+ * Structural pages (home/root/list) are never capped this way.
154
+ */
155
+ export function detailCapPerSection(maxPages, listParentCount = 8) {
156
+ const detailBudget = Math.max(8, Math.floor(maxPages * 0.5));
157
+ const sections = Math.max(1, listParentCount);
158
+ return Math.max(2, Math.floor(detailBudget / Math.min(sections, 8)));
159
+ }
160
+ /**
161
+ * Pick up to `maxPages` URLs using URL-tree shape: keep home/root/list pages,
162
+ * then round-robin a sample of leaf details under each detected list parent.
163
+ */
164
+ export function selectSmartCrawlUrls(urls, maxPages, options) {
165
+ if (!Number.isFinite(maxPages) || maxPages <= 0) {
166
+ return { urls: [], tree: analyzeUrlTree([], options) };
167
+ }
168
+ const limit = Math.floor(maxPages);
169
+ const unique = uniqueUrls(urls, options?.startUrl);
170
+ const tree = analyzeUrlTree(unique, options);
171
+ if (unique.length <= limit) {
172
+ return { urls: sortByCrawlPriority(unique, tree), tree };
173
+ }
174
+ const classified = unique.map((url) => classifyCrawlUrl(url, tree));
175
+ const home = [];
176
+ const roots = [];
177
+ const lists = [];
178
+ const others = [];
179
+ const detailsBySection = new Map();
180
+ for (const item of classified) {
181
+ if (item.kind === "home")
182
+ home.push(item);
183
+ else if (item.kind === "root")
184
+ roots.push(item);
185
+ else if (item.kind === "list")
186
+ lists.push(item);
187
+ else if (item.kind === "detail") {
188
+ const key = item.section ?? "_";
189
+ const bucket = detailsBySection.get(key) ?? [];
190
+ bucket.push(item);
191
+ detailsBySection.set(key, bucket);
192
+ }
193
+ else
194
+ others.push(item);
195
+ }
196
+ const selected = [];
197
+ const selectedSet = new Set();
198
+ const take = (items) => {
199
+ for (const item of items) {
200
+ if (selected.length >= limit)
201
+ return;
202
+ if (selectedSet.has(item.url))
203
+ continue;
204
+ selected.push(item.url);
205
+ selectedSet.add(item.url);
206
+ }
207
+ };
208
+ take(home);
209
+ take(roots);
210
+ take(lists);
211
+ const sectionQueues = [...detailsBySection.values()].map((bucket) => [...bucket]);
212
+ let progress = true;
213
+ while (selected.length < limit && progress) {
214
+ progress = false;
215
+ for (const queue of sectionQueues) {
216
+ if (selected.length >= limit)
217
+ break;
218
+ while (queue.length > 0) {
219
+ const next = queue.shift();
220
+ if (selectedSet.has(next.url))
221
+ continue;
222
+ selected.push(next.url);
223
+ selectedSet.add(next.url);
224
+ progress = true;
225
+ break;
226
+ }
227
+ }
228
+ }
229
+ take(others);
230
+ return { urls: selected, tree };
231
+ }
232
+ /** Sort URLs by crawl priority (desc), stable for equal priority. */
233
+ export function sortByCrawlPriority(urls, tree) {
234
+ return urls
235
+ .map((url, index) => ({ url, index, priority: crawlRequestPriority(url, tree) }))
236
+ .sort((a, b) => b.priority - a.priority || a.index - b.index)
237
+ .map((row) => row.url);
238
+ }
239
+ /**
240
+ * Tracks list parents during a live crawl: uses seed analysis up front, and
241
+ * promotes parents to "list" when enough leaf-like children are observed.
242
+ */
243
+ export function createDiscoveryGate(seedUrls, maxPages, options) {
244
+ const minChildren = options?.listMinChildren ?? DEFAULT_LIST_MIN_CHILDREN;
245
+ const leafRatio = options?.listLeafRatio ?? DEFAULT_LIST_LEAF_RATIO;
246
+ const tree = analyzeUrlTree(seedUrls, options);
247
+ const listParents = new Set(tree.listParents);
248
+ const childrenOf = new Map();
249
+ const known = new Set();
250
+ const detailCounts = new Map();
251
+ const register = (url) => {
252
+ const pathname = pathnameOf(url);
253
+ if (known.has(pathname))
254
+ return;
255
+ known.add(pathname);
256
+ let child = pathname;
257
+ let parent = parentPathname(child);
258
+ while (parent !== null) {
259
+ let bucket = childrenOf.get(parent);
260
+ if (!bucket) {
261
+ bucket = new Set();
262
+ childrenOf.set(parent, bucket);
263
+ }
264
+ bucket.add(child);
265
+ if (parent !== "/") {
266
+ const childList = [...bucket];
267
+ if (childList.length >= minChildren) {
268
+ const leafCount = childList.filter((c) => {
269
+ const grand = childrenOf.get(c);
270
+ return !grand || grand.size === 0;
271
+ }).length;
272
+ if (leafCount >= minChildren && leafCount / childList.length >= leafRatio) {
273
+ listParents.add(parent);
274
+ }
275
+ }
276
+ }
277
+ child = parent;
278
+ parent = parentPathname(child);
279
+ }
280
+ };
281
+ for (const seed of seedUrls)
282
+ register(seed);
283
+ const liveTree = () => ({ listParents, origin: tree.origin });
284
+ const capFor = () => detailCapPerSection(maxPages, Math.max(1, listParents.size));
285
+ return {
286
+ get tree() {
287
+ return liveTree();
288
+ },
289
+ allowEnqueue(url) {
290
+ const pathname = pathnameOf(url);
291
+ const parent = listParentForPathname(pathname, listParents);
292
+ if (!parent)
293
+ return true;
294
+ return (detailCounts.get(parent) ?? 0) < capFor();
295
+ },
296
+ remember(url) {
297
+ register(url);
298
+ const classified = classifyCrawlUrl(url, liveTree());
299
+ if (classified.kind !== "detail" || !classified.section)
300
+ return;
301
+ detailCounts.set(classified.section, (detailCounts.get(classified.section) ?? 0) + 1);
302
+ },
303
+ isStructural(url) {
304
+ const kind = classifyCrawlUrl(url, liveTree()).kind;
305
+ return kind === "home" || kind === "root" || kind === "list";
306
+ },
307
+ };
308
+ }
@@ -25,8 +25,8 @@ robots.txt existence and sitemaps remain in the existing `robots-txt` and `sitem
25
25
  - Network runner: `runAgentReadyCheck()` — parallel fetches of robots, llms.txt, homepage
26
26
  (HTML + markdown Accept), and well-known endpoints
27
27
  - Wave 1 (parallel with crawl / DNS / headers / robots)
28
- - Report: issues land in `findings[]` (`category: "infra"`) in `*-report.json`; the
29
- Markdown report lists them under **Findings**
28
+ - Report: issues land in `findings[]` (`category: "infra"`) in `*-report.json` with a
29
+ `fix` suggestion; the Markdown report lists them under **Findings** with a Fix line
30
30
 
31
31
  ## CLI
32
32
 
package/docs/analyze.md CHANGED
@@ -69,7 +69,8 @@ Without `--output`, paths are derived from the input: `-raw.json` becomes
69
69
  "category": "infra",
70
70
  "scope": "origin",
71
71
  "title": "Missing Hsts",
72
- "message": "Strict-Transport-Security header is missing on HTTPS."
72
+ "message": "Strict-Transport-Security header is missing on HTTPS.",
73
+ "fix": "Send `Strict-Transport-Security: max-age=31536000; includeSubDomains` on HTTPS responses (hosting or app headers)."
73
74
  }
74
75
  ]
75
76
  }
@@ -84,7 +85,8 @@ includes metadata so consumers can filter or regroup.
84
85
  | `category` | `infra` \| `seo` \| `a11y` \| `best-practices` \| `security` \| `performance` |
85
86
  | `scope` | `site-wide` \| `template` \| `page` \| `origin` |
86
87
  | `title` | Short human title |
87
- | `message` | What to do / what was found |
88
+ | `message` | What was found |
89
+ | `fix` | Concrete remediation suggestion when available |
88
90
  | `template` | Present when `scope` is `template` |
89
91
  | `urls` / `count` | Affected pages when relevant |
90
92
 
@@ -103,7 +105,7 @@ Generated from the JSON. Structure:
103
105
  1. Header — generated time, Flightcheck version, flags, source path
104
106
  2. **At a glance** — page counts and finding totals
105
107
  3. **Stack / DNS / redirect** — factual context when available
106
- 4. **Findings** — grouped by severity for reading order, with plain-language titles
108
+ 4. **Findings** — grouped by severity for reading order, with plain-language titles and a **Fix** line when available
107
109
 
108
110
  ## Notes
109
111
 
package/docs/crawler.md CHANGED
@@ -34,7 +34,7 @@ Key flags:
34
34
  - `--errors-only` include only critical/error findings in the Markdown report
35
35
  - `--checks <ids>` comma-separated checks; default: all implemented checks (`crawl`, `security-headers`, `dns`, `robots-txt`, `sitemap`, `analyze`)
36
36
  - `--parallel` run independent checks in the same wave concurrently
37
- - `--max-pages <n>`
37
+ - `--max-pages <n>` (default **100**) — hard crawl budget with smart URL selection (see below)
38
38
  - `--delay-ms <n>`
39
39
  - `--timeout-ms <n>`
40
40
  - `--concurrency <n>`
@@ -149,7 +149,20 @@ Additionally:
149
149
 
150
150
  - Only internal links are crawled. Links are discovered via `extractInternalLinks` and added to the queue; Crawlee's `enqueueLinks` is intentionally not used.
151
151
  - Sitemap URLs are added to the crawl queue by default so pages from `sitemap.xml` are visited even when they are not reachable via internal links.
152
- - `--max-pages` is the hard stop (`maxRequestsPerCrawl`). The crawl stops once this count is processed, even if unvisited links remain. Increase `--max-pages` to crawl the entire site.
152
+ - `--max-pages` (default **100**) is the hard stop (`maxRequestsPerCrawl`). The crawl stops once this count is processed, even if unvisited links remain. Raise `--max-pages` (for example `1000` or higher) when you need broader coverage of a large site.
153
+
154
+ ### Smart page selection
155
+
156
+ Large sitemaps often list thousands of archive detail URLs. Selection is based on **URL-tree shape**, not slug names (so `/stories`, `/nieuws`, or `/whatever` all work the same):
157
+
158
+ 1. Build a path tree from known URLs (sitemap seeds and later discoveries)
159
+ 2. Treat a path as a **list parent** when it has many direct children that are mostly **leaves** (no further subpages) — typical archive indexes
160
+ 3. Prefer **home**, other **structural** shallow pages, and those **list** indexes
161
+ 4. Under each list parent, take only a **round-robin sample** of leaf URLs until `--max-pages` is filled
162
+
163
+ Seed order follows that ranking (FIFO). Structural links discovered from HTML are pushed to the front of the queue (`forefront`); leaf URLs under list parents are capped so one archive cannot flood the budget.
164
+
165
+ Selection lives in `src/crawler/smartPageSelection.ts`.
153
166
  - Query parameters and hashes do not count toward URL identity.
154
167
  - Non-HTML responses are reported with `note`.
155
168
  - Links to files with a known non-HTML extension (such as `.pdf`, `.zip`, images, video, fonts) are not added to the queue, so Crawlee does not generate unnecessary retries and errors.
package/docs/index.md CHANGED
@@ -15,6 +15,7 @@
15
15
 
16
16
  - Package releases: [CHANGELOG.md](../CHANGELOG.md)
17
17
  - Branch notes: [changelog/](./changelog/)
18
+ - [2026-10-28 — smart-max-pages](./changelog/2026-10-28--smart-max-pages.md)
18
19
  - [2026-10-28 — report-version-flags](./changelog/2026-10-28--report-version-flags.md)
19
20
  - [2026-10-27 — npm-public-package](./changelog/2026-10-27--npm-public-package.md)
20
21
  - [2026-10-27 — structured-report-json](./changelog/2026-10-27--structured-report-json.md)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@eightmedia/flightcheck",
3
- "version": "1.0.1",
3
+ "version": "1.1.0",
4
4
  "description": "Pre-launch website audit: crawl SEO/a11y/security checks and emit structured JSON + Markdown reports.",
5
5
  "license": "GPL-3.0",
6
6
  "type": "module",