@forkpoint/agent-lighthouse-core 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +149 -9
- package/dist/index.d.ts +149 -9
- package/dist/index.js +626 -111
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +621 -111
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -48,6 +48,7 @@ __export(index_exports, {
|
|
|
48
48
|
DEFAULT_SCAN_LIMIT: () => DEFAULT_SCAN_LIMIT,
|
|
49
49
|
DeprecationNoticeSchema: () => DeprecationNoticeSchema,
|
|
50
50
|
EvidenceGradeSchema: () => EvidenceGradeSchema,
|
|
51
|
+
EvidenceKeySchema: () => EvidenceKeySchema,
|
|
51
52
|
FixEffortSchema: () => FixEffortSchema,
|
|
52
53
|
MAX_CONCURRENT_REQUESTS: () => MAX_CONCURRENT_REQUESTS,
|
|
53
54
|
MAX_PAGES_PER_SCAN: () => MAX_PAGES_PER_SCAN,
|
|
@@ -63,9 +64,12 @@ __export(index_exports, {
|
|
|
63
64
|
SCORE_TIER_LABELS: () => SCORE_TIER_LABELS,
|
|
64
65
|
ScoreDisplayModeSchema: () => ScoreDisplayModeSchema,
|
|
65
66
|
TAG_SCAN_ERROR: () => TAG_SCAN_ERROR,
|
|
67
|
+
TAG_SKIPPED_NO_EVIDENCE: () => TAG_SKIPPED_NO_EVIDENCE,
|
|
66
68
|
TAG_SKIPPED_PAGE_TYPE: () => TAG_SKIPPED_PAGE_TYPE,
|
|
69
|
+
allEvidenceMet: () => allEvidenceMet,
|
|
67
70
|
allJsonLdNodes: () => allJsonLdNodes,
|
|
68
71
|
buildCategoryResult: () => buildCategoryResult,
|
|
72
|
+
buildScanEvidence: () => buildScanEvidence,
|
|
69
73
|
calculateCategoryScore: () => calculateCategoryScore,
|
|
70
74
|
calculateOverallScore: () => calculateOverallScore,
|
|
71
75
|
classifyFetch: () => classifyFetch,
|
|
@@ -96,6 +100,7 @@ __export(index_exports, {
|
|
|
96
100
|
formatTrace: () => formatTrace,
|
|
97
101
|
getMainContentText: () => getMainContentText,
|
|
98
102
|
getPreset: () => getPreset,
|
|
103
|
+
getRenderedText: () => getRenderedText,
|
|
99
104
|
getScoreTier: () => getScoreTier,
|
|
100
105
|
getTierColor: () => getTierColor,
|
|
101
106
|
getTierLabel: () => getTierLabel,
|
|
@@ -145,6 +150,7 @@ var MAX_CONCURRENT_REQUESTS = 10;
|
|
|
145
150
|
var SCANNER_USER_AGENT = "AgentLighthouse/1.0 (+https://github.com/ForkPoint/agent-lighthouse)";
|
|
146
151
|
var TAG_SKIPPED_PAGE_TYPE = "skipped:page-type";
|
|
147
152
|
var TAG_SCAN_ERROR = "scan-error";
|
|
153
|
+
var TAG_SKIPPED_NO_EVIDENCE = "skipped:no-evidence";
|
|
148
154
|
var CATEGORY_NAMES = {
|
|
149
155
|
"access-crawl-control": "Access & Crawl Control",
|
|
150
156
|
"content-extraction": "Content Extraction",
|
|
@@ -363,6 +369,7 @@ function createFetcher() {
|
|
|
363
369
|
});
|
|
364
370
|
let gateArmed;
|
|
365
371
|
let hops = 0;
|
|
372
|
+
const redirectChain = [];
|
|
366
373
|
while (followRedirects && REDIRECT_STATUS.has(response.statusCode) && response.headers["location"] !== void 0 && hops < MAX_REDIRECTS) {
|
|
367
374
|
const rawLocation = response.headers["location"];
|
|
368
375
|
const location = Array.isArray(rawLocation) ? rawLocation[0] : rawLocation;
|
|
@@ -398,6 +405,7 @@ function createFetcher() {
|
|
|
398
405
|
currentMethod = "GET";
|
|
399
406
|
currentBody = void 0;
|
|
400
407
|
}
|
|
408
|
+
redirectChain.push({ status: response.statusCode, from: currentUrl, to: next });
|
|
401
409
|
currentUrl = next;
|
|
402
410
|
hops += 1;
|
|
403
411
|
response = await (0, import_undici.request)(currentUrl, {
|
|
@@ -444,7 +452,8 @@ function createFetcher() {
|
|
|
444
452
|
totalMs: Math.round(totalMs),
|
|
445
453
|
contentType: headers["content-type"] ?? "",
|
|
446
454
|
contentLength: bytes ? bytes.byteLength : truncatedBody.length,
|
|
447
|
-
...bytes ? { bytes } : {}
|
|
455
|
+
...bytes ? { bytes } : {},
|
|
456
|
+
...redirectChain.length > 0 ? { redirectChain } : {}
|
|
448
457
|
};
|
|
449
458
|
} catch (err) {
|
|
450
459
|
const totalMs = performance.now() - start;
|
|
@@ -693,12 +702,23 @@ function extractHeadings($) {
|
|
|
693
702
|
});
|
|
694
703
|
return headings;
|
|
695
704
|
}
|
|
696
|
-
function
|
|
697
|
-
const root = $("main").first().length ? $("main").first() : $("body");
|
|
705
|
+
function readableText(root) {
|
|
698
706
|
const clone = root.clone();
|
|
699
707
|
clone.find("script, style, noscript, template").remove();
|
|
700
708
|
return clone.text().replace(/\s+/g, " ").trim();
|
|
701
709
|
}
|
|
710
|
+
function getMainContentText($) {
|
|
711
|
+
let best = "";
|
|
712
|
+
$("body").find("main").each((_, el) => {
|
|
713
|
+
const text3 = readableText($(el));
|
|
714
|
+
if (text3.length > best.length) best = text3;
|
|
715
|
+
});
|
|
716
|
+
if (best) return best;
|
|
717
|
+
return readableText($("body"));
|
|
718
|
+
}
|
|
719
|
+
function getRenderedText($) {
|
|
720
|
+
return readableText($("body"));
|
|
721
|
+
}
|
|
702
722
|
function getWordCount($) {
|
|
703
723
|
const text3 = getMainContentText($);
|
|
704
724
|
return text3.split(/\s+/).filter(Boolean).length;
|
|
@@ -932,6 +952,12 @@ var DeprecationNoticeSchema = import_zod.z.object({
|
|
|
932
952
|
var EvidenceGradeSchema = import_zod.z.enum(["A", "B", "C", "D"]);
|
|
933
953
|
var AuditTierSchema = import_zod.z.enum(["scored", "informative", "experimental"]);
|
|
934
954
|
var AUDIT_ID_PATTERN = /^[a-z-]+\/[a-z0-9-]+$/;
|
|
955
|
+
var EvidenceKeySchema = import_zod.z.enum([
|
|
956
|
+
"origin-reachable",
|
|
957
|
+
"unblocked-fetches",
|
|
958
|
+
"rendered-body",
|
|
959
|
+
"sample-adequate"
|
|
960
|
+
]);
|
|
935
961
|
var AuditMetaSchema = import_zod.z.object({
|
|
936
962
|
id: import_zod.z.string().regex(AUDIT_ID_PATTERN, "audit id must be a `category/slug` path"),
|
|
937
963
|
category: import_zod.z.string(),
|
|
@@ -950,7 +976,10 @@ var AuditMetaSchema = import_zod.z.object({
|
|
|
950
976
|
// its weight comes from (grade + tier) and which dossier proves it.
|
|
951
977
|
evidenceGrade: EvidenceGradeSchema,
|
|
952
978
|
tier: AuditTierSchema,
|
|
953
|
-
dossier: import_zod.z.string().min(1).max(500)
|
|
979
|
+
dossier: import_zod.z.string().min(1).max(500),
|
|
980
|
+
// What the audit needs the scan to have obtained. Checked against the
|
|
981
|
+
// source by `scripts/check-requires.mjs`, not enforced here beyond shape.
|
|
982
|
+
requires: import_zod.z.array(EvidenceKeySchema).optional()
|
|
954
983
|
});
|
|
955
984
|
var CheckResultSchema = import_zod.z.object({
|
|
956
985
|
// v2 ids are `category/slug` paths, which outgrew the old 20-char cap.
|
|
@@ -1175,6 +1204,19 @@ function calculateOverallScore(categories) {
|
|
|
1175
1204
|
if (totalMass === 0) return 0;
|
|
1176
1205
|
return Math.round(weighted / totalMass);
|
|
1177
1206
|
}
|
|
1207
|
+
var GATED_MASS_UNSCORED_THRESHOLD = 0.35;
|
|
1208
|
+
function gatedMassShare(checks2) {
|
|
1209
|
+
let gated = 0;
|
|
1210
|
+
let total = 0;
|
|
1211
|
+
for (const check of checks2) {
|
|
1212
|
+
if (isInformative(check)) continue;
|
|
1213
|
+
const mass = check.weight ?? 0;
|
|
1214
|
+
if (mass <= 0) continue;
|
|
1215
|
+
total += mass;
|
|
1216
|
+
if (check.tags?.includes(TAG_SKIPPED_NO_EVIDENCE)) gated += mass;
|
|
1217
|
+
}
|
|
1218
|
+
return total === 0 ? 0 : gated / total;
|
|
1219
|
+
}
|
|
1178
1220
|
|
|
1179
1221
|
// src/audits/access-crawl-control/no-nofollow.ts
|
|
1180
1222
|
var NoNofollowAudit = class _NoNofollowAudit extends Audit {
|
|
@@ -1189,6 +1231,8 @@ var NoNofollowAudit = class _NoNofollowAudit extends Audit {
|
|
|
1189
1231
|
evidenceGrade: "A",
|
|
1190
1232
|
tier: "scored",
|
|
1191
1233
|
dossier: "docs/evidence/audits/access-crawl-control/no-nofollow.md",
|
|
1234
|
+
// Gate exemption: being refused is what this category reports.
|
|
1235
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
1192
1236
|
defaultPriority: "high",
|
|
1193
1237
|
guidance: {
|
|
1194
1238
|
impact: "A nofollow directive prevents AI crawlers from following links on your pages, effectively hiding all linked content from AI indexing. Your deeper pages become invisible to AI search engines, drastically reducing discoverability.",
|
|
@@ -1270,6 +1314,8 @@ var NoRedirectChainsAudit = class _NoRedirectChainsAudit extends Audit {
|
|
|
1270
1314
|
evidenceGrade: "A",
|
|
1271
1315
|
tier: "scored",
|
|
1272
1316
|
dossier: "docs/evidence/audits/access-crawl-control/no-redirect-chains.md",
|
|
1317
|
+
// Gate exemption: being refused is what this category reports.
|
|
1318
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
1273
1319
|
defaultPriority: "medium",
|
|
1274
1320
|
guidance: {
|
|
1275
1321
|
impact: "Redirect chains slow down AI crawlers and waste their limited crawl budget. Each extra redirect adds latency and increases the chance a crawler gives up before reaching the final page, leaving content unindexed.",
|
|
@@ -1415,6 +1461,8 @@ var CanonicalLinksAudit = class extends Audit {
|
|
|
1415
1461
|
evidenceGrade: "A",
|
|
1416
1462
|
tier: "scored",
|
|
1417
1463
|
dossier: "docs/evidence/audits/access-crawl-control/canonical.md",
|
|
1464
|
+
// Gate exemption: being refused is what this category reports.
|
|
1465
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
1418
1466
|
defaultPriority: "medium",
|
|
1419
1467
|
guidance: {
|
|
1420
1468
|
impact: "A canonical pointing at the wrong URL is worse than no canonical at all: when every page canonicalizes onto the homepage \u2014 a common CMS and SPA template bug \u2014 the pages consolidate onto one URL and drop out of the index that AI Overviews and AI Mode draw on. A canonical pointing at another domain hands the attribution there.",
|
|
@@ -1863,6 +1911,8 @@ var GptbotAudit = class extends CrawlerBotAudit {
|
|
|
1863
1911
|
evidenceGrade: "A",
|
|
1864
1912
|
tier: "scored",
|
|
1865
1913
|
dossier: "docs/evidence/audits/access-crawl-control/gptbot.md",
|
|
1914
|
+
// Gate exemption: being refused is what this category reports.
|
|
1915
|
+
requires: ["origin-reachable"],
|
|
1866
1916
|
defaultPriority: "medium",
|
|
1867
1917
|
guidance: {
|
|
1868
1918
|
impact: "Blocking GPTBot prevents your content from being used by OpenAI's models and appearing in ChatGPT responses. Explicitly allowing it signals that your site welcomes AI indexing for the largest AI platform by user base.",
|
|
@@ -1893,6 +1943,8 @@ var GoogleExtendedAudit = class extends CrawlerBotAudit {
|
|
|
1893
1943
|
evidenceGrade: "A",
|
|
1894
1944
|
tier: "scored",
|
|
1895
1945
|
dossier: "docs/evidence/audits/access-crawl-control/google-extended.md",
|
|
1946
|
+
// Gate exemption: being refused is what this category reports.
|
|
1947
|
+
requires: ["origin-reachable"],
|
|
1896
1948
|
defaultPriority: "medium",
|
|
1897
1949
|
guidance: {
|
|
1898
1950
|
impact: "Blocking Google-Extended prevents your content from being used in Google's AI features like Gemini and AI Overviews. Allowing it ensures your site appears in Google's AI-powered search experiences alongside traditional results.",
|
|
@@ -1927,6 +1979,8 @@ var AnthropicAudit = class extends CrawlerBotAudit {
|
|
|
1927
1979
|
evidenceGrade: "A",
|
|
1928
1980
|
tier: "scored",
|
|
1929
1981
|
dossier: "docs/evidence/audits/access-crawl-control/anthropic-ai.md",
|
|
1982
|
+
// Gate exemption: being refused is what this category reports.
|
|
1983
|
+
requires: ["origin-reachable"],
|
|
1930
1984
|
defaultPriority: "medium",
|
|
1931
1985
|
guidance: {
|
|
1932
1986
|
impact: "Disallowing ClaudeBot keeps the site out of the web content Anthropic collects for potential model training. It is an effective, documented control, so it is only a problem where the block was not intended. It buys back very little traffic either way: Cloudflare Radar measures Anthropic's crawl-to-refer ratio at roughly 50,000:1, so the allow-side case is about corpus inclusion rather than referral visibility.",
|
|
@@ -2036,6 +2090,8 @@ var PerplexitybotAudit = class extends CrawlerBotAudit {
|
|
|
2036
2090
|
evidenceGrade: "A",
|
|
2037
2091
|
tier: "scored",
|
|
2038
2092
|
dossier: "docs/evidence/audits/access-crawl-control/perplexitybot.md",
|
|
2093
|
+
// Gate exemption: being refused is what this category reports.
|
|
2094
|
+
requires: ["origin-reachable"],
|
|
2039
2095
|
defaultPriority: "medium",
|
|
2040
2096
|
guidance: {
|
|
2041
2097
|
impact: "Blocking PerplexityBot prevents your content from appearing in Perplexity AI search results, one of the fastest-growing AI answer engines. Allowing it gives your content visibility in AI-native search.",
|
|
@@ -2066,6 +2122,8 @@ var ApplebotExtendedAudit = class extends CrawlerBotAudit {
|
|
|
2066
2122
|
evidenceGrade: "A",
|
|
2067
2123
|
tier: "scored",
|
|
2068
2124
|
dossier: "docs/evidence/audits/access-crawl-control/applebot-extended.md",
|
|
2125
|
+
// Gate exemption: being refused is what this category reports.
|
|
2126
|
+
requires: ["origin-reachable"],
|
|
2069
2127
|
defaultPriority: "medium",
|
|
2070
2128
|
guidance: {
|
|
2071
2129
|
impact: "Blocking Applebot-Extended prevents your content from being used in Apple Intelligence features, Siri AI answers, and Safari Highlights. Allowing it ensures visibility across Apple's AI ecosystem.",
|
|
@@ -2096,6 +2154,8 @@ var CcbotAudit = class extends CrawlerBotAudit {
|
|
|
2096
2154
|
evidenceGrade: "A",
|
|
2097
2155
|
tier: "scored",
|
|
2098
2156
|
dossier: "docs/evidence/audits/access-crawl-control/ccbot.md",
|
|
2157
|
+
// Gate exemption: being refused is what this category reports.
|
|
2158
|
+
requires: ["origin-reachable"],
|
|
2099
2159
|
defaultPriority: "medium",
|
|
2100
2160
|
guidance: {
|
|
2101
2161
|
impact: "Blocking CCBot prevents your content from being included in the Common Crawl dataset, which is a foundational training data source for many AI models. Allowing it broadens your content's reach across multiple AI systems.",
|
|
@@ -2127,6 +2187,8 @@ var MetaExternalAgentAudit = class extends CrawlerBotAudit {
|
|
|
2127
2187
|
evidenceGrade: "A",
|
|
2128
2188
|
tier: "scored",
|
|
2129
2189
|
dossier: "docs/evidence/audits/access-crawl-control/meta-external-agent.md",
|
|
2190
|
+
// Gate exemption: being refused is what this category reports.
|
|
2191
|
+
requires: ["origin-reachable"],
|
|
2130
2192
|
defaultPriority: "medium",
|
|
2131
2193
|
guidance: {
|
|
2132
2194
|
impact: "Disallowing Meta-ExternalAgent keeps the site out of Meta's foundation-model training corpus and out of the direct content indexing that improves Meta products. It is an effective, documented control, so it is only a problem where the block was not intended. It does not by itself govern Meta AI search citations \u2014 Meta documents Meta-WebIndexer as the token behind those.",
|
|
@@ -2219,6 +2281,8 @@ var AmazonbotAudit = class extends CrawlerBotAudit {
|
|
|
2219
2281
|
evidenceGrade: "A",
|
|
2220
2282
|
tier: "scored",
|
|
2221
2283
|
dossier: "docs/evidence/audits/access-crawl-control/amazonbot.md",
|
|
2284
|
+
// Gate exemption: being refused is what this category reports.
|
|
2285
|
+
requires: ["origin-reachable"],
|
|
2222
2286
|
defaultPriority: "medium",
|
|
2223
2287
|
guidance: {
|
|
2224
2288
|
impact: "Blocking Amazonbot prevents your content from appearing in Alexa AI answers and Amazon's AI-powered search features. Allowing it gives your content visibility in Amazon's voice and commerce AI ecosystem.",
|
|
@@ -2293,6 +2357,8 @@ var AiBotDirectivesAudit = class extends Audit {
|
|
|
2293
2357
|
evidenceGrade: "B",
|
|
2294
2358
|
tier: "scored",
|
|
2295
2359
|
dossier: "docs/evidence/audits/access-crawl-control/ai-bot-directives.md",
|
|
2360
|
+
// Gate exemption: being refused is what this category reports.
|
|
2361
|
+
requires: ["origin-reachable"],
|
|
2296
2362
|
defaultPriority: "medium",
|
|
2297
2363
|
guidance: {
|
|
2298
2364
|
impact: "Blocking YouBot removes the site from You.com's live search index; blocking AI2Bot removes it from the Allen Institute's open training corpora while leaving closed commercial crawlers untouched. Leaving either to the wildcard rule means the policy silently flips the day a blanket block is added. The other three tokens carry no comparable consumer, so this audit never penalises blocking them.",
|
|
@@ -2359,6 +2425,8 @@ var ChatgptUserAudit = class extends CrawlerBotAudit {
|
|
|
2359
2425
|
evidenceGrade: "C",
|
|
2360
2426
|
tier: "informative",
|
|
2361
2427
|
dossier: "docs/evidence/audits/access-crawl-control/chatgpt-user.md",
|
|
2428
|
+
// Gate exemption: being refused is what this category reports.
|
|
2429
|
+
requires: ["origin-reachable"],
|
|
2362
2430
|
defaultPriority: "medium",
|
|
2363
2431
|
guidance: {
|
|
2364
2432
|
impact: "Blocking ChatGPT-User prevents ChatGPT from browsing your site in real-time when users ask it to visit your pages. This blocks your content from being cited in ChatGPT Browse conversations, losing a significant source of AI-driven traffic.",
|
|
@@ -2389,6 +2457,8 @@ var ClaudeUserAudit = class extends CrawlerBotAudit {
|
|
|
2389
2457
|
evidenceGrade: "A",
|
|
2390
2458
|
tier: "scored",
|
|
2391
2459
|
dossier: "docs/evidence/audits/access-crawl-control/claude-user.md",
|
|
2460
|
+
// Gate exemption: being refused is what this category reports.
|
|
2461
|
+
requires: ["origin-reachable"],
|
|
2392
2462
|
defaultPriority: "medium",
|
|
2393
2463
|
guidance: {
|
|
2394
2464
|
impact: "Blocking Claude-User prevents Claude from browsing your site in real-time when users ask it to visit your pages. This blocks your content from being cited in Claude conversations with web access enabled.",
|
|
@@ -2418,6 +2488,8 @@ var OaiSearchbotAudit = class extends CrawlerBotAudit {
|
|
|
2418
2488
|
evidenceGrade: "A",
|
|
2419
2489
|
tier: "scored",
|
|
2420
2490
|
dossier: "docs/evidence/audits/access-crawl-control/oai-searchbot.md",
|
|
2491
|
+
// Gate exemption: being refused is what this category reports.
|
|
2492
|
+
requires: ["origin-reachable"],
|
|
2421
2493
|
defaultPriority: "medium",
|
|
2422
2494
|
guidance: {
|
|
2423
2495
|
impact: "Blocking OAI-SearchBot prevents your content from appearing in OpenAI's SearchGPT and ChatGPT web search results. Allowing it ensures your site is discoverable through OpenAI's real-time search features.",
|
|
@@ -2448,6 +2520,8 @@ var MetaExternalFetcherAudit = class extends CrawlerBotAudit {
|
|
|
2448
2520
|
evidenceGrade: "A",
|
|
2449
2521
|
tier: "scored",
|
|
2450
2522
|
dossier: "docs/evidence/audits/access-crawl-control/meta-external-fetcher.md",
|
|
2523
|
+
// Gate exemption: being refused is what this category reports.
|
|
2524
|
+
requires: ["origin-reachable"],
|
|
2451
2525
|
defaultPriority: "medium",
|
|
2452
2526
|
guidance: {
|
|
2453
2527
|
impact: "Blocking Meta-ExternalFetcher prevents Meta's AI from fetching your content in real-time for AI-powered features across Facebook, Instagram, and WhatsApp. Allowing it ensures your content can be surfaced in Meta's real-time AI experiences.",
|
|
@@ -2477,6 +2551,8 @@ var BravebotAudit = class extends CrawlerBotAudit {
|
|
|
2477
2551
|
evidenceGrade: "C",
|
|
2478
2552
|
tier: "informative",
|
|
2479
2553
|
dossier: "docs/evidence/audits/access-crawl-control/bravebot.md",
|
|
2554
|
+
// Gate exemption: being refused is what this category reports.
|
|
2555
|
+
requires: ["origin-reachable"],
|
|
2480
2556
|
defaultPriority: "medium",
|
|
2481
2557
|
guidance: {
|
|
2482
2558
|
impact: "Blocking Bravebot prevents your content from appearing in Brave Search AI answers and Brave Leo AI assistant responses. Allowing it gives your content visibility in the privacy-focused Brave browser ecosystem.",
|
|
@@ -2506,6 +2582,8 @@ var DuckassistbotAudit = class extends CrawlerBotAudit {
|
|
|
2506
2582
|
evidenceGrade: "A",
|
|
2507
2583
|
tier: "scored",
|
|
2508
2584
|
dossier: "docs/evidence/audits/access-crawl-control/duckassistbot.md",
|
|
2585
|
+
// Gate exemption: being refused is what this category reports.
|
|
2586
|
+
requires: ["origin-reachable"],
|
|
2509
2587
|
defaultPriority: "medium",
|
|
2510
2588
|
guidance: {
|
|
2511
2589
|
impact: "Blocking DuckAssistBot prevents your content from appearing in DuckDuckGo's AI-powered DuckAssist feature, which generates instant answers from crawled web pages. Allowing it ensures visibility in this privacy-first AI search experience.",
|
|
@@ -2535,6 +2613,8 @@ var MistralaiUserAudit = class extends CrawlerBotAudit {
|
|
|
2535
2613
|
evidenceGrade: "A",
|
|
2536
2614
|
tier: "scored",
|
|
2537
2615
|
dossier: "docs/evidence/audits/access-crawl-control/mistralai-user.md",
|
|
2616
|
+
// Gate exemption: being refused is what this category reports.
|
|
2617
|
+
requires: ["origin-reachable"],
|
|
2538
2618
|
defaultPriority: "medium",
|
|
2539
2619
|
guidance: {
|
|
2540
2620
|
impact: "Blocking MistralAI-User prevents Mistral AI's Le Chat from browsing your site in real-time when users ask it to visit your pages. Allowing it ensures your content can be cited in Mistral-powered AI conversations.",
|
|
@@ -2564,6 +2644,8 @@ var ClaudeSearchbotAudit = class extends CrawlerBotAudit {
|
|
|
2564
2644
|
evidenceGrade: "A",
|
|
2565
2645
|
tier: "scored",
|
|
2566
2646
|
dossier: "docs/evidence/audits/access-crawl-control/claude-searchbot.md",
|
|
2647
|
+
// Gate exemption: being refused is what this category reports.
|
|
2648
|
+
requires: ["origin-reachable"],
|
|
2567
2649
|
defaultPriority: "medium",
|
|
2568
2650
|
guidance: {
|
|
2569
2651
|
impact: "Blocking Claude-SearchBot prevents your content from appearing in Claude's web search results. Allowing it ensures your site is included when Claude searches the web to answer user questions.",
|
|
@@ -2593,6 +2675,8 @@ var NoBlanketBlockAudit = class extends Audit {
|
|
|
2593
2675
|
evidenceGrade: "B",
|
|
2594
2676
|
tier: "scored",
|
|
2595
2677
|
dossier: "docs/evidence/audits/access-crawl-control/no-blanket-block.md",
|
|
2678
|
+
// Gate exemption: being refused is what this category reports.
|
|
2679
|
+
requires: ["origin-reachable"],
|
|
2596
2680
|
defaultPriority: "critical",
|
|
2597
2681
|
guidance: {
|
|
2598
2682
|
impact: "A blanket Disallow: / under User-agent: * blocks every crawler, including all AI agents. Your site becomes completely invisible to AI search engines, ChatGPT Browse, Perplexity, Claude, and all other AI-powered discovery tools.",
|
|
@@ -2746,6 +2830,8 @@ var SensitivePathsAudit = class extends Audit {
|
|
|
2746
2830
|
evidenceGrade: "A",
|
|
2747
2831
|
tier: "scored",
|
|
2748
2832
|
dossier: "docs/evidence/audits/access-crawl-control/sensitive-paths.md",
|
|
2833
|
+
// Gate exemption: being refused is what this category reports.
|
|
2834
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
2749
2835
|
defaultPriority: "low",
|
|
2750
2836
|
guidance: {
|
|
2751
2837
|
impact: 'Cart, checkout, site-search, login and account URLs carry nothing an answer engine can cite, but they are crawled and can surface in AI answers as dead, session-bearing links. Apple documents Applebot and Applebot-Extended honouring "Disallow: /private/", and Meta documents the same for meta-externalagent, so a path-level rule keeps that noise out of AI crawls. Two limits matter: RFC 9309 states the protocol "is not a substitute for valid content security measures" and that listed paths become publicly discoverable, so never use robots.txt to protect anything; and user-initiated fetchers are documented not to obey it \u2014 OpenAI says of ChatGPT-User "Because these actions are initiated by a user, robots.txt rules may not apply", and Perplexity says Perplexity-User "generally ignores robots.txt rules".',
|
|
@@ -2841,6 +2927,8 @@ var CrawlDelayAudit = class extends Audit {
|
|
|
2841
2927
|
evidenceGrade: "C",
|
|
2842
2928
|
tier: "informative",
|
|
2843
2929
|
dossier: "docs/evidence/audits/access-crawl-control/crawl-delay.md",
|
|
2930
|
+
// Gate exemption: being refused is what this category reports.
|
|
2931
|
+
requires: ["origin-reachable"],
|
|
2844
2932
|
defaultPriority: "high",
|
|
2845
2933
|
guidance: {
|
|
2846
2934
|
impact: "Excessive Crawl-delay values (over 10 seconds) dramatically slow AI indexing, meaning your latest content may take days or weeks to appear in AI search results while competitors with lower delays get indexed faster.",
|
|
@@ -2981,6 +3069,8 @@ var MetaRobotsNotBlockingAudit = class extends Audit {
|
|
|
2981
3069
|
evidenceGrade: "A",
|
|
2982
3070
|
tier: "scored",
|
|
2983
3071
|
dossier: "docs/evidence/audits/access-crawl-control/robots-directives.md",
|
|
3072
|
+
// Gate exemption: being refused is what this category reports.
|
|
3073
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
2984
3074
|
defaultPriority: "high",
|
|
2985
3075
|
guidance: {
|
|
2986
3076
|
impact: 'A content page carrying "noindex" (in a robots meta tag, a per-bot meta tag, or the X-Robots-Tag response header) is dropped from the search index, and Google documents that a page must be indexed to appear in AI Overviews or AI Mode. "nosnippet", "noarchive" and "max-snippet:0" keep the page indexed but stop its text being used as a direct input for AI answers.',
|
|
@@ -3069,6 +3159,8 @@ var NoBotDetectionAudit = class extends Audit {
|
|
|
3069
3159
|
evidenceGrade: "A",
|
|
3070
3160
|
tier: "scored",
|
|
3071
3161
|
dossier: "docs/evidence/audits/access-crawl-control/no-bot-detection.md",
|
|
3162
|
+
// Gate exemption: being refused is what this category reports.
|
|
3163
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
3072
3164
|
defaultPriority: "high",
|
|
3073
3165
|
guidance: {
|
|
3074
3166
|
impact: "Bot-detection services like Cloudflare Turnstile, DataDome, and reCAPTCHA can block legitimate AI agents from accessing your content. When agents are challenged, they cannot complete page fetches, making your content inaccessible to AI-powered search and assistants.",
|
|
@@ -3230,6 +3322,8 @@ var TdmRepAudit = class extends Audit {
|
|
|
3230
3322
|
evidenceGrade: "C",
|
|
3231
3323
|
tier: "experimental",
|
|
3232
3324
|
dossier: "docs/evidence/audits/access-crawl-control/tdm-rep.md",
|
|
3325
|
+
// Gate exemption: being refused is what this category reports.
|
|
3326
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
3233
3327
|
// Nothing consumes the signal, so nothing here should outrank an item that
|
|
3234
3328
|
// changes what an agent can do.
|
|
3235
3329
|
defaultPriority: "low",
|
|
@@ -3378,6 +3472,8 @@ var AgentGovernanceAudit = class extends Audit {
|
|
|
3378
3472
|
evidenceGrade: "A",
|
|
3379
3473
|
tier: "scored",
|
|
3380
3474
|
dossier: "docs/evidence/audits/access-crawl-control/agent-governance.md",
|
|
3475
|
+
// Gate exemption: being refused is what this category reports.
|
|
3476
|
+
requires: ["origin-reachable"],
|
|
3381
3477
|
defaultPriority: "medium",
|
|
3382
3478
|
guidance: {
|
|
3383
3479
|
impact: "Without separate rules for training crawlers and live conversational agents, you cannot block dataset scraping while still appearing in ChatGPT, Claude, and Perplexity answers. A blanket policy either locks you out of AI-powered discovery entirely or leaves your content open to bulk training crawls you never agreed to.",
|
|
@@ -3513,6 +3609,8 @@ var AiContentDeclarationAudit = class extends Audit {
|
|
|
3513
3609
|
evidenceGrade: "D",
|
|
3514
3610
|
tier: "experimental",
|
|
3515
3611
|
dossier: "docs/evidence/audits/access-crawl-control/ai-content-declaration.md",
|
|
3612
|
+
// Gate exemption: being refused is what this category reports.
|
|
3613
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
3516
3614
|
// Was `medium` on an invented directive; the whole class of signals is
|
|
3517
3615
|
// pre-consumer, so nothing here should outrank an actionable item.
|
|
3518
3616
|
defaultPriority: "low",
|
|
@@ -3576,6 +3674,8 @@ var HttpsEnabledAudit = class extends Audit {
|
|
|
3576
3674
|
evidenceGrade: "A",
|
|
3577
3675
|
tier: "scored",
|
|
3578
3676
|
dossier: "docs/evidence/audits/access-crawl-control/https-enabled.md",
|
|
3677
|
+
// Gate exemption: being refused is what this category reports.
|
|
3678
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
3579
3679
|
defaultPriority: "critical",
|
|
3580
3680
|
guidance: {
|
|
3581
3681
|
impact: "HTTP-only sites are completely excluded from all major AI systems. GPTBot, ClaudeBot, Perplexity, and enterprise RAG pipelines refuse to connect to non-HTTPS origins due to security policies. Your entire site is invisible to AI-generated answers, product recommendations, and agentic workflows.",
|
|
@@ -3724,6 +3824,8 @@ var RobotsAiGroupShadowingAudit = class _RobotsAiGroupShadowingAudit extends Aud
|
|
|
3724
3824
|
evidenceGrade: "A",
|
|
3725
3825
|
tier: "scored",
|
|
3726
3826
|
dossier: "docs/evidence/audits/access-crawl-control/robots-ai-group-shadowing.md",
|
|
3827
|
+
// Gate exemption: being refused is what this category reports.
|
|
3828
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
3727
3829
|
defaultPriority: "high",
|
|
3728
3830
|
guidance: {
|
|
3729
3831
|
impact: "RFC 9309 \xA72.2.1 states the wildcard group is consulted only 'if no matching group exists'. Therefore, for any site with a named AI-bot group, the wildcard group's Disallow rules provably do not apply to that bot, and the operator's stated intent (expressed once in `*`) diverges from the enforced policy by exactly the symmetric difference of the two rule sets. Falsifiable by construction: given robots.txt R and token T, the set of paths where R_T and R_star disagree is computable and either empty or not.",
|
|
@@ -4182,6 +4284,8 @@ var AiCrawlerEdgeParityAudit = class extends Audit {
|
|
|
4182
4284
|
evidenceGrade: "A",
|
|
4183
4285
|
tier: "scored",
|
|
4184
4286
|
dossier: "docs/evidence/audits/access-crawl-control/ai-crawler-edge-parity.md",
|
|
4287
|
+
// Gate exemption: being refused is what this category reports.
|
|
4288
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
4185
4289
|
defaultPriority: "critical",
|
|
4186
4290
|
guidance: {
|
|
4187
4291
|
impact: 'robots.txt (RFC 9309) is advisory metadata parsed by the crawler; the edge access decision is enforced independently by the WAF. A site can therefore publish "User-agent: PerplexityBot / Allow: /" and return a non-200 to every request carrying that user agent, and the operator \u2014 who reads their own robots.txt \u2014 believes they are open while the crawler never sees a byte. Falsifiable: fetch URL U with a browser UA and with crawler UA C; if robots.txt permits C for U and the C request is not 2xx while the browser request is 200, the two policy layers contradict each other. Cloudflare makes one branch deterministic \u2014 a challenge always carries cf-mitigated: challenge \u2014 and a 200 whose main-content text is under 40% of the baseline is a block wearing a 200.',
|
|
@@ -4397,6 +4501,8 @@ var BotContentDeltaDeclaredAudit = class extends Audit {
|
|
|
4397
4501
|
evidenceGrade: "A",
|
|
4398
4502
|
tier: "scored",
|
|
4399
4503
|
dossier: "docs/evidence/audits/access-crawl-control/bot-content-delta-declared.md",
|
|
4504
|
+
// Gate exemption: being refused is what this category reports.
|
|
4505
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
4400
4506
|
defaultPriority: "high",
|
|
4401
4507
|
guidance: {
|
|
4402
4508
|
impact: "Google states that isAccessibleForFree: false with hasPart/cssSelector markup 'helps Google differentiate paywalled content from the practice of cloaking, which violates spam policies' \u2014 serving a crawler less than a user is sanctioned only when it is declared. The measurement is falsifiable both ways: extract the main text of URL U under a browser UA and under crawler UA C, and if the length ratio falls below 0.6 or the 5-gram shingle similarity below 0.7, the site conditions content on the User-Agent. The declaration is equally checkable, and the declared cssSelector must match a real element in the served HTML \u2014 which is where most implementations silently fail, leaving markup that validates and points at nothing. The second-order cost is not the spam risk: an answer engine that only ever sees the stub cites the stub.",
|
|
@@ -4697,6 +4803,8 @@ var AiUsageSignalCoherenceAcrossChannelsAudit = class extends Audit {
|
|
|
4697
4803
|
weight: weightForGrade("B", "scored"),
|
|
4698
4804
|
defaultPriority: "high",
|
|
4699
4805
|
dossier: "docs/evidence/audits/access-crawl-control/ai-usage-signal-coherence-across-channels.md",
|
|
4806
|
+
// Gate exemption: being refused is what this category reports.
|
|
4807
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
4700
4808
|
guidance: {
|
|
4701
4809
|
impact: "No standard defines precedence between these channels; each specifies only its own parsing. A crawler that reads TDM-Rep and a crawler that reads AIPREF therefore read disjoint inputs, and when those inputs disagree the two reach opposite conclusions about the same page. Whichever one you did not mean to publish is the one some operator will act on. The documented worst case is not even yours to make: Cloudflare\u2019s managed robots.txt prepends its own Content-Signal block above your file, so your stated policy can be contradicted at the edge without you knowing.",
|
|
4702
4810
|
fix: "Decide the policy once, then say the same thing in every channel you publish. If you do not intend to maintain a channel, remove it rather than leaving a stale value \u2014 a contradicted signal is worse than a missing one. Where your CDN prepends its own robots.txt block, either turn that feature off or make your own declarations match it.",
|
|
@@ -4909,6 +5017,8 @@ var AiprefContentUsageDeclarationValidityAudit = class extends Audit {
|
|
|
4909
5017
|
weight: weightForGrade("B", "scored"),
|
|
4910
5018
|
defaultPriority: "medium",
|
|
4911
5019
|
dossier: "docs/evidence/audits/access-crawl-control/aipref-content-usage-declaration-validity.md",
|
|
5020
|
+
// Gate exemption: being refused is what this category reports.
|
|
5021
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
4912
5022
|
guidance: {
|
|
4913
5023
|
impact: "AIPREF is the one AI-usage vocabulary on the IETF standards track, so a declaration written in it is the one a future crawler is most likely to read. A crawler that cannot parse the line ignores it, and the site is then treated as having no preference at all \u2014 the same outcome as publishing nothing, after the work of publishing something. The costliest version is invisible: a preference attached to a path robots.txt disallows is discarded by the spec itself, so the line looks right and does nothing.",
|
|
4914
5024
|
fix: "Write `Content-Usage: train-ai=n` \u2014 an RFC 8941 dictionary of `y`/`n` values against the `train-ai` and `search` categories. Use `yes`/`no` only in a Cloudflare `Content-Signal:` line, which is a different directive. Attach preferences to paths a crawler is allowed to fetch, and keep the robots.txt line and the response header saying the same thing for the same path.",
|
|
@@ -5100,6 +5210,8 @@ var RslLicensingTermsConformanceAudit = class extends Audit {
|
|
|
5100
5210
|
weight: weightForGrade("B", "scored"),
|
|
5101
5211
|
defaultPriority: "medium",
|
|
5102
5212
|
dossier: "docs/evidence/audits/access-crawl-control/rsl-licensing-terms-conformance.md",
|
|
5213
|
+
// Gate exemption: being refused is what this category reports.
|
|
5214
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
5103
5215
|
guidance: {
|
|
5104
5216
|
impact: 'RSL is the machine-readable form of "here are my terms". A crawler that cannot find the document applies its own defaults instead, and a document it finds but cannot parse is worth no more than one it never found. The specification mandates no default location, so a licence reachable only at a guessed path is one no crawler is obliged to look for. The quiet failure is a `<content url>` prefix that does not cover the pages the licence was written for: the terms load, parse, and apply to nothing.',
|
|
5105
5217
|
fix: 'Point at the licence from robots.txt with an absolute `License:` URI, and add the `Link: <...>; rel="license"; type="application/rsl+xml"` response header so a crawler that never reads robots.txt still finds it. Serve the document as `application/rsl+xml`, keep the `https://rslstandard.org/rsl` namespace on the root element, and make every `<content url>` prefix cover the paths it licenses.',
|
|
@@ -5407,6 +5519,8 @@ var MachineActionable402PaidAccessAudit = class extends Audit {
|
|
|
5407
5519
|
weight: weightForGrade("B", "scored"),
|
|
5408
5520
|
defaultPriority: "medium",
|
|
5409
5521
|
dossier: "docs/evidence/audits/access-crawl-control/machine-actionable-402-paid-access.md",
|
|
5522
|
+
// Gate exemption: being refused is what this category reports.
|
|
5523
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
5410
5524
|
guidance: {
|
|
5411
5525
|
impact: "Charging for crawler access is a legitimate choice, and 402 is the status code for it. But a crawler is a program: it can pay only what it can parse. A 402 whose body is an HTML page explaining your licensing terms reads, to the client, as an unexplained refusal \u2014 the same outcome as a 403, after you built a paywall meant to earn revenue. A 402 that a shared cache is allowed to store is worse: the next crawler gets a stored refusal even after paying.",
|
|
5412
5526
|
fix: 'Send one of the machine-readable forms with the 402: Cloudflare\u2019s `crawler-price: USD 0.01`, an x402 `PAYMENT-REQUIRED` challenge listing what you accept, or a `Link: rel=license` pointing at an RSL document whose `<payment type="crawl">` covers the path. Mark the response `Cache-Control: no-store` so a proxy cannot hand your 402 to a crawler that already paid.',
|
|
@@ -5611,6 +5725,8 @@ var WebBotAuthRequestToleranceAudit = class _WebBotAuthRequestToleranceAudit ext
|
|
|
5611
5725
|
weight: weightForGrade("B", "scored"),
|
|
5612
5726
|
defaultPriority: "medium",
|
|
5613
5727
|
dossier: "docs/evidence/audits/access-crawl-control/web-bot-auth-request-tolerance.md",
|
|
5728
|
+
// Gate exemption: being refused is what this category reports.
|
|
5729
|
+
requires: ["origin-reachable"],
|
|
5614
5730
|
guidance: {
|
|
5615
5731
|
impact: "Web Bot Auth is how an agent says who it is in a way an origin can check, and the operators building it are the ones whose traffic you would most want to identify. An edge that answers a signed request with 400 or 403 turns that identification into a reason for refusal: the agents willing to declare themselves are the ones you turn away, and the ones that lie carry no signature headers at all and sail through. A 431 is the same outcome from a different cause \u2014 a header-size limit \u2014 and it is fixed differently.",
|
|
5616
5732
|
fix: "Let unknown request headers through: `Signature`, `Signature-Input` and `Signature-Agent` are additive and safe to ignore. If your edge enforces a header-size budget, raise it enough for an Ed25519 signature. If you do vary behaviour on those headers, list them in `Vary` so a shared cache cannot serve the rejected variant to everyone.",
|
|
@@ -5799,6 +5915,7 @@ var ServerResponsivenessAudit = class extends Audit {
|
|
|
5799
5915
|
evidenceGrade: "B",
|
|
5800
5916
|
tier: "scored",
|
|
5801
5917
|
dossier: "docs/evidence/audits/content-extraction/server-responsiveness.md",
|
|
5918
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
5802
5919
|
defaultPriority: "medium",
|
|
5803
5920
|
guidance: {
|
|
5804
5921
|
impact: 'Google documents that crawl capacity falls when a host slows down ("if the site slows down\u2026 the limit goes down and Google crawls less"), and slow origins are where logged HTTP 499 client-closed-request clusters from AI fetchers appear. A slow origin therefore gets less of its content into the indexes AI answers are drawn from.',
|
|
@@ -5877,6 +5994,7 @@ var LanguageAttributeAudit = class extends Audit {
|
|
|
5877
5994
|
evidenceGrade: "A",
|
|
5878
5995
|
tier: "scored",
|
|
5879
5996
|
dossier: "docs/evidence/audits/content-extraction/language-attribute.md",
|
|
5997
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
5880
5998
|
defaultPriority: "high",
|
|
5881
5999
|
guidance: {
|
|
5882
6000
|
impact: "AI agents use the lang attribute to select the correct language model and tokenizer when processing your content. Without it, agents may misinterpret content language, leading to poor translations or incorrect answers in multilingual AI systems.",
|
|
@@ -6026,6 +6144,7 @@ var MarkdownAlternateAudit = class extends Audit {
|
|
|
6026
6144
|
weight: weightForGrade("A", "scored"),
|
|
6027
6145
|
defaultPriority: "medium",
|
|
6028
6146
|
dossier: "docs/evidence/audits/content-extraction/markdown-alternate.md",
|
|
6147
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6029
6148
|
guidance: {
|
|
6030
6149
|
impact: "A markdown alternate is a promise that an agent can read the page cheaply and get the same answer. A stale or partial alternate breaks that promise silently: the agent gets a document that looks authoritative, costs less, and says less than the page it claims to mirror. Serving it as `text/plain` or `text/html` is the same failure one level down \u2014 the client that negotiated for markdown cannot tell it got any. The consumers this is graded on are interactive coding agents \u2014 Claude Code, Cursor, Copilot Chat and CLI, Codex CLI \u2014 and GPTBot, measured taking markdown on 34.8% of fetches where a `.md` URL exists.",
|
|
6031
6150
|
fix: 'Serve the alternate from the same source as the HTML, so headings and prose cannot drift, with `Content-Type: text/markdown` (a `charset` parameter is fine). Publish it on the page URL plus `.md`, or answer `Accept: text/markdown` on the page URL itself \u2014 those are the two routes with documented consumers. Declaring it with `<link rel="alternate" type="text/markdown" href="...">` saves an agent a guess, but the link relation itself has one single-sourced consumer, so this audit reports it rather than scoring it.',
|
|
@@ -6256,6 +6375,7 @@ var JsonLdDuplicationMassAudit = class extends Audit {
|
|
|
6256
6375
|
weight: weightForGrade("C", "informative"),
|
|
6257
6376
|
defaultPriority: "low",
|
|
6258
6377
|
dossier: "docs/evidence/audits/content-extraction/json-ld-duplication-mass.md",
|
|
6378
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6259
6379
|
guidance: {
|
|
6260
6380
|
impact: "A non-rendering agent tokenizes the whole document, JSON-LD included. Where a block repeats the article body the DOM already carries, the page ships that text twice and the agent pays for both copies out of one context window. The same holds for a node declared identically in two blocks: the second copy adds tokens and no facts.",
|
|
6261
6381
|
fix: "Keep JSON-LD to the facts a parser needs \u2014 identifiers, prices, dates, relationships \u2014 and let the prose live in the DOM. Where a schema property genuinely needs body text, a summary is usually enough. Merge blocks that declare the same `@id` into one.",
|
|
@@ -6365,6 +6485,7 @@ var SingleH1Audit = class extends Audit {
|
|
|
6365
6485
|
evidenceGrade: "B",
|
|
6366
6486
|
tier: "scored",
|
|
6367
6487
|
dossier: "docs/evidence/audits/content-extraction/single-h1.md",
|
|
6488
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6368
6489
|
defaultPriority: "high",
|
|
6369
6490
|
guidance: {
|
|
6370
6491
|
impact: "AI agents use the single <h1> as the authoritative page title for content indexing and answer generation. Multiple <h1> elements create ambiguity about the page's primary topic, causing agents to misidentify or conflate subjects when generating answers.",
|
|
@@ -6427,6 +6548,7 @@ var SequentialHeadingsAudit = class extends Audit {
|
|
|
6427
6548
|
evidenceGrade: "B",
|
|
6428
6549
|
tier: "scored",
|
|
6429
6550
|
dossier: "docs/evidence/audits/content-extraction/sequential-headings.md",
|
|
6551
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6430
6552
|
defaultPriority: "high",
|
|
6431
6553
|
guidance: {
|
|
6432
6554
|
impact: "AI systems build content outlines from heading levels to understand document hierarchy. Skipped levels (e.g., h1 directly to h3) break this hierarchy, causing agents to misinterpret section nesting and produce inaccurate content summaries with wrong parent-child relationships.",
|
|
@@ -6516,6 +6638,7 @@ var MainElementAudit = class extends Audit {
|
|
|
6516
6638
|
evidenceGrade: "A",
|
|
6517
6639
|
tier: "scored",
|
|
6518
6640
|
dossier: "docs/evidence/audits/content-extraction/main-element.md",
|
|
6641
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6519
6642
|
defaultPriority: "high",
|
|
6520
6643
|
guidance: {
|
|
6521
6644
|
impact: "Without a <main> element, AI scrapers cannot distinguish primary content from navigation, sidebars, and footer boilerplate. This causes agents to ingest menus, disclaimers, and repeated chrome into their context window, increasing hallucination risk and reducing answer relevance.",
|
|
@@ -6573,6 +6696,7 @@ var ArticleElementAudit = class extends Audit {
|
|
|
6573
6696
|
evidenceGrade: "A",
|
|
6574
6697
|
tier: "scored",
|
|
6575
6698
|
dossier: "docs/evidence/audits/content-extraction/article-element.md",
|
|
6699
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6576
6700
|
applicablePageTypes: ["content"],
|
|
6577
6701
|
defaultPriority: "medium",
|
|
6578
6702
|
guidance: {
|
|
@@ -6631,6 +6755,7 @@ var HeaderFooterAudit = class extends Audit {
|
|
|
6631
6755
|
evidenceGrade: "A",
|
|
6632
6756
|
tier: "scored",
|
|
6633
6757
|
dossier: "docs/evidence/audits/content-extraction/header-footer.md",
|
|
6758
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6634
6759
|
defaultPriority: "medium",
|
|
6635
6760
|
guidance: {
|
|
6636
6761
|
impact: "AI agents use <header> and <footer> landmarks to identify and exclude boilerplate content (navigation menus, copyright notices, legal links) from primary content extraction. Without these landmarks, agents may include footer disclaimers or nav menus in their content summaries, reducing answer accuracy.",
|
|
@@ -6722,6 +6847,7 @@ var AsideElementAudit = class extends Audit {
|
|
|
6722
6847
|
evidenceGrade: "B",
|
|
6723
6848
|
tier: "scored",
|
|
6724
6849
|
dossier: "docs/evidence/audits/content-extraction/aside-element.md",
|
|
6850
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6725
6851
|
applicablePageTypes: ["content"],
|
|
6726
6852
|
defaultPriority: "low",
|
|
6727
6853
|
guidance: {
|
|
@@ -6804,6 +6930,7 @@ var SectionHeadingsAudit = class extends Audit {
|
|
|
6804
6930
|
evidenceGrade: "B",
|
|
6805
6931
|
tier: "scored",
|
|
6806
6932
|
dossier: "docs/evidence/audits/content-extraction/section-headings.md",
|
|
6933
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6807
6934
|
defaultPriority: "medium",
|
|
6808
6935
|
guidance: {
|
|
6809
6936
|
impact: "AI agents use section headings to build a topic map of your page for retrieval-augmented generation. Unlabeled <section> elements are opaque to AI chunking systems, preventing them from indexing and retrieving your content by topic, which reduces your visibility in AI-generated answers.",
|
|
@@ -6982,6 +7109,7 @@ var SemanticListsAudit = class extends Audit {
|
|
|
6982
7109
|
evidenceGrade: "B",
|
|
6983
7110
|
tier: "scored",
|
|
6984
7111
|
dossier: "docs/evidence/audits/content-extraction/semantic-lists.md",
|
|
7112
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6985
7113
|
defaultPriority: "medium",
|
|
6986
7114
|
guidance: {
|
|
6987
7115
|
impact: 'AI agents recognize <ul>, <ol>, and <dl> as structured lists and extract them as bullet points, numbered steps or term/definition pairs. Content formatted as styled <div> elements \u2014 or as paragraphs that start with "1.", "2." \u2014 collapses into undelimited prose when the page is converted to markdown or an accessibility tree, so the agent has to re-infer where each item begins.',
|
|
@@ -7053,6 +7181,7 @@ var DataTablesAudit = class extends Audit {
|
|
|
7053
7181
|
evidenceGrade: "B",
|
|
7054
7182
|
tier: "scored",
|
|
7055
7183
|
dossier: "docs/evidence/audits/content-extraction/data-tables.md",
|
|
7184
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7056
7185
|
defaultPriority: "medium",
|
|
7057
7186
|
guidance: {
|
|
7058
7187
|
impact: "AI agents rely on <thead> and <th> elements to understand column headers and map cell values to their meanings. Without proper table structure, agents cannot interpret tabular data correctly, leading to garbled comparisons and inaccurate data extraction in AI-generated summaries.",
|
|
@@ -7129,6 +7258,7 @@ var CodeLanguageAudit = class extends Audit {
|
|
|
7129
7258
|
evidenceGrade: "C",
|
|
7130
7259
|
tier: "informative",
|
|
7131
7260
|
dossier: "docs/evidence/audits/content-extraction/code-language.md",
|
|
7261
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7132
7262
|
applicablePageTypes: ["content"],
|
|
7133
7263
|
defaultPriority: "low",
|
|
7134
7264
|
guidance: {
|
|
@@ -7210,6 +7340,7 @@ var TimeElementAudit = class extends Audit {
|
|
|
7210
7340
|
evidenceGrade: "C",
|
|
7211
7341
|
tier: "informative",
|
|
7212
7342
|
dossier: "docs/evidence/audits/content-extraction/time-element.md",
|
|
7343
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7213
7344
|
applicablePageTypes: ["content"],
|
|
7214
7345
|
defaultPriority: "medium",
|
|
7215
7346
|
guidance: {
|
|
@@ -7260,6 +7391,7 @@ var ContentDepthAudit = class extends Audit {
|
|
|
7260
7391
|
evidenceGrade: "B",
|
|
7261
7392
|
tier: "scored",
|
|
7262
7393
|
dossier: "docs/evidence/audits/content-extraction/content-depth.md",
|
|
7394
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7263
7395
|
defaultPriority: "medium",
|
|
7264
7396
|
guidance: {
|
|
7265
7397
|
impact: "Pages with fewer than 300 words provide too little context for AI RAG systems to generate accurate, detailed answers. Thin content produces weak vector embeddings that rank poorly in retrieval, causing your pages to be excluded from AI-generated responses entirely.",
|
|
@@ -7342,6 +7474,7 @@ var ImageAltTextAudit = class extends Audit {
|
|
|
7342
7474
|
evidenceGrade: "A",
|
|
7343
7475
|
tier: "scored",
|
|
7344
7476
|
dossier: "docs/evidence/audits/content-extraction/image-alt-text.md",
|
|
7477
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7345
7478
|
defaultPriority: "high",
|
|
7346
7479
|
guidance: {
|
|
7347
7480
|
impact: "An image with no text alternative has no accessible name, so it is an unnamed node in the accessibility-tree snapshots agent toolkits send to a model \u2014 Playwright MCP, Claude-in-Chrome read_page, Chrome DevTools take_snapshot \u2014 and it carries no subject matter for Google Images, which states it uses alt text to understand what an image shows. A multimodal agent that fetches the image bytes can caption it without one; a text-only crawler or a snapshot-driven agent cannot.",
|
|
@@ -7428,6 +7561,7 @@ var FigureFigcaptionAudit = class extends Audit {
|
|
|
7428
7561
|
evidenceGrade: "C",
|
|
7429
7562
|
tier: "informative",
|
|
7430
7563
|
dossier: "docs/evidence/audits/content-extraction/figure-figcaption.md",
|
|
7564
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7431
7565
|
defaultPriority: "medium",
|
|
7432
7566
|
guidance: {
|
|
7433
7567
|
impact: "AI agents use <figcaption> to understand the purpose and context of visual content beyond what alt text provides. Without captions, figures are treated as opaque image containers, and your charts, diagrams, and illustrations cannot be meaningfully cited in AI-generated answers.",
|
|
@@ -7550,6 +7684,7 @@ var SvgBloatAudit = class extends Audit {
|
|
|
7550
7684
|
evidenceGrade: "B",
|
|
7551
7685
|
tier: "scored",
|
|
7552
7686
|
dossier: "docs/evidence/audits/content-extraction/svg-bloat.md",
|
|
7687
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7553
7688
|
defaultPriority: "medium",
|
|
7554
7689
|
guidance: {
|
|
7555
7690
|
impact: "Large inline SVGs are inlined verbatim as path-data tokens when an LLM converts your page to Markdown. A single 10KB icon or chart can consume thousands of tokens of agent context per page load, inflating agent cost and pushing real content out of the context window \u2014 reducing the quality of what agents extract and say about your site.",
|
|
@@ -12960,6 +13095,7 @@ var TokenRatioAudit = class extends Audit {
|
|
|
12960
13095
|
evidenceGrade: "B",
|
|
12961
13096
|
tier: "scored",
|
|
12962
13097
|
dossier: "docs/evidence/audits/content-extraction/token-ratio.md",
|
|
13098
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
12963
13099
|
defaultPriority: "high",
|
|
12964
13100
|
guidance: {
|
|
12965
13101
|
impact: "When less than 15% of your HTML is actual content, AI agents burn most of their context window and token budget on markup noise: inline scripts, CSS, SVG sprites, tracking tags, and deeply nested divs. The useful text that remains gets weaker attention from the model, and pages with extreme bloat may be truncated before the real content is even read.",
|
|
@@ -13091,6 +13227,7 @@ var FakeHeadingsAudit = class extends Audit {
|
|
|
13091
13227
|
evidenceGrade: "B",
|
|
13092
13228
|
tier: "scored",
|
|
13093
13229
|
dossier: "docs/evidence/audits/content-extraction/fake-headings.md",
|
|
13230
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
13094
13231
|
defaultPriority: "medium",
|
|
13095
13232
|
guidance: {
|
|
13096
13233
|
impact: "AI agents build content outlines exclusively from <h1>\u2013<h6> elements. Text that only looks like a heading is treated as ordinary body copy, so agents miss your section structure entirely \u2014 summaries flatten into a wall of text, section-level citations become impossible, and chunking for retrieval splits content at arbitrary points instead of at your intended section boundaries.",
|
|
@@ -13153,7 +13290,189 @@ var FakeHeadingsAudit = class extends Audit {
|
|
|
13153
13290
|
}
|
|
13154
13291
|
};
|
|
13155
13292
|
|
|
13293
|
+
// src/gatherers/domains.ts
|
|
13294
|
+
var MULTI_SUFFIX = /* @__PURE__ */ new Set([
|
|
13295
|
+
"co.uk",
|
|
13296
|
+
"org.uk",
|
|
13297
|
+
"ac.uk",
|
|
13298
|
+
"gov.uk",
|
|
13299
|
+
"me.uk",
|
|
13300
|
+
"net.uk",
|
|
13301
|
+
"com.au",
|
|
13302
|
+
"net.au",
|
|
13303
|
+
"org.au",
|
|
13304
|
+
"edu.au",
|
|
13305
|
+
"gov.au",
|
|
13306
|
+
"co.nz",
|
|
13307
|
+
"co.jp",
|
|
13308
|
+
"or.jp",
|
|
13309
|
+
"ne.jp",
|
|
13310
|
+
"co.za",
|
|
13311
|
+
"co.kr",
|
|
13312
|
+
"co.il",
|
|
13313
|
+
"co.id",
|
|
13314
|
+
"co.th",
|
|
13315
|
+
"com.br",
|
|
13316
|
+
"com.mx",
|
|
13317
|
+
"com.ar",
|
|
13318
|
+
"com.co",
|
|
13319
|
+
"com.pe",
|
|
13320
|
+
"co.in",
|
|
13321
|
+
"com.sg",
|
|
13322
|
+
"com.tr",
|
|
13323
|
+
"com.cn",
|
|
13324
|
+
"com.hk",
|
|
13325
|
+
"com.tw",
|
|
13326
|
+
"com.my",
|
|
13327
|
+
"com.ph",
|
|
13328
|
+
"com.ua",
|
|
13329
|
+
"com.pl",
|
|
13330
|
+
"com.es",
|
|
13331
|
+
"com.pt",
|
|
13332
|
+
"com.gr"
|
|
13333
|
+
]);
|
|
13334
|
+
function registrableDomain(host) {
|
|
13335
|
+
const parts = host.toLowerCase().replace(/\.$/, "").split(".").filter(Boolean);
|
|
13336
|
+
if (parts.length <= 2) return parts.join(".");
|
|
13337
|
+
const lastTwo = parts.slice(-2).join(".");
|
|
13338
|
+
return MULTI_SUFFIX.has(lastTwo) ? parts.slice(-3).join(".") : lastTwo;
|
|
13339
|
+
}
|
|
13340
|
+
function registrableOf(url) {
|
|
13341
|
+
try {
|
|
13342
|
+
return registrableDomain(new URL(url).hostname);
|
|
13343
|
+
} catch {
|
|
13344
|
+
return "";
|
|
13345
|
+
}
|
|
13346
|
+
}
|
|
13347
|
+
|
|
13348
|
+
// src/scan-evidence.ts
|
|
13349
|
+
var ALL_PAGE_TYPES = ["homepage", "category", "product", "content"];
|
|
13350
|
+
var HTML_TYPES = ["text/html", "application/xhtml+xml"];
|
|
13351
|
+
var PERMANENT_REDIRECT = /* @__PURE__ */ new Set([301, 308]);
|
|
13352
|
+
function bareHost(url) {
|
|
13353
|
+
try {
|
|
13354
|
+
return new URL(url).hostname.toLowerCase().replace(/^www\./, "");
|
|
13355
|
+
} catch {
|
|
13356
|
+
return "";
|
|
13357
|
+
}
|
|
13358
|
+
}
|
|
13359
|
+
function registrableName(url) {
|
|
13360
|
+
const domain = registrableOf(url);
|
|
13361
|
+
if (!domain) return "";
|
|
13362
|
+
const parts = domain.split(".");
|
|
13363
|
+
return parts.length > 1 ? parts.slice(0, -1).join(".") : domain;
|
|
13364
|
+
}
|
|
13365
|
+
function reachedTheRequestedSite(requestedUrl, result) {
|
|
13366
|
+
const requested = bareHost(requestedUrl);
|
|
13367
|
+
const final = bareHost(result.finalUrl || result.url);
|
|
13368
|
+
if (!final) return { ok: false, reason: `The homepage response carried no usable URL.` };
|
|
13369
|
+
if (requested === final) return { ok: true };
|
|
13370
|
+
const requestedDomain = registrableOf(requestedUrl);
|
|
13371
|
+
const finalDomain = registrableOf(result.finalUrl || result.url);
|
|
13372
|
+
if (requestedDomain && requestedDomain === finalDomain) return { ok: true };
|
|
13373
|
+
const requestedName = registrableName(requestedUrl);
|
|
13374
|
+
if (requestedName && requestedName === registrableName(result.finalUrl || result.url)) {
|
|
13375
|
+
return { ok: true };
|
|
13376
|
+
}
|
|
13377
|
+
const chain = result.redirectChain ?? [];
|
|
13378
|
+
const leaving = chain.filter((hop) => registrableOf(hop.from) !== registrableOf(hop.to));
|
|
13379
|
+
if (leaving.length > 0 && leaving.every((hop) => PERMANENT_REDIRECT.has(hop.status))) {
|
|
13380
|
+
return { ok: true };
|
|
13381
|
+
}
|
|
13382
|
+
return {
|
|
13383
|
+
ok: false,
|
|
13384
|
+
reason: `The requested host redirected to ${final}, a different site, without a permanent redirect.`
|
|
13385
|
+
};
|
|
13386
|
+
}
|
|
13387
|
+
function originReachable(requestedUrl, result) {
|
|
13388
|
+
if (result.error) {
|
|
13389
|
+
return { met: false, reason: `The homepage could not be fetched: ${result.error}.` };
|
|
13390
|
+
}
|
|
13391
|
+
if (result.status < 200 || result.status > 299) {
|
|
13392
|
+
return { met: false, reason: `The homepage answered HTTP ${result.status}.` };
|
|
13393
|
+
}
|
|
13394
|
+
const type = (result.contentType || "").toLowerCase();
|
|
13395
|
+
if (!HTML_TYPES.some((html) => type.includes(html))) {
|
|
13396
|
+
return {
|
|
13397
|
+
met: false,
|
|
13398
|
+
reason: `The homepage served ${result.contentType || "no content type"}, not HTML.`
|
|
13399
|
+
};
|
|
13400
|
+
}
|
|
13401
|
+
const reached = reachedTheRequestedSite(requestedUrl, result);
|
|
13402
|
+
return reached.ok ? { met: true } : { met: false, reason: reached.reason };
|
|
13403
|
+
}
|
|
13404
|
+
function unblockedFetches(homepageResult, waf) {
|
|
13405
|
+
if (waf?.isBlocked) {
|
|
13406
|
+
return waf.isRateLimit ? {
|
|
13407
|
+
met: false,
|
|
13408
|
+
reason: `The scan was throttled (${waf.name}): ${waf.reason}.`
|
|
13409
|
+
} : { met: false, reason: `${waf.name} refused the scan: ${waf.reason}.` };
|
|
13410
|
+
}
|
|
13411
|
+
if (homepageResult.status === 429) {
|
|
13412
|
+
return { met: false, reason: "The homepage answered HTTP 429: the scan was throttled." };
|
|
13413
|
+
}
|
|
13414
|
+
return { met: true };
|
|
13415
|
+
}
|
|
13416
|
+
function pageRendersText(page) {
|
|
13417
|
+
const text3 = getRenderedText(page.$);
|
|
13418
|
+
const wordCount2 = text3.split(/\s+/).filter(Boolean).length;
|
|
13419
|
+
return wordCount2 > 50 || text3.length > 200;
|
|
13420
|
+
}
|
|
13421
|
+
function buildScanEvidence(input) {
|
|
13422
|
+
const origin = originReachable(input.requestedUrl, input.homepageResult);
|
|
13423
|
+
const unblocked = unblockedFetches(input.homepageResult, input.wafProtection);
|
|
13424
|
+
const renderedByPage = {};
|
|
13425
|
+
const usablePageTypes = /* @__PURE__ */ new Set();
|
|
13426
|
+
for (const page of input.pages) {
|
|
13427
|
+
const rendered = pageRendersText(page);
|
|
13428
|
+
renderedByPage[page.url] = rendered;
|
|
13429
|
+
if (rendered) usablePageTypes.add(page.pageType);
|
|
13430
|
+
}
|
|
13431
|
+
const renderedCount = Object.values(renderedByPage).filter(Boolean).length;
|
|
13432
|
+
const met = {
|
|
13433
|
+
"origin-reachable": origin.met,
|
|
13434
|
+
"unblocked-fetches": unblocked.met,
|
|
13435
|
+
"rendered-body": renderedCount > 0,
|
|
13436
|
+
"sample-adequate": usablePageTypes.size > 0
|
|
13437
|
+
};
|
|
13438
|
+
const reasons = {};
|
|
13439
|
+
if (origin.reason) reasons["origin-reachable"] = origin.reason;
|
|
13440
|
+
if (unblocked.reason) reasons["unblocked-fetches"] = unblocked.reason;
|
|
13441
|
+
if (!met["rendered-body"]) {
|
|
13442
|
+
reasons["rendered-body"] = input.pages.length === 0 ? "The scan fetched no pages." : `None of the ${input.pages.length} fetched page(s) served readable text.`;
|
|
13443
|
+
}
|
|
13444
|
+
if (!met["sample-adequate"]) {
|
|
13445
|
+
reasons["sample-adequate"] = input.pages.length === 0 ? "The scan fetched no pages." : "No fetched page of any type served readable text.";
|
|
13446
|
+
}
|
|
13447
|
+
return {
|
|
13448
|
+
met,
|
|
13449
|
+
reasons,
|
|
13450
|
+
renderedByPage,
|
|
13451
|
+
usablePageTypes,
|
|
13452
|
+
// A shell site was seen. What it serves is a finding about it, so
|
|
13453
|
+
// `rendered-body` and `sample-adequate` do not clear `judgeable`.
|
|
13454
|
+
judgeable: met["origin-reachable"] && met["unblocked-fetches"]
|
|
13455
|
+
};
|
|
13456
|
+
}
|
|
13457
|
+
function allEvidenceMet() {
|
|
13458
|
+
return {
|
|
13459
|
+
met: {
|
|
13460
|
+
"origin-reachable": true,
|
|
13461
|
+
"unblocked-fetches": true,
|
|
13462
|
+
"rendered-body": true,
|
|
13463
|
+
"sample-adequate": true
|
|
13464
|
+
},
|
|
13465
|
+
reasons: {},
|
|
13466
|
+
renderedByPage: {},
|
|
13467
|
+
usablePageTypes: new Set(ALL_PAGE_TYPES),
|
|
13468
|
+
judgeable: true
|
|
13469
|
+
};
|
|
13470
|
+
}
|
|
13471
|
+
|
|
13156
13472
|
// src/audits/content-extraction/server-rendered.ts
|
|
13473
|
+
function withDetails(result, details) {
|
|
13474
|
+
return { ...result, details: { ...result.details ?? {}, ...details } };
|
|
13475
|
+
}
|
|
13157
13476
|
var ServerRenderedAudit = class extends Audit {
|
|
13158
13477
|
static meta = {
|
|
13159
13478
|
id: "content-extraction/server-rendered",
|
|
@@ -13166,6 +13485,8 @@ var ServerRenderedAudit = class extends Audit {
|
|
|
13166
13485
|
evidenceGrade: "B",
|
|
13167
13486
|
tier: "scored",
|
|
13168
13487
|
dossier: "docs/evidence/audits/content-extraction/server-rendered.md",
|
|
13488
|
+
// Gate exemption: A shell is what this audit reports. Gating it would delete the finding.
|
|
13489
|
+
requires: ["origin-reachable", "unblocked-fetches"],
|
|
13169
13490
|
defaultPriority: "critical",
|
|
13170
13491
|
guidance: {
|
|
13171
13492
|
impact: "AI crawlers (GPTBot, ClaudeBot, PerplexityBot) do not execute JavaScript. If your content is only rendered client-side, these crawlers see an empty or near-empty page. Your products, articles, and brand information are completely absent from AI knowledge bases, meaning AI-generated answers never reference your site.",
|
|
@@ -13177,37 +13498,57 @@ var ServerRenderedAudit = class extends Audit {
|
|
|
13177
13498
|
}
|
|
13178
13499
|
};
|
|
13179
13500
|
audit(ctx) {
|
|
13180
|
-
const
|
|
13181
|
-
if (
|
|
13182
|
-
return this.
|
|
13183
|
-
"
|
|
13184
|
-
"
|
|
13185
|
-
"No
|
|
13186
|
-
|
|
13187
|
-
|
|
13501
|
+
const pages = ctx.pages ?? [];
|
|
13502
|
+
if (pages.length === 0) {
|
|
13503
|
+
return this.notApplicable(
|
|
13504
|
+
"The scan fetched no page, so there is no served HTML to judge.",
|
|
13505
|
+
"Every fetched page serves > 50 words or > 200 characters of readable text",
|
|
13506
|
+
"No page fetched"
|
|
13507
|
+
);
|
|
13508
|
+
}
|
|
13509
|
+
const rendered = ctx.evidence.renderedByPage;
|
|
13510
|
+
const emptyPages = pages.filter((page) => !(rendered[page.url] ?? pageRendersText(page))).map((page) => page.url);
|
|
13511
|
+
const total = pages.length;
|
|
13512
|
+
const renderedCount = total - emptyPages.length;
|
|
13513
|
+
const expected = "Every fetched page serves > 50 words or > 200 characters of readable text";
|
|
13514
|
+
const found = `${renderedCount} of ${total} page(s) served readable text`;
|
|
13515
|
+
if (emptyPages.length === 0) {
|
|
13516
|
+
return withDetails(
|
|
13517
|
+
this.pass(
|
|
13518
|
+
`All ${total} fetched page(s) serve their content in the HTML response.`,
|
|
13519
|
+
expected,
|
|
13520
|
+
found,
|
|
13521
|
+
pages[0].url
|
|
13522
|
+
),
|
|
13523
|
+
{ pagesChecked: total, renderedPages: renderedCount }
|
|
13188
13524
|
);
|
|
13189
13525
|
}
|
|
13190
|
-
const
|
|
13191
|
-
|
|
13192
|
-
|
|
13193
|
-
|
|
13194
|
-
|
|
13195
|
-
|
|
13196
|
-
|
|
13197
|
-
|
|
13198
|
-
|
|
13526
|
+
const failGuidance = {
|
|
13527
|
+
priority: "critical",
|
|
13528
|
+
description: "AI crawlers like GPTBot and ClaudeBot do not execute JavaScript. Content only visible after JS execution is completely invisible to them, meaning your site effectively has no content in AI knowledge bases. Use SSR (server-side rendering) or SSG (static site generation) to serve content in the initial HTML response.",
|
|
13529
|
+
code: "// Next.js SSR example:\nexport async function getServerSideProps() {\n const data = await fetchData();\n return { props: { data } };\n}"
|
|
13530
|
+
};
|
|
13531
|
+
if (renderedCount === 0) {
|
|
13532
|
+
return withDetails(
|
|
13533
|
+
this.fail(
|
|
13534
|
+
`None of the ${total} fetched page(s) serve readable content in the HTML response. AI agents cannot read client-side-only rendered content.`,
|
|
13535
|
+
expected,
|
|
13536
|
+
found,
|
|
13537
|
+
failGuidance,
|
|
13538
|
+
pages[0].url
|
|
13539
|
+
),
|
|
13540
|
+
{ pagesChecked: total, renderedPages: 0, emptyPages }
|
|
13199
13541
|
);
|
|
13200
13542
|
}
|
|
13201
|
-
return
|
|
13202
|
-
|
|
13203
|
-
|
|
13204
|
-
|
|
13205
|
-
|
|
13206
|
-
|
|
13207
|
-
|
|
13208
|
-
|
|
13209
|
-
}
|
|
13210
|
-
page.url
|
|
13543
|
+
return withDetails(
|
|
13544
|
+
this.warn(
|
|
13545
|
+
`${emptyPages.length} of ${total} fetched page(s) serve no readable content in the HTML response. AI agents read nothing on those pages.`,
|
|
13546
|
+
expected,
|
|
13547
|
+
found,
|
|
13548
|
+
failGuidance,
|
|
13549
|
+
emptyPages[0]
|
|
13550
|
+
),
|
|
13551
|
+
{ pagesChecked: total, renderedPages: renderedCount, emptyPages }
|
|
13211
13552
|
);
|
|
13212
13553
|
}
|
|
13213
13554
|
};
|
|
@@ -13452,6 +13793,7 @@ var CssHiddenGhostContentAudit = class _CssHiddenGhostContentAudit extends Audit
|
|
|
13452
13793
|
evidenceGrade: "A",
|
|
13453
13794
|
tier: "scored",
|
|
13454
13795
|
dossier: "docs/evidence/audits/content-extraction/css-hidden-ghost-content.md",
|
|
13796
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
13455
13797
|
defaultPriority: "medium",
|
|
13456
13798
|
guidance: {
|
|
13457
13799
|
impact: "This is provable from source, not inferred. Readability's visibility test consults only node.style.display, node.style.visibility, the hidden attribute and aria-hidden \u2014 it explicitly does not evaluate class-based CSS rules from stylesheets. AI crawlers do not render, so no cascade is ever computed. Therefore any subtree hidden by `.mobile-only{display:none}`, `.tab-panel:not(.active){display:none}` or `[data-state=closed]{display:none}` reaches the model as ordinary body text with full weight. Consequence is not just cost: the agent sees three parallel copies of a nav, both the collapsed and expanded FAQ answers, and often stale price text from a hidden variant block, and irrelevant/contradictory context measurably degrades answers.",
|
|
@@ -13630,6 +13972,7 @@ var HydrationPayloadShareAudit = class _HydrationPayloadShareAudit extends Audit
|
|
|
13630
13972
|
evidenceGrade: "A",
|
|
13631
13973
|
tier: "scored",
|
|
13632
13974
|
dossier: "docs/evidence/audits/content-extraction/hydration-payload-share.md",
|
|
13975
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
13633
13976
|
defaultPriority: "medium",
|
|
13634
13977
|
guidance: {
|
|
13635
13978
|
impact: "These blobs are inlined into every HTML response by design, and the framework vendor itself flags > 128 kB as a defect. A browser parses them and throws them away after hydration; a non-rendering AI crawler cannot \u2014 it tokenizes the JSON verbatim, including escaped HTML, CDN image variants, GraphQL type metadata and the full body text a second time. The causal claim is falsifiable per page: strip these script nodes, re-tokenize, and the delta is the exact context cost that carries zero incremental information, since duplicate #3 is byte-identical content the agent already has.",
|
|
@@ -13784,6 +14127,7 @@ var PreambleTaxTokensBeforeTheFirstContentTokenAudit = class extends Audit {
|
|
|
13784
14127
|
weight: weightForGrade("B", "scored"),
|
|
13785
14128
|
defaultPriority: "medium",
|
|
13786
14129
|
dossier: "docs/evidence/audits/content-extraction/preamble-tax.md",
|
|
14130
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
13787
14131
|
guidance: {
|
|
13788
14132
|
impact: "A non-rendering agent ingests the document as a linear stream, so DOM order is context order. A page that inlines a critical-CSS block and a serialized state blob ahead of its content does two things at once: it pushes the answer into the middle of the context window, where retrieval is measurably weakest, and it guarantees the answer is what gets cut when the fetching harness truncates to a byte or token cap.",
|
|
13789
14133
|
fix: "Move inline `<style>` and `<script>` blocks below the main content or into external files, and put `<main>` as early in the body as the layout allows. Where critical CSS must be inline, keep it to the rules that paint the first screen rather than the whole stylesheet.",
|
|
@@ -13928,6 +14272,7 @@ var BoilerplateTaxAudit = class extends Audit {
|
|
|
13928
14272
|
weight: weightForGrade("B", "scored"),
|
|
13929
14273
|
defaultPriority: "medium",
|
|
13930
14274
|
dossier: "docs/evidence/audits/content-extraction/boilerplate-tax.md",
|
|
14275
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
13931
14276
|
guidance: {
|
|
13932
14277
|
impact: "An agent answering a question about a site fetches several of its pages. If each fetch delivers the same navigation, the same promotional header and the same footer around a thin body, the agent pays for those tokens once per fetch and learns nothing new from them. The cost compounds with every page, and the distinct content it came for competes for what is left of the context window.",
|
|
13933
14278
|
fix: "Cut repeated chrome down to what a reader needs on every page: collapse mega-menus to a short nav, move legal and marketing boilerplate to the pages that are about it, and let each page carry more of its own content. Where the chrome must stay for humans, keeping it out of `<main>` at least lets an extractor drop it.",
|
|
@@ -14053,6 +14398,7 @@ var ExtractionDeterminismAudit = class extends Audit {
|
|
|
14053
14398
|
weight: weightForGrade("B", "scored"),
|
|
14054
14399
|
defaultPriority: "high",
|
|
14055
14400
|
dossier: "docs/evidence/audits/content-extraction/extraction-determinism.md",
|
|
14401
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
14056
14402
|
guidance: {
|
|
14057
14403
|
impact: "Every agent pipeline strips a page down before a model reads it, and they do not all strip the same way. When the extractors disagree, the same URL yields different answers depending on which tool fetched it \u2014 and the page cannot be tested, because there is no single thing it says. When readability declines a page outright, the most widely deployed extractor of the three hands an agent nothing at all.",
|
|
14058
14404
|
fix: "Put the article in one container \u2014 `<main>` or `<article>` \u2014 with the chrome outside it, and keep the largest block of prose on the page the one you want quoted. Readability keys on paragraph density and link density, so a body split across many small wrappers, or padded with link-heavy blocks, is what makes the three disagree.",
|
|
@@ -14230,6 +14576,7 @@ var LlmsTxtExistsAudit = class extends Audit {
|
|
|
14230
14576
|
evidenceGrade: "C",
|
|
14231
14577
|
tier: "informative",
|
|
14232
14578
|
dossier: "docs/evidence/audits/machine-discovery/llms-txt-exists.md",
|
|
14579
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
14233
14580
|
defaultPriority: "low",
|
|
14234
14581
|
guidance: {
|
|
14235
14582
|
impact: "Thousands of sites publish an llms.txt, including every major AI lab, but as publishers rather than readers. No vendor documentation names an agent that fetches it, and Google Search Central states Search ignores it. Publishing one is cheap and harmless; it is not a documented path to any AI answer.",
|
|
@@ -14310,6 +14657,7 @@ var LlmsTxtStructureAudit = class extends Audit {
|
|
|
14310
14657
|
evidenceGrade: "C",
|
|
14311
14658
|
tier: "informative",
|
|
14312
14659
|
dossier: "docs/evidence/audits/machine-discovery/llms-txt-structure.md",
|
|
14660
|
+
requires: ["origin-reachable"],
|
|
14313
14661
|
defaultPriority: "low",
|
|
14314
14662
|
guidance: {
|
|
14315
14663
|
impact: "The reference llms.txt parser extracts the blockquote as a `summary` field and the H2 headings as a `sections` map, so a file that carries both is machine-navigable: an agent can read the summary and pick a section instead of consuming the whole file. No vendor documents an agent behaving differently when either element is absent, so this is reported, not scored.",
|
|
@@ -14373,6 +14721,7 @@ var LlmsTxtLinkDescriptionsAudit = class extends Audit {
|
|
|
14373
14721
|
evidenceGrade: "C",
|
|
14374
14722
|
tier: "informative",
|
|
14375
14723
|
dossier: "docs/evidence/audits/machine-discovery/llms-txt-link-descriptions.md",
|
|
14724
|
+
requires: ["origin-reachable"],
|
|
14376
14725
|
defaultPriority: "medium",
|
|
14377
14726
|
guidance: {
|
|
14378
14727
|
impact: "Links without descriptions force AI agents to visit every page to understand its content, wasting crawl budget and slowing down response generation. Described links let agents filter relevant pages instantly.",
|
|
@@ -14468,6 +14817,7 @@ var LlmsTxtLinksValidAudit = class extends Audit {
|
|
|
14468
14817
|
evidenceGrade: "C",
|
|
14469
14818
|
tier: "informative",
|
|
14470
14819
|
dossier: "docs/evidence/audits/machine-discovery/llms-txt-links-valid.md",
|
|
14820
|
+
requires: ["origin-reachable"],
|
|
14471
14821
|
defaultPriority: "low",
|
|
14472
14822
|
guidance: {
|
|
14473
14823
|
impact: "A broken link inside llms.txt points at nothing, the same as a broken link anywhere else. No documented agent consumer reads the file, so the cost is to any human or tool that follows it, not to a measured AI outcome.",
|
|
@@ -14549,6 +14899,7 @@ var LlmsFullTxtAudit = class extends Audit {
|
|
|
14549
14899
|
evidenceGrade: "C",
|
|
14550
14900
|
tier: "informative",
|
|
14551
14901
|
dossier: "docs/evidence/audits/machine-discovery/llms-full-txt.md",
|
|
14902
|
+
requires: ["origin-reachable"],
|
|
14552
14903
|
defaultPriority: "high",
|
|
14553
14904
|
guidance: {
|
|
14554
14905
|
impact: "Without llms-full.txt, AI agents must crawl your site page by page, which is slow and often incomplete. This means AI assistants give shallow or outdated answers about your products and services.",
|
|
@@ -14617,6 +14968,7 @@ var SitemapExistsAudit = class extends Audit {
|
|
|
14617
14968
|
evidenceGrade: "A",
|
|
14618
14969
|
tier: "scored",
|
|
14619
14970
|
dossier: "docs/evidence/audits/machine-discovery/sitemap-exists.md",
|
|
14971
|
+
requires: ["origin-reachable"],
|
|
14620
14972
|
defaultPriority: "critical",
|
|
14621
14973
|
guidance: {
|
|
14622
14974
|
impact: "Without a sitemap, AI crawlers must discover your pages solely through link-following, which is slow and incomplete. Pages deep in your site hierarchy may never be found, meaning AI search engines like Perplexity and ChatGPT Browse cannot surface your full content.",
|
|
@@ -14756,6 +15108,7 @@ var DiscoveryIndexCoverageAudit = class extends Audit {
|
|
|
14756
15108
|
evidenceGrade: "B",
|
|
14757
15109
|
tier: "scored",
|
|
14758
15110
|
dossier: "docs/evidence/audits/machine-discovery/discovery-index-coverage.md",
|
|
15111
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
14759
15112
|
defaultPriority: "medium",
|
|
14760
15113
|
guidance: {
|
|
14761
15114
|
impact: "A page listed in no discovery index is reachable only through the link graph, and the major AI crawlers do not execute JavaScript \u2014 so a page missing from both the sitemap and llms.txt can stay invisible to AI search even though it exists on your site.",
|
|
@@ -14888,6 +15241,7 @@ var SitemapAbsoluteUrlsAudit = class extends Audit {
|
|
|
14888
15241
|
evidenceGrade: "B",
|
|
14889
15242
|
tier: "scored",
|
|
14890
15243
|
dossier: "docs/evidence/audits/machine-discovery/sitemap-absolute-urls.md",
|
|
15244
|
+
requires: ["origin-reachable"],
|
|
14891
15245
|
defaultPriority: "high",
|
|
14892
15246
|
guidance: {
|
|
14893
15247
|
impact: "Relative URLs in your sitemap cannot be resolved by AI crawlers, causing them to silently skip those pages. Any page listed with a relative URL is effectively invisible to AI search engines.",
|
|
@@ -14992,6 +15346,7 @@ var SitemapLastmodAudit = class extends Audit {
|
|
|
14992
15346
|
evidenceGrade: "A",
|
|
14993
15347
|
tier: "scored",
|
|
14994
15348
|
dossier: "docs/evidence/audits/machine-discovery/sitemap-lastmod.md",
|
|
15349
|
+
requires: ["origin-reachable"],
|
|
14995
15350
|
defaultPriority: "medium",
|
|
14996
15351
|
guidance: {
|
|
14997
15352
|
impact: "Without <lastmod> dates, AI crawlers must re-fetch every page on every visit because they cannot tell which pages have changed. This wastes crawl budget and delays indexing of your freshest content.",
|
|
@@ -15128,6 +15483,7 @@ var RssFeedAudit = class extends Audit {
|
|
|
15128
15483
|
evidenceGrade: "B",
|
|
15129
15484
|
tier: "scored",
|
|
15130
15485
|
dossier: "docs/evidence/audits/machine-discovery/rss-feed.md",
|
|
15486
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15131
15487
|
defaultPriority: "medium",
|
|
15132
15488
|
guidance: {
|
|
15133
15489
|
impact: "Without an RSS/Atom feed, AI agents have no efficient way to track new and updated content on your site. They must re-crawl your entire site to find changes, which means your latest posts and pages may take much longer to appear in AI search results.",
|
|
@@ -15216,6 +15572,7 @@ var RssFeedContentAudit = class extends Audit {
|
|
|
15216
15572
|
evidenceGrade: "C",
|
|
15217
15573
|
tier: "informative",
|
|
15218
15574
|
dossier: "docs/evidence/audits/machine-discovery/rss-feed-content.md",
|
|
15575
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15219
15576
|
defaultPriority: "medium",
|
|
15220
15577
|
guidance: {
|
|
15221
15578
|
impact: "Truncated RSS feed items force AI agents to visit each page individually, increasing crawl time and often resulting in incomplete indexing. Full-content feeds let agents ingest all your articles in a single request, producing richer AI-generated answers.",
|
|
@@ -15371,6 +15728,7 @@ var InContentLinksAudit = class extends Audit {
|
|
|
15371
15728
|
evidenceGrade: "A",
|
|
15372
15729
|
tier: "scored",
|
|
15373
15730
|
dossier: "docs/evidence/audits/machine-discovery/in-content-links.md",
|
|
15731
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15374
15732
|
defaultPriority: "medium",
|
|
15375
15733
|
guidance: {
|
|
15376
15734
|
impact: "Google can only crawl a link that is an <a> element with an href, and the measured behaviour of GPTBot and ClaudeBot is that they do not execute JavaScript \u2014 so a page whose only links are in a client-rendered nav is a dead end for them. Links inside the body copy also tell an agent which pages belong together, which template chrome (identical on every page) cannot.",
|
|
@@ -15447,6 +15805,7 @@ var NoBrokenLinksAudit = class _NoBrokenLinksAudit extends Audit {
|
|
|
15447
15805
|
evidenceGrade: "A",
|
|
15448
15806
|
tier: "scored",
|
|
15449
15807
|
dossier: "docs/evidence/audits/machine-discovery/no-broken-links.md",
|
|
15808
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15450
15809
|
defaultPriority: "high",
|
|
15451
15810
|
guidance: {
|
|
15452
15811
|
impact: "Broken internal links waste AI crawlers' limited crawl budget by sending them to dead ends. This means fewer of your pages get indexed, and users asking AI about your site may encounter errors or missing information.",
|
|
@@ -15547,6 +15906,7 @@ var CorsAiFilesAudit = class extends Audit {
|
|
|
15547
15906
|
evidenceGrade: "C",
|
|
15548
15907
|
tier: "informative",
|
|
15549
15908
|
dossier: "docs/evidence/audits/machine-discovery/cors-ai-files.md",
|
|
15909
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15550
15910
|
defaultPriority: "medium",
|
|
15551
15911
|
guidance: {
|
|
15552
15912
|
impact: "Browser-based AI tools, ChatGPT plugins, and MCP clients all run in browser contexts governed by the same-origin policy. Without CORS headers on your llms.txt and AI catalog, these agents receive a network error instead of your content \u2014 making your AI-facing files completely invisible to the fastest-growing category of AI consumers.",
|
|
@@ -15668,6 +16028,7 @@ var AiFileDeliveryAudit = class extends Audit {
|
|
|
15668
16028
|
evidenceGrade: "B",
|
|
15669
16029
|
tier: "informative",
|
|
15670
16030
|
dossier: "docs/evidence/audits/machine-discovery/ai-file-delivery.md",
|
|
16031
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15671
16032
|
defaultPriority: "medium",
|
|
15672
16033
|
guidance: {
|
|
15673
16034
|
impact: "Incorrect Content-Type headers cause AI agents to misparse your files: JSON served as text/html breaks structured-data extraction, an XML sitemap served as text/plain hides it from crawl discovery, and llms.txt served as application/octet-stream triggers a download instead of a read. Missing caching headers make every agent re-download the full file on each visit rather than revalidating it.",
|
|
@@ -15761,6 +16122,7 @@ var NoBrokenAiEndpointsAudit = class extends Audit {
|
|
|
15761
16122
|
evidenceGrade: "A",
|
|
15762
16123
|
tier: "scored",
|
|
15763
16124
|
dossier: "docs/evidence/audits/machine-discovery/no-broken-ai-endpoints.md",
|
|
16125
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15764
16126
|
defaultPriority: "high",
|
|
15765
16127
|
guidance: {
|
|
15766
16128
|
impact: "Broken URLs in your AI manifest files (ai-catalog.json, llms.txt, navigation.json) cause agents to lose trust in your entire manifest. After encountering broken links, AI systems may stop following any of your listed endpoints, effectively making all your AI-facing resources undiscoverable.",
|
|
@@ -16030,6 +16392,7 @@ var AiCrawlerSurfaceReachabilityAudit = class extends Audit {
|
|
|
16030
16392
|
evidenceGrade: "A",
|
|
16031
16393
|
tier: "scored",
|
|
16032
16394
|
dossier: "docs/evidence/audits/machine-discovery/ai-crawler-surface-reachability.md",
|
|
16395
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
16033
16396
|
defaultPriority: "high",
|
|
16034
16397
|
guidance: {
|
|
16035
16398
|
impact: "The Sitemap: directive is host-global and user-agent independent (RFC 9309 \xA72.2.3), but the sitemap file, the feed files and every URL they list obey per-crawler rules \u2014 and under \xA72.2.1 a crawler with a named group ignores the '*' group entirely. OpenAI documents the consequence at the extreme: 'Sites that are opted out of OAI-SearchBot will not be shown in ChatGPT search answers.' So for any crawler whose named group disallows the advertised sitemap or feed path, or a majority of the URLs the sitemap lists, the site's whole pull-indexing surface is unreachable to that agent no matter how good the sitemap is. The common trigger is a bot-blocking plugin adding a broad pattern (Disallow: /*.xml$, Disallow: /feed/, Disallow: /) to an AI-bot group while the site keeps advertising those exact paths.",
|
|
@@ -16215,6 +16578,7 @@ var SitemapLastmodVerifiabilityAudit = class extends Audit {
|
|
|
16215
16578
|
evidenceGrade: "A",
|
|
16216
16579
|
tier: "scored",
|
|
16217
16580
|
dossier: "docs/evidence/audits/machine-discovery/sitemap-lastmod-verifiability.md",
|
|
16581
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
16218
16582
|
defaultPriority: "medium",
|
|
16219
16583
|
guidance: {
|
|
16220
16584
|
impact: `Google states it uses <lastmod> "if it's consistently and verifiably (for example by comparing to the last modification of the page) accurate". lastmod is therefore a conditional signal an engine silently discards on divergence \u2014 and it is the only freshness hint a pull-based AI crawler gets from a sitemap. If sampled values disagree with every available page-level signal for a material share of URLs, the freshness channel is inert and re-crawl scheduling degrades to organic rediscovery. Two specific pathologies are detectable without guessing: over 90% of URLs sharing one lastmod equal to the last deploy date \u2014 a build stamp, exactly the pattern Google's "copyright date is not significant" rule disqualifies \u2014 and a lastmod in the future relative to the scan, which is never valid.`,
|
|
@@ -16596,6 +16960,7 @@ var CheckoutOfferFieldMappingAudit = class _CheckoutOfferFieldMappingAudit exten
|
|
|
16596
16960
|
evidenceGrade: "A",
|
|
16597
16961
|
tier: "scored",
|
|
16598
16962
|
dossier: "docs/evidence/audits/agentic-commerce/checkout-offer-field-mapping.md",
|
|
16963
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
16599
16964
|
applicablePageTypes: ["product"],
|
|
16600
16965
|
defaultPriority: "high",
|
|
16601
16966
|
guidance: {
|
|
@@ -16792,6 +17157,7 @@ var AgentCommerceFeedParityAudit = class extends Audit {
|
|
|
16792
17157
|
evidenceGrade: "A",
|
|
16793
17158
|
tier: "scored",
|
|
16794
17159
|
dossier: "docs/evidence/audits/machine-discovery/agent-commerce-feed-parity.md",
|
|
17160
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
16795
17161
|
defaultPriority: "high",
|
|
16796
17162
|
guidance: {
|
|
16797
17163
|
impact: `Google's automatic item updates repair feed/page discrepancies "using the structured data markup the crawlers find on your website", and state that where extractors cannot determine price, availability and condition, "your products will be subject to item-level disapprovals". Merchant Center separately requires that feed availability match the landing page and that price match the landing page and checkout. OpenAI's Product Feed Spec requires a strictly larger per-item set than Google's rich-result minimum: a stable item_id (<=100 chars), brand (<=70), seller_name, target_countries as ISO 3166-1 alpha-2, a plain-text description under 5000 characters, availability from a fixed enum, and price with an ISO 4217 currency. Falsifiable claim: a PDP missing brand, seller, itemCondition-as-URL, a stable SKU or a country signal passes every Google rich-result test yet cannot be reconciled by automatic item updates, so feed rejections are silent and unattributable. Second claim, sharper: where the JSON-LD price disagrees with the price the page renders, automatic item updates overwrite the feed with one value while an agent reading the page quotes the other.`,
|
|
@@ -17332,6 +17698,7 @@ var ConditionalRequestSupportAudit = class extends Audit {
|
|
|
17332
17698
|
weight: weightForGrade("B", "scored"),
|
|
17333
17699
|
defaultPriority: "medium",
|
|
17334
17700
|
dossier: "docs/evidence/audits/machine-discovery/conditional-request-support.md",
|
|
17701
|
+
requires: ["origin-reachable"],
|
|
17335
17702
|
guidance: {
|
|
17336
17703
|
impact: 'A crawler that wants to know what changed re-reads your sitemap and your feed on a schedule. If those responses carry no `ETag` and no `Last-Modified`, it cannot ask "has this changed?" \u2014 it can only download the file again, every time, forever. The cost is yours as much as theirs: bandwidth you serve for no new information, and a crawl budget spent re-reading a list instead of fetching the pages on it. A validator that changes on every build is the same cost wearing a correct-looking header.',
|
|
17337
17704
|
fix: "Emit a strong `ETag` derived from the file\u2019s content, not from the build, and a `Last-Modified` that moves only when the content does. Answer `If-None-Match` and `If-Modified-Since` with 304 and an empty body. Keep `no-store` and `private` off public discovery surfaces \u2014 they tell a crawler not to keep the copy it just paid for.",
|
|
@@ -17485,6 +17852,7 @@ var FeedEntryIdentityAndCanonicalIntegrityAudit = class extends Audit {
|
|
|
17485
17852
|
weight: weightForGrade("B", "scored"),
|
|
17486
17853
|
defaultPriority: "medium",
|
|
17487
17854
|
dossier: "docs/evidence/audits/machine-discovery/feed-entry-identity-and-canonical-integrity.md",
|
|
17855
|
+
requires: ["origin-reachable"],
|
|
17488
17856
|
guidance: {
|
|
17489
17857
|
impact: 'A feed is how a consumer tracks what changed without re-crawling the site, and identity is what makes that possible: the id says "this is the same item you saw last time". An entry with no id, or with an id that repeats, forces the consumer to guess \u2014 usually by URL, which is exactly the thing that changes. A link that carries `utm_` parameters or redirects somewhere else creates a second address for one page, so the item the consumer stores is not the page the site considers canonical.',
|
|
17490
17858
|
fix: "Give every entry a stable id \u2014 an `atom:id` that never changes, or an RSS `<guid>` that is an absolute URL when `isPermaLink` is true \u2014 and never reuse one. Point item links at the canonical URL itself, with no tracking parameters and no redirect in between. Serve the feed as its registered media type, with no byte-order mark before the first element.",
|
|
@@ -17664,6 +18032,7 @@ var RootTextFileResolutionIntegrityAudit = class extends Audit {
|
|
|
17664
18032
|
weight: weightForGrade("B", "scored"),
|
|
17665
18033
|
defaultPriority: "medium",
|
|
17666
18034
|
dossier: "docs/evidence/audits/machine-discovery/root-text-file-resolution-integrity.md",
|
|
18035
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
17667
18036
|
guidance: {
|
|
17668
18037
|
impact: "IndexNow proves ownership by fetching `https://host/{key}.txt` and byte-comparing the body to the key, and six engines discard the submission when that comparison fails. The same property decides whether any other root `.txt` file means anything: if an origin answers 200 for a path that does not exist, then a 200 for `/llms.txt` is not evidence the file is there. A catch-all rewrite ahead of static file serving turns every one of those signals into noise, with no visible symptom on the site itself.",
|
|
17669
18038
|
fix: "Serve root-level `.txt` paths from static files and let a missing one answer 404. Order the static-file handler ahead of any SPA or catch-all rewrite, and make sure the rewrite does not cover `*.txt`. Serve `/robots.txt` as `text/plain`, not as `text/html` or `application/octet-stream`.",
|
|
@@ -17839,6 +18208,7 @@ var ThreeWayFreshnessLagAudit = class extends Audit {
|
|
|
17839
18208
|
weight: weightForGrade("B", "scored"),
|
|
17840
18209
|
defaultPriority: "medium",
|
|
17841
18210
|
dossier: "docs/evidence/audits/machine-discovery/three-way-freshness-lag.md",
|
|
18211
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
17842
18212
|
guidance: {
|
|
17843
18213
|
impact: "A pull-based crawler fetches the sitemap and the feed on a schedule and reads nothing else. When those two surfaces trail the site, everything published in between is discoverable only by link-following, which is the slow path the site published a sitemap to avoid. A feed whose `lastBuildDate` is older than its own newest item is worse than stale: consumers that poll conditionally on that timestamp skip the feed entirely, so the new items are never read at all.",
|
|
17844
18214
|
fix: "Regenerate the sitemap and the feed when content changes, not on a nightly cron that can fail silently. Stamp `<lastBuildDate>` (or the Atom feed-level `<updated>`) from the newest item at generation time. Order feed items newest-first, since many consumers read only the head. Remove sitemap entries whose URLs 404 or are noindex.",
|
|
@@ -17993,6 +18363,7 @@ var WebsubHubAdvertisementAudit = class extends Audit {
|
|
|
17993
18363
|
weight: 0,
|
|
17994
18364
|
defaultPriority: "low",
|
|
17995
18365
|
dossier: "docs/evidence/audits/machine-discovery/websub-hub-advertisement.md",
|
|
18366
|
+
requires: ["origin-reachable"],
|
|
17996
18367
|
guidance: {
|
|
17997
18368
|
impact: "A hub subscription is verified against the feed\u2019s own `rel=self`. When that link is missing, relative, or points at a different URL than the one the feed is served from, verification cannot complete, and the push path degrades to whatever polling cadence subscribers happen to use. The publisher sees a hub that looks configured and no error anywhere. The benefit side is unproven: WebSub is a W3C Recommendation, but no AI answer engine is documented as a subscriber, which is why this audit reports and does not score.",
|
|
17998
18369
|
fix: "Advertise the hub and the canonical topic URL in the feed\u2019s `Link:` response headers, which is where a subscriber looks first. Emit exactly one `rel=self` with an absolute URL identical to the address the feed is served from, and at least one `rel=hub` over HTTPS. If you run no hub, a hosted one (Google\u2019s pubsubhubbub, Superfeedr, websub.rocks) needs only the two link relations.",
|
|
@@ -18136,6 +18507,7 @@ var JsonLdPresentAudit = class extends Audit {
|
|
|
18136
18507
|
evidenceGrade: "A",
|
|
18137
18508
|
tier: "scored",
|
|
18138
18509
|
dossier: "docs/evidence/audits/structured-data/json-ld-present.md",
|
|
18510
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18139
18511
|
defaultPriority: "critical",
|
|
18140
18512
|
guidance: {
|
|
18141
18513
|
impact: "Without any JSON-LD structured data, AI agents like ChatGPT and Perplexity treat your site as unstructured text with no machine-readable identity. Your brand, products, and services become invisible to AI-powered discovery, search, and recommendation systems.",
|
|
@@ -18199,6 +18571,7 @@ var SchemaValidationAudit = class extends Audit {
|
|
|
18199
18571
|
evidenceGrade: "A",
|
|
18200
18572
|
tier: "scored",
|
|
18201
18573
|
dossier: "docs/evidence/audits/structured-data/schema-validation.md",
|
|
18574
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18202
18575
|
defaultPriority: "critical",
|
|
18203
18576
|
guidance: {
|
|
18204
18577
|
impact: "JSON-LD blocks missing @context or @type are silently ignored by every schema consumer, including Google, ChatGPT plugins, and RAG pipelines. Even if you have structured data on the page, invalid blocks provide zero value to AI agents.",
|
|
@@ -18325,6 +18698,7 @@ var OrganizationSchemaAudit = class extends Audit {
|
|
|
18325
18698
|
evidenceGrade: "A",
|
|
18326
18699
|
tier: "scored",
|
|
18327
18700
|
dossier: "docs/evidence/audits/structured-data/organization-schema.md",
|
|
18701
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18328
18702
|
applicablePageTypes: ["homepage"],
|
|
18329
18703
|
defaultPriority: "high",
|
|
18330
18704
|
guidance: {
|
|
@@ -18428,6 +18802,7 @@ var BreadcrumbSchemaAudit = class extends Audit {
|
|
|
18428
18802
|
evidenceGrade: "A",
|
|
18429
18803
|
tier: "scored",
|
|
18430
18804
|
dossier: "docs/evidence/audits/structured-data/breadcrumb-schema.md",
|
|
18805
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18431
18806
|
applicablePageTypes: ["category", "product", "content"],
|
|
18432
18807
|
defaultPriority: "medium",
|
|
18433
18808
|
guidance: {
|
|
@@ -18550,6 +18925,7 @@ var ArticleSchemaAudit = class extends Audit {
|
|
|
18550
18925
|
evidenceGrade: "A",
|
|
18551
18926
|
tier: "scored",
|
|
18552
18927
|
dossier: "docs/evidence/audits/structured-data/article-schema.md",
|
|
18928
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18553
18929
|
applicablePageTypes: ["content"],
|
|
18554
18930
|
defaultPriority: "high",
|
|
18555
18931
|
guidance: {
|
|
@@ -18676,6 +19052,7 @@ var FaqPageSchemaAudit = class extends Audit {
|
|
|
18676
19052
|
evidenceGrade: "C",
|
|
18677
19053
|
tier: "informative",
|
|
18678
19054
|
dossier: "docs/evidence/audits/structured-data/faqpage-schema.md",
|
|
19055
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18679
19056
|
defaultPriority: "medium",
|
|
18680
19057
|
guidance: {
|
|
18681
19058
|
impact: "AI answer engines like Perplexity and Google SGE give priority to FAQ-structured content for direct answers. Without FAQPage schema, your Q&A content is treated as unstructured text and is less likely to be surfaced as a featured answer in AI-generated responses.",
|
|
@@ -18829,6 +19206,7 @@ var ServiceSchemaAudit = class _ServiceSchemaAudit extends Audit {
|
|
|
18829
19206
|
evidenceGrade: "A",
|
|
18830
19207
|
tier: "scored",
|
|
18831
19208
|
dossier: "docs/evidence/audits/structured-data/service-schema.md",
|
|
19209
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18832
19210
|
// Where a service business publishes its offerings. NOT ['product'] —
|
|
18833
19211
|
// that was inherited from the pre-split audit and inverted this check:
|
|
18834
19212
|
// it skipped every service site (no product page in the scan) and ran only
|
|
@@ -18972,6 +19350,7 @@ var SpeakableSchemaAudit = class _SpeakableSchemaAudit extends Audit {
|
|
|
18972
19350
|
evidenceGrade: "B",
|
|
18973
19351
|
tier: "scored",
|
|
18974
19352
|
dossier: "docs/evidence/audits/structured-data/speakable-schema.md",
|
|
19353
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18975
19354
|
// News and article publishing is the whole documented scope of the
|
|
18976
19355
|
// feature, so a scan with no content page never runs this audit at all.
|
|
18977
19356
|
// The runtime guard below repeats the precondition for the pages that
|
|
@@ -19070,6 +19449,7 @@ var HowToSchemaAudit = class extends Audit {
|
|
|
19070
19449
|
evidenceGrade: "C",
|
|
19071
19450
|
tier: "informative",
|
|
19072
19451
|
dossier: "docs/evidence/audits/structured-data/howto-schema.md",
|
|
19452
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19073
19453
|
applicablePageTypes: ["content"],
|
|
19074
19454
|
defaultPriority: "low",
|
|
19075
19455
|
guidance: {
|
|
@@ -19218,6 +19598,7 @@ var LocalBusinessSchemaAudit = class extends Audit {
|
|
|
19218
19598
|
evidenceGrade: "A",
|
|
19219
19599
|
tier: "scored",
|
|
19220
19600
|
dossier: "docs/evidence/audits/structured-data/local-business-schema.md",
|
|
19601
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19221
19602
|
applicablePageTypes: ["homepage"],
|
|
19222
19603
|
defaultPriority: "medium",
|
|
19223
19604
|
guidance: {
|
|
@@ -19383,6 +19764,7 @@ var ReviewSchemaAudit = class extends Audit {
|
|
|
19383
19764
|
evidenceGrade: "A",
|
|
19384
19765
|
tier: "scored",
|
|
19385
19766
|
dossier: "docs/evidence/audits/structured-data/review-schema.md",
|
|
19767
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19386
19768
|
applicablePageTypes: ["homepage", "product"],
|
|
19387
19769
|
defaultPriority: "medium",
|
|
19388
19770
|
guidance: {
|
|
@@ -19500,6 +19882,7 @@ var AuthorSchemaAudit = class extends Audit {
|
|
|
19500
19882
|
evidenceGrade: "C",
|
|
19501
19883
|
tier: "informative",
|
|
19502
19884
|
dossier: "docs/evidence/audits/structured-data/author-schema.md",
|
|
19885
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19503
19886
|
applicablePageTypes: ["content"],
|
|
19504
19887
|
defaultPriority: "medium",
|
|
19505
19888
|
guidance: {
|
|
@@ -19622,6 +20005,7 @@ var ProductDetailsAudit = class extends Audit {
|
|
|
19622
20005
|
evidenceGrade: "A",
|
|
19623
20006
|
tier: "scored",
|
|
19624
20007
|
dossier: "docs/evidence/audits/structured-data/advanced-product-details.md",
|
|
20008
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19625
20009
|
applicablePageTypes: ["product"],
|
|
19626
20010
|
defaultPriority: "medium",
|
|
19627
20011
|
guidance: {
|
|
@@ -19775,6 +20159,7 @@ var ClaimreviewAdvisoryAudit = class extends Audit {
|
|
|
19775
20159
|
evidenceGrade: "A",
|
|
19776
20160
|
tier: "informative",
|
|
19777
20161
|
dossier: "docs/evidence/audits/structured-data/claimreview-advisory.md",
|
|
20162
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19778
20163
|
defaultPriority: "low",
|
|
19779
20164
|
guidance: {
|
|
19780
20165
|
impact: "Google's fact check documentation states plainly: 'We're phasing out support for ClaimReview markup in Google Search', with no deprecation date, and notes only one ClaimReview element per page qualifies for rich results. A check that scored ClaimReview coverage as an AI-readiness win would therefore push publishers to invest in a channel its largest documented consumer is actively withdrawing from. FALSIFIABLE and grade A on the evidence, but it measures the state of an external product, not the quality of the site \u2014 which is exactly why it must not contribute to a score.",
|
|
@@ -19930,6 +20315,7 @@ var MetaDescriptionAudit = class extends Audit {
|
|
|
19930
20315
|
evidenceGrade: "B",
|
|
19931
20316
|
tier: "scored",
|
|
19932
20317
|
dossier: "docs/evidence/audits/answer-readiness/meta-description.md",
|
|
20318
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19933
20319
|
defaultPriority: "high",
|
|
19934
20320
|
guidance: {
|
|
19935
20321
|
impact: "Google sometimes uses the meta description as the search snippet when it describes the page more accurately than the body text, and AI Overviews and AI Mode inherit that snippet pipeline. A missing, keyword-stuffed or off-topic description means the summary shown alongside your page is written by someone else.",
|
|
@@ -20024,6 +20410,7 @@ var MetaAuthorAudit = class extends Audit {
|
|
|
20024
20410
|
evidenceGrade: "C",
|
|
20025
20411
|
tier: "informative",
|
|
20026
20412
|
dossier: "docs/evidence/audits/answer-readiness/meta-author.md",
|
|
20413
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20027
20414
|
applicablePageTypes: ["content"],
|
|
20028
20415
|
defaultPriority: "medium",
|
|
20029
20416
|
guidance: {
|
|
@@ -20072,6 +20459,7 @@ var UniqueMetaAudit = class extends Audit {
|
|
|
20072
20459
|
evidenceGrade: "C",
|
|
20073
20460
|
tier: "informative",
|
|
20074
20461
|
dossier: "docs/evidence/audits/answer-readiness/unique-meta.md",
|
|
20462
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20075
20463
|
defaultPriority: "high",
|
|
20076
20464
|
guidance: {
|
|
20077
20465
|
impact: "AI crawlers use title and description pairs to distinguish between pages. Duplicate meta across pages causes agents to merge or skip content, meaning some pages become invisible in AI-generated answers.",
|
|
@@ -20185,6 +20573,7 @@ var CoreOpenGraphAudit = class extends Audit {
|
|
|
20185
20573
|
evidenceGrade: "A",
|
|
20186
20574
|
tier: "scored",
|
|
20187
20575
|
dossier: "docs/evidence/audits/answer-readiness/core-open-graph.md",
|
|
20576
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20188
20577
|
defaultPriority: "high",
|
|
20189
20578
|
guidance: {
|
|
20190
20579
|
impact: "Link-preview crawlers use Open Graph tags to build the card shown wherever your page is shared, and Google uses og:title and og:site_name as inputs to the title link and site name on a result \u2014 the same labels that carry into AI Overviews source cards. Without them the crawler falls back to guessing.",
|
|
@@ -20268,6 +20657,7 @@ var OgTypeAudit = class extends Audit {
|
|
|
20268
20657
|
evidenceGrade: "B",
|
|
20269
20658
|
tier: "scored",
|
|
20270
20659
|
dossier: "docs/evidence/audits/answer-readiness/og-type.md",
|
|
20660
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20271
20661
|
defaultPriority: "medium",
|
|
20272
20662
|
guidance: {
|
|
20273
20663
|
impact: "AI agents use og:type to classify page content for type-specific handling. Without it, agents treat every page as generic content, missing opportunities for article freshness scoring or product-specific handling.",
|
|
@@ -20331,6 +20721,7 @@ var OgImageAltAudit = class extends Audit {
|
|
|
20331
20721
|
evidenceGrade: "C",
|
|
20332
20722
|
tier: "informative",
|
|
20333
20723
|
dossier: "docs/evidence/audits/answer-readiness/og-image-alt.md",
|
|
20724
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20334
20725
|
defaultPriority: "medium",
|
|
20335
20726
|
guidance: {
|
|
20336
20727
|
impact: "AI agents cannot process images directly and rely on og:image:alt text to understand your page's visual content. Without alt text, the OG image is invisible to text-based AI systems generating answers about your page.",
|
|
@@ -20402,6 +20793,7 @@ var FaqSectionsAudit = class _FaqSectionsAudit extends Audit {
|
|
|
20402
20793
|
evidenceGrade: "C",
|
|
20403
20794
|
tier: "informative",
|
|
20404
20795
|
dossier: "docs/evidence/audits/answer-readiness/faq-sections.md",
|
|
20796
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20405
20797
|
defaultPriority: "medium",
|
|
20406
20798
|
guidance: {
|
|
20407
20799
|
impact: 'FAQ sections with clear question headings are the highest-priority extraction target for AI-generated answers and "People Also Ask" results. Without them, your content misses the most direct path to appearing in AI answer snippets.',
|
|
@@ -20496,6 +20888,7 @@ var QuestionHeadingsAudit = class _QuestionHeadingsAudit extends Audit {
|
|
|
20496
20888
|
evidenceGrade: "C",
|
|
20497
20889
|
tier: "informative",
|
|
20498
20890
|
dossier: "docs/evidence/audits/answer-readiness/question-headings.md",
|
|
20891
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20499
20892
|
defaultPriority: "medium",
|
|
20500
20893
|
guidance: {
|
|
20501
20894
|
impact: "AI answer engines directly match user questions to heading text. Question-formatted headings are the primary signal for identifying which section answers a specific query. Without them, agents must guess which section is relevant, reducing your content's match rate.",
|
|
@@ -20664,6 +21057,7 @@ var DatesOnContentAudit = class extends Audit {
|
|
|
20664
21057
|
evidenceGrade: "A",
|
|
20665
21058
|
tier: "scored",
|
|
20666
21059
|
dossier: "docs/evidence/audits/answer-readiness/dates-on-content.md",
|
|
21060
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20667
21061
|
applicablePageTypes: ["content"],
|
|
20668
21062
|
defaultPriority: "medium",
|
|
20669
21063
|
guidance: {
|
|
@@ -20755,6 +21149,7 @@ var FirstParagraphAnswersAudit = class extends Audit {
|
|
|
20755
21149
|
evidenceGrade: "C",
|
|
20756
21150
|
tier: "informative",
|
|
20757
21151
|
dossier: "docs/evidence/audits/answer-readiness/first-paragraph-answers.md",
|
|
21152
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20758
21153
|
applicablePageTypes: ["content"],
|
|
20759
21154
|
defaultPriority: "high",
|
|
20760
21155
|
guidance: {
|
|
@@ -20920,6 +21315,7 @@ var DirectDefinitionsAudit = class extends Audit {
|
|
|
20920
21315
|
evidenceGrade: "C",
|
|
20921
21316
|
tier: "informative",
|
|
20922
21317
|
dossier: "docs/evidence/audits/answer-readiness/direct-definitions.md",
|
|
21318
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20923
21319
|
applicablePageTypes: ["content"],
|
|
20924
21320
|
// Never a defect, so never above the actionable items.
|
|
20925
21321
|
defaultPriority: "low",
|
|
@@ -20978,6 +21374,7 @@ var ComparisonTablesAudit = class _ComparisonTablesAudit extends Audit {
|
|
|
20978
21374
|
evidenceGrade: "C",
|
|
20979
21375
|
tier: "informative",
|
|
20980
21376
|
dossier: "docs/evidence/audits/answer-readiness/comparison-tables.md",
|
|
21377
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20981
21378
|
applicablePageTypes: ["category", "product", "content"],
|
|
20982
21379
|
defaultPriority: "low",
|
|
20983
21380
|
guidance: {
|
|
@@ -21053,6 +21450,7 @@ var SpecificNumbersAudit = class _SpecificNumbersAudit extends Audit {
|
|
|
21053
21450
|
evidenceGrade: "B",
|
|
21054
21451
|
tier: "scored",
|
|
21055
21452
|
dossier: "docs/evidence/audits/answer-readiness/specific-numbers.md",
|
|
21453
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21056
21454
|
defaultPriority: "medium",
|
|
21057
21455
|
guidance: {
|
|
21058
21456
|
impact: "AI answer engines strongly prefer content with concrete data points over vague claims. Pages with specific numbers, percentages, and metrics are ranked higher for data-driven queries because agents can cite exact figures in generated answers.",
|
|
@@ -21111,17 +21509,6 @@ var SpecificNumbersAudit = class _SpecificNumbersAudit extends Audit {
|
|
|
21111
21509
|
};
|
|
21112
21510
|
|
|
21113
21511
|
// src/audits/answer-readiness/content-without-clickthrough.ts
|
|
21114
|
-
function contentWordCount($) {
|
|
21115
|
-
const main = $("main").first();
|
|
21116
|
-
const extract = (sel) => {
|
|
21117
|
-
const clone = sel.clone();
|
|
21118
|
-
clone.find("script, style, noscript, template").remove();
|
|
21119
|
-
return clone.text().replace(/\s+/g, " ").trim();
|
|
21120
|
-
};
|
|
21121
|
-
let text3 = main.length ? extract(main) : "";
|
|
21122
|
-
if (!text3) text3 = extract($("body"));
|
|
21123
|
-
return text3.split(/\s+/).filter(Boolean).length;
|
|
21124
|
-
}
|
|
21125
21512
|
var TEASER_PATTERNS = [
|
|
21126
21513
|
/click\s+(here\s+)?to\s+read\s+more/i,
|
|
21127
21514
|
/contact\s+us\s+to\s+learn/i,
|
|
@@ -21144,6 +21531,7 @@ var ContentWithoutClickthroughAudit = class _ContentWithoutClickthroughAudit ext
|
|
|
21144
21531
|
evidenceGrade: "B",
|
|
21145
21532
|
tier: "scored",
|
|
21146
21533
|
dossier: "docs/evidence/audits/answer-readiness/content-without-clickthrough.md",
|
|
21534
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21147
21535
|
defaultPriority: "high",
|
|
21148
21536
|
guidance: {
|
|
21149
21537
|
impact: 'AI answer engines skip pages dominated by teaser content ("click to read more", "sign up to access"). These pages provide no extractable answers, so agents will never surface your content in AI-generated responses, costing you visibility in AI search.',
|
|
@@ -21195,7 +21583,7 @@ var ContentWithoutClickthroughAudit = class _ContentWithoutClickthroughAudit ext
|
|
|
21195
21583
|
return !head2.startsWith("<?xml");
|
|
21196
21584
|
});
|
|
21197
21585
|
if (checkPage) {
|
|
21198
|
-
const wordCount2 =
|
|
21586
|
+
const wordCount2 = getWordCount(checkPage.$);
|
|
21199
21587
|
if (wordCount2 < 50) {
|
|
21200
21588
|
return this.warn(
|
|
21201
21589
|
"Insufficient content to evaluate.",
|
|
@@ -21286,6 +21674,7 @@ var NamedAuthorAudit = class extends Audit {
|
|
|
21286
21674
|
evidenceGrade: "C",
|
|
21287
21675
|
tier: "informative",
|
|
21288
21676
|
dossier: "docs/evidence/audits/answer-readiness/named-author.md",
|
|
21677
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21289
21678
|
applicablePageTypes: ["content"],
|
|
21290
21679
|
defaultPriority: "high",
|
|
21291
21680
|
guidance: {
|
|
@@ -21412,6 +21801,7 @@ var AuthorSameAsAudit = class extends Audit {
|
|
|
21412
21801
|
evidenceGrade: "C",
|
|
21413
21802
|
tier: "informative",
|
|
21414
21803
|
dossier: "docs/evidence/audits/answer-readiness/author-same-as.md",
|
|
21804
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21415
21805
|
applicablePageTypes: ["content"],
|
|
21416
21806
|
defaultPriority: "medium",
|
|
21417
21807
|
guidance: {
|
|
@@ -21531,6 +21921,7 @@ var AuthorPageAudit = class extends Audit {
|
|
|
21531
21921
|
evidenceGrade: "C",
|
|
21532
21922
|
tier: "informative",
|
|
21533
21923
|
dossier: "docs/evidence/audits/answer-readiness/author-page.md",
|
|
21924
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21534
21925
|
applicablePageTypes: ["content"],
|
|
21535
21926
|
defaultPriority: "medium",
|
|
21536
21927
|
guidance: {
|
|
@@ -21665,6 +22056,7 @@ var AboutCredentialsAudit = class extends Audit {
|
|
|
21665
22056
|
evidenceGrade: "C",
|
|
21666
22057
|
tier: "informative",
|
|
21667
22058
|
dossier: "docs/evidence/audits/answer-readiness/about-credentials.md",
|
|
22059
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21668
22060
|
defaultPriority: "medium",
|
|
21669
22061
|
guidance: {
|
|
21670
22062
|
impact: "AI engines crawl your about page to build an organizational authority profile. Without credential-rich content (team bios, expertise areas, certifications), agents cannot assess your organization's authority, reducing your content's trust score in AI-generated recommendations.",
|
|
@@ -21774,6 +22166,7 @@ var ExternalCitationsAudit = class extends Audit {
|
|
|
21774
22166
|
evidenceGrade: "B",
|
|
21775
22167
|
tier: "scored",
|
|
21776
22168
|
dossier: "docs/evidence/audits/answer-readiness/external-citations.md",
|
|
22169
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21777
22170
|
applicablePageTypes: ["content"],
|
|
21778
22171
|
defaultPriority: "medium",
|
|
21779
22172
|
guidance: {
|
|
@@ -21888,6 +22281,7 @@ var BrandNameAudit = class extends Audit {
|
|
|
21888
22281
|
evidenceGrade: "C",
|
|
21889
22282
|
tier: "informative",
|
|
21890
22283
|
dossier: "docs/evidence/audits/answer-readiness/brand-name.md",
|
|
22284
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21891
22285
|
defaultPriority: "medium",
|
|
21892
22286
|
guidance: {
|
|
21893
22287
|
impact: "AI engines build entity graphs by matching Organization schema names to in-content mentions. If your brand name only appears in schema but not body text, agents cannot associate your content with your entity, weakening brand recognition in AI responses.",
|
|
@@ -21982,7 +22376,7 @@ function statesZeroReviews(record3) {
|
|
|
21982
22376
|
}
|
|
21983
22377
|
return false;
|
|
21984
22378
|
}
|
|
21985
|
-
function
|
|
22379
|
+
function readableText2(page) {
|
|
21986
22380
|
const body = page.$("body").clone();
|
|
21987
22381
|
body.find("script, style, noscript, template").remove();
|
|
21988
22382
|
return body.text().replace(/\s+/g, " ").trim();
|
|
@@ -22083,6 +22477,7 @@ var ReviewSignalsAudit = class extends Audit {
|
|
|
22083
22477
|
evidenceGrade: "B",
|
|
22084
22478
|
tier: "scored",
|
|
22085
22479
|
dossier: "docs/evidence/audits/answer-readiness/review-signals.md",
|
|
22480
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22086
22481
|
applicablePageTypes: ["homepage", "product"],
|
|
22087
22482
|
defaultPriority: "medium",
|
|
22088
22483
|
guidance: {
|
|
@@ -22130,7 +22525,7 @@ var ReviewSignalsAudit = class extends Audit {
|
|
|
22130
22525
|
).toArray().filter((el) => p.$(el).text().trim() !== "" || p.$(el).children().length > 0);
|
|
22131
22526
|
if (widget.length > 0) {
|
|
22132
22527
|
noteWeak("review widget markup", p.url);
|
|
22133
|
-
} else if (/\b\d[\d,]*\s+reviews?\b/i.test(
|
|
22528
|
+
} else if (/\b\d[\d,]*\s+reviews?\b/i.test(readableText2(p))) {
|
|
22134
22529
|
noteWeak('"N reviews" text', p.url);
|
|
22135
22530
|
}
|
|
22136
22531
|
}
|
|
@@ -22190,7 +22585,7 @@ function isNonEnglish(page) {
|
|
|
22190
22585
|
const lang = (page.$("html").attr("lang") ?? "").trim().toLowerCase();
|
|
22191
22586
|
return lang !== "" && !lang.startsWith("en");
|
|
22192
22587
|
}
|
|
22193
|
-
function
|
|
22588
|
+
function readableText3(page) {
|
|
22194
22589
|
const body = page.$("body").clone();
|
|
22195
22590
|
body.find("script, style, noscript, template").remove();
|
|
22196
22591
|
return body.text().replace(/\s+/g, " ").trim();
|
|
@@ -22242,6 +22637,7 @@ var TrustSignalsAudit = class _TrustSignalsAudit extends Audit {
|
|
|
22242
22637
|
evidenceGrade: "B",
|
|
22243
22638
|
tier: "scored",
|
|
22244
22639
|
dossier: "docs/evidence/audits/answer-readiness/trust-signals.md",
|
|
22640
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22245
22641
|
applicablePageTypes: ["homepage"],
|
|
22246
22642
|
defaultPriority: "low",
|
|
22247
22643
|
guidance: {
|
|
@@ -22269,7 +22665,7 @@ var TrustSignalsAudit = class _TrustSignalsAudit extends Audit {
|
|
|
22269
22665
|
"Non-English homepage \u2014 detector not applicable"
|
|
22270
22666
|
);
|
|
22271
22667
|
}
|
|
22272
|
-
const text3 =
|
|
22668
|
+
const text3 = readableText3(page);
|
|
22273
22669
|
const satisfied = [];
|
|
22274
22670
|
const missing = [];
|
|
22275
22671
|
let counted = 0;
|
|
@@ -22379,6 +22775,7 @@ var PublicationDateAudit = class extends Audit {
|
|
|
22379
22775
|
evidenceGrade: "B",
|
|
22380
22776
|
tier: "scored",
|
|
22381
22777
|
dossier: "docs/evidence/audits/answer-readiness/publication-date.md",
|
|
22778
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22382
22779
|
applicablePageTypes: ["content"],
|
|
22383
22780
|
defaultPriority: "medium",
|
|
22384
22781
|
guidance: {
|
|
@@ -22475,6 +22872,7 @@ var LastModifiedSchemaAudit = class extends Audit {
|
|
|
22475
22872
|
evidenceGrade: "B",
|
|
22476
22873
|
tier: "scored",
|
|
22477
22874
|
dossier: "docs/evidence/audits/answer-readiness/last-modified-schema.md",
|
|
22875
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22478
22876
|
applicablePageTypes: ["content"],
|
|
22479
22877
|
defaultPriority: "medium",
|
|
22480
22878
|
guidance: {
|
|
@@ -22561,6 +22959,7 @@ var UniqueDataAudit = class extends Audit {
|
|
|
22561
22959
|
evidenceGrade: "B",
|
|
22562
22960
|
tier: "scored",
|
|
22563
22961
|
dossier: "docs/evidence/audits/answer-readiness/unique-data.md",
|
|
22962
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22564
22963
|
defaultPriority: "medium",
|
|
22565
22964
|
guidance: {
|
|
22566
22965
|
impact: "AI generative engines prioritize content with unique, citable data points because agents can quote exact figures in generated answers. Content without specific numbers reads as opinion rather than evidence, reducing its chances of being cited.",
|
|
@@ -22658,6 +23057,7 @@ var DescriptiveUrlsAudit = class extends Audit {
|
|
|
22658
23057
|
evidenceGrade: "C",
|
|
22659
23058
|
tier: "informative",
|
|
22660
23059
|
dossier: "docs/evidence/audits/answer-readiness/descriptive-urls.md",
|
|
23060
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22661
23061
|
defaultPriority: "high",
|
|
22662
23062
|
guidance: {
|
|
22663
23063
|
impact: "AI engines use URL text as a pre-fetch topic signal and display URLs in generated citations. Non-descriptive slugs with UUIDs or numeric IDs provide no topical context, reducing your content's relevance score before the page is even crawled.",
|
|
@@ -22913,6 +23313,7 @@ var SnippetGateCoverageAudit = class _SnippetGateCoverageAudit extends Audit {
|
|
|
22913
23313
|
evidenceGrade: "A",
|
|
22914
23314
|
tier: "scored",
|
|
22915
23315
|
dossier: "docs/evidence/audits/answer-readiness/snippet-gate-coverage.md",
|
|
23316
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22916
23317
|
defaultPriority: "high",
|
|
22917
23318
|
guidance: {
|
|
22918
23319
|
impact: "Google states the eligibility gate directly: to appear as a supporting link a page 'must be indexed and eligible to be shown in Google Search with a snippet', and names nosnippet, data-nosnippet, max-snippet and noindex as the controls that limit what AI Overviews and AI Mode can show. This makes the causal chain fully documented rather than inferred: a max-snippet value shorter than the answer sentence truncates the answer below usefulness, and data-nosnippet wrapping the answer removes it from AI surfaces entirely while leaving it visible to humans \u2014 an invisible failure that page-level SEO reports do not surface because the directive itself is technically 'valid'.",
|
|
@@ -23125,6 +23526,7 @@ var TextFragmentAddressabilityAudit = class _TextFragmentAddressabilityAudit ext
|
|
|
23125
23526
|
evidenceGrade: "A",
|
|
23126
23527
|
tier: "scored",
|
|
23127
23528
|
dossier: "docs/evidence/audits/answer-readiness/text-fragment-addressability.md",
|
|
23529
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
23128
23530
|
defaultPriority: "medium",
|
|
23129
23531
|
guidance: {
|
|
23130
23532
|
impact: "Google Search auto-generates text-fragment URLs to land users on the exact featured-snippet text, and the spec requires each of prefix/start/end/suffix to match within a single block-level element. When an answer sentence is fragmented across block boundaries, or the header opt-out is set, the fragment silently fails and the link degrades to page-top. Falsifiable and directly testable: take the citing surface\u2019s own generated URL, load it, and observe whether the browser scrolls and highlights. Two failure classes are binary and deterministic \u2014 the opt-out header, and a start string that straddles two blocks.",
|
|
@@ -23272,6 +23674,7 @@ var ChunkBoundaryReferentIntegrityAudit = class extends Audit {
|
|
|
23272
23674
|
weight: weightForGrade("B", "scored"),
|
|
23273
23675
|
defaultPriority: "high",
|
|
23274
23676
|
dossier: "docs/evidence/audits/answer-readiness/chunk-boundary-referent-integrity.md",
|
|
23677
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
23275
23678
|
guidance: {
|
|
23276
23679
|
impact: 'An answer engine retrieves a passage, not a page. A section that opens "This means you should..." and never names its subject is unusable on arrival: the model either drops it or attributes it to whatever else is in the window. The fix is per-sentence and cheap, and it is invisible to a reader of the whole page \u2014 which is why it survives editing.',
|
|
23277
23680
|
fix: 'Open each section with its subject rather than a pronoun, name the product or topic once in every section over about forty words, and replace "as described above" with the name of the thing described.',
|
|
@@ -23470,6 +23873,7 @@ var ExtractorSurvivalRecallAudit = class extends Audit {
|
|
|
23470
23873
|
weight: weightForGrade("B", "scored"),
|
|
23471
23874
|
defaultPriority: "high",
|
|
23472
23875
|
dossier: "docs/evidence/audits/answer-readiness/extractor-survival-recall.md",
|
|
23876
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
23473
23877
|
guidance: {
|
|
23474
23878
|
impact: 'An answer engine never sees the page; it sees whatever its extractor kept. A specification table inside `<aside class="related-specs">` is invisible to every pipeline that strips asides, and the answer about that product gets written without it. The loss is silent: the page looks complete to its author and to every human reviewer.',
|
|
23475
23879
|
fix: 'Put facts inside the main content container, not in an aside, a footer, or a block whose class says "related" or "promo". Where a table must sit outside the article, repeat its facts in the prose so at least one copy survives.',
|
|
@@ -23602,6 +24006,7 @@ var SectionSplitRiskProfileAudit = class extends Audit {
|
|
|
23602
24006
|
weight: weightForGrade("B", "scored"),
|
|
23603
24007
|
defaultPriority: "medium",
|
|
23604
24008
|
dossier: "docs/evidence/audits/answer-readiness/section-split-risk-profile.md",
|
|
24009
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
23605
24010
|
guidance: {
|
|
23606
24011
|
impact: "Retrieval pipelines cut pages into fixed windows. A section longer than the window becomes one chunk carrying the heading and one or more tail chunks carrying none \u2014 and a tail chunk is text with no subject, which retrieves badly and cites worse. A page with no headings at all is cut at arbitrary offsets throughout.",
|
|
23607
24012
|
fix: "Add an `h2` or `h3` roughly every 400 tokens of prose, and split a specification table that runs past the window into per-topic tables so the header row stays with its rows.",
|
|
@@ -23791,6 +24196,7 @@ var SiteWidePassageUniquenessRatioAudit = class extends Audit {
|
|
|
23791
24196
|
weight: weightForGrade("B", "scored"),
|
|
23792
24197
|
defaultPriority: "medium",
|
|
23793
24198
|
dossier: "docs/evidence/audits/answer-readiness/site-wide-passage-uniqueness-ratio.md",
|
|
24199
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
23794
24200
|
guidance: {
|
|
23795
24201
|
impact: "A search engine clusters duplicate and near-duplicate URLs and elects one canonical; the losers have their signals folded into the winner. A cluster of near-duplicate pages that each name themselves canonical therefore competes against itself, and at most one member stays citable however good the others are. Separately, a page whose sentences are mostly site-wide template produces chunks whose embeddings encode the template rather than the page, so every page built from that template lands in the same place in vector space and none is a distinctive match for any question.",
|
|
23796
24202
|
fix: 'Merge near-duplicate pages into one, or point the weaker members at the strongest with rel="canonical" so the election has an answer. For pages that stay, raise the share of text that is theirs alone: cut the repeated intro, the repeated legal paragraph and the repeated call to action, and let each page carry the sentences only it can carry.',
|
|
@@ -24024,6 +24430,7 @@ var TableMarkdownRoundTripLossAudit = class extends Audit {
|
|
|
24024
24430
|
weight: weightForGrade("B", "scored"),
|
|
24025
24431
|
defaultPriority: "medium",
|
|
24026
24432
|
dossier: "docs/evidence/audits/answer-readiness/table-markdown-round-trip-loss.md",
|
|
24433
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
24027
24434
|
guidance: {
|
|
24028
24435
|
impact: "A model does not read your table markup. Something converts it to markdown first, and GFM markdown has no merged cells, no second header row and no lists inside a cell. A header spanning two columns arrives heading one of them; the other column of numbers arrives with no header at all. The model still answers the question \u2014 with a number read from the wrong column, stated as confidently as a right one.",
|
|
24029
24436
|
fix: "Flatten spanned headers into one header row of plain `th` cells, repeating the text where a span used to cover two columns. Put the unit or currency in the header cell rather than in the caption. Take paragraphs and lists out of cells. Where a table is genuinely two tables, publish it as two.",
|
|
@@ -24291,6 +24698,7 @@ var OpenApiExistsAudit = class _OpenApiExistsAudit extends Audit {
|
|
|
24291
24698
|
evidenceGrade: "B",
|
|
24292
24699
|
tier: "informative",
|
|
24293
24700
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-exists.md",
|
|
24701
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
24294
24702
|
defaultPriority: "medium",
|
|
24295
24703
|
guidance: {
|
|
24296
24704
|
impact: "An agent that cannot find your API description cannot call it. Note that every documented consumer today (GPT Actions, Microsoft 365 Copilot API plugins) receives the document from a developer rather than fetching it from your site, so this check is informative and unscored.",
|
|
@@ -24431,6 +24839,7 @@ var OpenApiEndpointsAudit = class _OpenApiEndpointsAudit extends Audit {
|
|
|
24431
24839
|
evidenceGrade: "B",
|
|
24432
24840
|
tier: "scored",
|
|
24433
24841
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-endpoints.md",
|
|
24842
|
+
requires: ["origin-reachable"],
|
|
24434
24843
|
defaultPriority: "high",
|
|
24435
24844
|
guidance: {
|
|
24436
24845
|
impact: "An OpenAPI spec without endpoints is unusable -- AI agents see a spec file but have zero actions they can perform. Your site remains a passive document that agents cannot interact with programmatically.",
|
|
@@ -24558,6 +24967,7 @@ var OpenApiOperationIdsAudit = class _OpenApiOperationIdsAudit extends Audit {
|
|
|
24558
24967
|
evidenceGrade: "B",
|
|
24559
24968
|
tier: "scored",
|
|
24560
24969
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-operation-ids.md",
|
|
24970
|
+
requires: ["origin-reachable"],
|
|
24561
24971
|
defaultPriority: "medium",
|
|
24562
24972
|
guidance: {
|
|
24563
24973
|
impact: "AI agents use operationIds as stable function names when calling your API. Without unique operationIds, agents must infer endpoint names from URL paths, leading to ambiguous calls, naming collisions, and broken integrations.",
|
|
@@ -24737,6 +25147,7 @@ var OpenApiServersAudit = class _OpenApiServersAudit extends Audit {
|
|
|
24737
25147
|
evidenceGrade: "B",
|
|
24738
25148
|
tier: "scored",
|
|
24739
25149
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-servers.md",
|
|
25150
|
+
requires: ["origin-reachable"],
|
|
24740
25151
|
defaultPriority: "high",
|
|
24741
25152
|
guidance: {
|
|
24742
25153
|
impact: "Without a servers array, AI agents cannot determine the base URL for your API. Even if your endpoints are perfectly documented, agents cannot construct valid request URLs, rendering the entire OpenAPI spec unusable.",
|
|
@@ -24901,6 +25312,7 @@ var OpenApiSchemasAudit = class _OpenApiSchemasAudit extends Audit {
|
|
|
24901
25312
|
evidenceGrade: "B",
|
|
24902
25313
|
tier: "scored",
|
|
24903
25314
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-schemas.md",
|
|
25315
|
+
requires: ["origin-reachable"],
|
|
24904
25316
|
defaultPriority: "medium",
|
|
24905
25317
|
guidance: {
|
|
24906
25318
|
impact: "Without request/response schemas, AI agents must guess what data to send and what to expect back. This leads to malformed requests, failed API calls, and agents that cannot reliably use your endpoints.",
|
|
@@ -25305,6 +25717,7 @@ var AiCatalogExistsAudit = class _AiCatalogExistsAudit extends Audit {
|
|
|
25305
25717
|
evidenceGrade: "C",
|
|
25306
25718
|
tier: "informative",
|
|
25307
25719
|
dossier: "docs/evidence/audits/agent-interfaces/ai-catalog-exists.md",
|
|
25720
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
25308
25721
|
defaultPriority: "medium",
|
|
25309
25722
|
guidance: {
|
|
25310
25723
|
impact: "Without an AI catalog, agents must probe multiple endpoints to discover your services. This wastes time, increases error rates, and often results in agents skipping your site entirely in favor of competitors with a machine-readable capability manifest.",
|
|
@@ -25425,6 +25838,7 @@ var AiCatalogMetadataAudit = class _AiCatalogMetadataAudit extends Audit {
|
|
|
25425
25838
|
evidenceGrade: "B",
|
|
25426
25839
|
tier: "scored",
|
|
25427
25840
|
dossier: "docs/evidence/audits/agent-interfaces/ai-catalog-metadata.md",
|
|
25841
|
+
requires: ["origin-reachable"],
|
|
25428
25842
|
defaultPriority: "medium",
|
|
25429
25843
|
guidance: {
|
|
25430
25844
|
impact: "A thin catalog entry is a catalog entry nobody finds. Consumers match a user query against the entry text, so entries with no description, tags, capabilities or representative queries lose to better-described alternatives even when your service is the better answer.",
|
|
@@ -25560,6 +25974,7 @@ var AiCatalogUrlsAudit = class _AiCatalogUrlsAudit extends Audit {
|
|
|
25560
25974
|
evidenceGrade: "B",
|
|
25561
25975
|
tier: "scored",
|
|
25562
25976
|
dossier: "docs/evidence/audits/agent-interfaces/ai-catalog-urls.md",
|
|
25977
|
+
requires: ["origin-reachable"],
|
|
25563
25978
|
defaultPriority: "medium",
|
|
25564
25979
|
guidance: {
|
|
25565
25980
|
impact: "A broken entry url makes an agent fail mid-task: it read your manifest, followed the link you published, and got nothing. Entries whose url points at a nested catalog or registry cut off everything behind them as well.",
|
|
@@ -25687,6 +26102,7 @@ var AgentsJsonAudit = class extends Audit {
|
|
|
25687
26102
|
evidenceGrade: "C",
|
|
25688
26103
|
tier: "informative",
|
|
25689
26104
|
dossier: "docs/evidence/audits/agent-interfaces/agents-json.md",
|
|
26105
|
+
requires: ["origin-reachable"],
|
|
25690
26106
|
defaultPriority: "low",
|
|
25691
26107
|
guidance: {
|
|
25692
26108
|
impact: "Publishing agents.json is not known to make a site reachable to any agent: no vendor documents reading the file, and the specification has been dormant since 2025-08-21. What does matter is that a document already published at a well-known path can be read \u2014 a 200 carrying the site's HTML shell tells a conforming client the resource exists and then gives it nothing to parse, which is worse than a clean 404.",
|
|
@@ -25810,6 +26226,7 @@ var McpDiscoveryAudit = class _McpDiscoveryAudit extends Audit {
|
|
|
25810
26226
|
evidenceGrade: "C",
|
|
25811
26227
|
tier: "informative",
|
|
25812
26228
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-discovery.md",
|
|
26229
|
+
requires: ["origin-reachable"],
|
|
25813
26230
|
defaultPriority: "medium",
|
|
25814
26231
|
guidance: {
|
|
25815
26232
|
impact: "No shipping MCP client is documented as fetching `/.well-known/mcp/servers.json` or `/.well-known/ucp`, so publishing one is not known to make a site reachable to any agent. What does matter is that a document published at a well-known path can be read: a 200 carrying HTML or unparseable JSON tells a conforming client the resource exists and then gives it nothing to parse.",
|
|
@@ -26129,6 +26546,7 @@ var McpEndpointAudit = class _McpEndpointAudit extends Audit {
|
|
|
26129
26546
|
evidenceGrade: "C",
|
|
26130
26547
|
tier: "informative",
|
|
26131
26548
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-endpoint.md",
|
|
26549
|
+
requires: ["origin-reachable"],
|
|
26132
26550
|
defaultPriority: "high",
|
|
26133
26551
|
guidance: {
|
|
26134
26552
|
impact: "If your MCP endpoint does not answer an initialize handshake, AI assistants cannot connect at all. Capabilities and tool annotations come off the same connection: without them an agent cannot tell whether your server offers tools, or which of them are destructive enough to need user confirmation.",
|
|
@@ -26426,6 +26844,7 @@ var SearchEndpointAudit = class _SearchEndpointAudit extends Audit {
|
|
|
26426
26844
|
evidenceGrade: "C",
|
|
26427
26845
|
tier: "informative",
|
|
26428
26846
|
dossier: "docs/evidence/audits/agent-interfaces/search-endpoint.md",
|
|
26847
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
26429
26848
|
defaultPriority: "low",
|
|
26430
26849
|
guidance: {
|
|
26431
26850
|
impact: 'Without a declared search endpoint, an agent asked to "find pricing info on Example.com" has to crawl the site to answer. Note that no vendor documents an agent that reads SearchAction today \u2014 Google retired its only documented consumer in 2024 \u2014 so this check is informative and unscored.',
|
|
@@ -26596,6 +27015,7 @@ var WebmcpRegisteredToolsAudit = class extends Audit {
|
|
|
26596
27015
|
// detector that cannot distinguish "no tools" from "cannot see the tools".
|
|
26597
27016
|
tier: "experimental",
|
|
26598
27017
|
dossier: "docs/evidence/audits/agent-interfaces/webmcp-registered-tools.md",
|
|
27018
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
26599
27019
|
// Was `high` on an admittedly non-standard convention, so it outranked
|
|
26600
27020
|
// genuinely actionable items in the recommendation list.
|
|
26601
27021
|
defaultPriority: "low",
|
|
@@ -26711,6 +27131,7 @@ var WebmcpDeclarativeFormsAudit = class extends Audit {
|
|
|
26711
27131
|
evidenceGrade: "B",
|
|
26712
27132
|
tier: "scored",
|
|
26713
27133
|
dossier: "docs/evidence/audits/agent-interfaces/webmcp-declarative-forms.md",
|
|
27134
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
26714
27135
|
// Softened from 'high': the feature is Baseline "limited" (Chrome 149 /
|
|
26715
27136
|
// Edge 150 origin trials, Brave Leo experimental) and Apple's WebKit
|
|
26716
27137
|
// standards position is "oppose", so this is worth doing, not urgent.
|
|
@@ -26838,6 +27259,7 @@ var OpenApiDescriptionQualityAudit = class _OpenApiDescriptionQualityAudit exten
|
|
|
26838
27259
|
evidenceGrade: "A",
|
|
26839
27260
|
tier: "scored",
|
|
26840
27261
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-description-quality.md",
|
|
27262
|
+
requires: ["origin-reachable"],
|
|
26841
27263
|
defaultPriority: "high",
|
|
26842
27264
|
guidance: {
|
|
26843
27265
|
impact: "LLM tool-calling treats your OpenAPI descriptions as the function-calling prompt. Missing or terse descriptions force the model to guess what each endpoint does and what each parameter accepts, producing wrong tool selection, malformed arguments, and failed API calls that erode user trust in agent-driven workflows on your site.",
|
|
@@ -27019,6 +27441,7 @@ var CorsApiRoutesAudit = class _CorsApiRoutesAudit extends Audit {
|
|
|
27019
27441
|
evidenceGrade: "C",
|
|
27020
27442
|
tier: "informative",
|
|
27021
27443
|
dossier: "docs/evidence/audits/agent-interfaces/cors-api-routes.md",
|
|
27444
|
+
requires: ["origin-reachable"],
|
|
27022
27445
|
// The affected consumer class is small; nothing here should outrank an
|
|
27023
27446
|
// item that changes what a crawler or an MCP client can do.
|
|
27024
27447
|
defaultPriority: "low",
|
|
@@ -27178,6 +27601,7 @@ var McpModernEraReachabilityAudit = class extends Audit {
|
|
|
27178
27601
|
evidenceGrade: "A",
|
|
27179
27602
|
tier: "scored",
|
|
27180
27603
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-modern-era-reachability.md",
|
|
27604
|
+
requires: ["origin-reachable"],
|
|
27181
27605
|
defaultPriority: "high",
|
|
27182
27606
|
guidance: {
|
|
27183
27607
|
impact: "Revision 2026-07-28 abolished the `initialize` handshake and protocol-level sessions: version, client identity and capabilities now travel as per-request `_meta`, and `server/discover` is a MUST-implement RPC. The spec's own compatibility matrix states verbatim that a Modern client against a Legacy server FAILS, with no fall-forward path. Therefore: if a single POST of `server/discover` carrying `_meta` + `MCP-Protocol-Version: 2026-07-28` does not yield either a DiscoverResult or a recognized modern JSON-RPC error, then every client that has moved to the current revision cannot invoke a single tool on this server \u2014 the failure is total, not degraded. Conversely a 404/-32601 on `server/discover` from a server that otherwise answers modern requests is a direct MUST violation that breaks pre-consent capability presentation.",
|
|
@@ -27428,6 +27852,7 @@ var McpOauthDiscoveryChainAudit = class extends Audit {
|
|
|
27428
27852
|
evidenceGrade: "A",
|
|
27429
27853
|
tier: "scored",
|
|
27430
27854
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-oauth-discovery-chain.md",
|
|
27855
|
+
requires: ["origin-reachable"],
|
|
27431
27856
|
defaultPriority: "high",
|
|
27432
27857
|
guidance: {
|
|
27433
27858
|
impact: "The spec makes RFC 9728 mandatory for MCP servers and makes clients apply two hard identity checks: RFC 9728 \xA73.3 requires the PRM's `resource` value to be string-identical to the resource identifier used to construct the request URL, and the MCP AS-discovery rules require the fetched AS metadata's `issuer` to be string-identical to the issuer used to construct the well-known URL \u2014 on either mismatch the client MUST NOT use the metadata. MCP additionally strengthens RFC 9728 by requiring `authorization_servers` to carry at least one entry (it is merely OPTIONAL in the RFC). Each of these is a silent, total blocker: the discovery chain either resolves end to end or the agent never reaches an authorization prompt, so a single character of drift between the deployed endpoint URL and the `resource` claim makes the server unusable to every conforming client while the server's own logs show nothing but 401s.",
|
|
@@ -27684,6 +28109,7 @@ var McpToolContractValidityAudit = class extends Audit {
|
|
|
27684
28109
|
evidenceGrade: "A",
|
|
27685
28110
|
tier: "scored",
|
|
27686
28111
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-tool-contract-validity.md",
|
|
28112
|
+
requires: ["origin-reachable"],
|
|
27687
28113
|
defaultPriority: "critical",
|
|
27688
28114
|
guidance: {
|
|
27689
28115
|
impact: "The spec gives clients an explicit deletion instruction: 'Clients using the Streamable HTTP transport MUST reject tool definitions where any x-mcp-header value violates these constraints. Rejection means the client MUST exclude the invalid tool from the result of tools/list.' This makes malformed tool metadata a silent-invisibility bug rather than an error: the server returns the tool, logs a successful tools/list, and the model never sees it. The constraint set is fully machine-checkable with no network calls beyond the one list fetch \u2014 token syntax, no CR/LF, case-insensitive uniqueness, primitive types only with `number` explicitly excluded, and static reachability through a chain consisting solely of `properties` keys. Alongside it, `inputSchema` MUST be a valid JSON Schema object and not null; a null or scalar inputSchema breaks argument construction in every SDK.",
|
|
@@ -27940,6 +28366,7 @@ var McpToolsListDeterminismAudit = class extends Audit {
|
|
|
27940
28366
|
evidenceGrade: "A",
|
|
27941
28367
|
tier: "scored",
|
|
27942
28368
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-tools-list-determinism.md",
|
|
28369
|
+
requires: ["origin-reachable"],
|
|
27943
28370
|
defaultPriority: "medium",
|
|
27944
28371
|
guidance: {
|
|
27945
28372
|
impact: "The spec states its own causal rationale verbatim: deterministic ordering 'enables clients to reliably cache the tool list and improves LLM prompt cache hit rates when tools are included in model context.' Tool definitions sit near the front of the model's prompt; if their serialized bytes change between turns, the provider-side prefix cache misses and the full tool block is re-billed at uncached rates on every single turn. Separately, servers MUST include caching hints on complete results, and when ttlMs is absent clients SHOULD assume 0 \u2014 immediately stale \u2014 so an omitted hint converts one cheap cached read into a network round-trip on every access. Both defects are invisible in functional testing and both are measurable with three identical requests.",
|
|
@@ -28124,6 +28551,7 @@ var McpVersionDowngradeAudit = class extends Audit {
|
|
|
28124
28551
|
evidenceGrade: "A",
|
|
28125
28552
|
tier: "scored",
|
|
28126
28553
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-version-downgrade.md",
|
|
28554
|
+
requires: ["origin-reachable"],
|
|
28127
28555
|
defaultPriority: "medium",
|
|
28128
28556
|
guidance: {
|
|
28129
28557
|
impact: "With the handshake removed, the ONLY mechanism by which a client discovers a mutually supported version mid-flight is the `UnsupportedProtocolVersionError`: the spec requires code -32022 with `data.supported[]` listing the server's versions, and instructs clients to select from that list and retry. A server that instead returns a 500, a generic -32600/-32602, or a 400 with no `supported` array gives the client nothing to downgrade to \u2014 so a client whose preferred version is one revision ahead of the server's fails permanently even though a mutually supported version exists on both sides. Separately, the spec requires the header and the `_meta` value to agree, with a 400 + -32020 HeaderMismatch on divergence; a server that silently ignores the mismatch is trusting whichever source of truth its proxy layer did not, which is the exact split-brain the header-validation rules exist to prevent.",
|
|
@@ -28287,6 +28715,7 @@ var McpOriginValidationCorsAudit = class extends Audit {
|
|
|
28287
28715
|
weight: weightForGrade("B", "scored"),
|
|
28288
28716
|
defaultPriority: "high",
|
|
28289
28717
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-origin-validation-cors.md",
|
|
28718
|
+
requires: ["origin-reachable"],
|
|
28290
28719
|
guidance: {
|
|
28291
28720
|
impact: "The transport spec is unambiguous: servers MUST validate the Origin header on all incoming connections, and answer 403 when it is present and invalid, because a server that does not is reachable from any web page the user has open. The provable defect is the CORS pairing: an endpoint that reflects the requesting Origin into `Access-Control-Allow-Origin` and returns `Access-Control-Allow-Credentials: true` has authorized any page to enumerate its tool surface and invoke tools with the user\u2019s session.",
|
|
28292
28721
|
fix: "Validate `Origin` on every request and answer 403 when it is present and not one you allow. Never reflect an arbitrary Origin while allowing credentials: return a fixed allow-list, or drop `Access-Control-Allow-Credentials`. `Access-Control-Allow-Origin: *` is only safe on an endpoint that accepts no credentials at all.",
|
|
@@ -28408,61 +28837,6 @@ var McpOriginValidationCorsAudit = class extends Audit {
|
|
|
28408
28837
|
}
|
|
28409
28838
|
};
|
|
28410
28839
|
|
|
28411
|
-
// src/gatherers/domains.ts
|
|
28412
|
-
var MULTI_SUFFIX = /* @__PURE__ */ new Set([
|
|
28413
|
-
"co.uk",
|
|
28414
|
-
"org.uk",
|
|
28415
|
-
"ac.uk",
|
|
28416
|
-
"gov.uk",
|
|
28417
|
-
"me.uk",
|
|
28418
|
-
"net.uk",
|
|
28419
|
-
"com.au",
|
|
28420
|
-
"net.au",
|
|
28421
|
-
"org.au",
|
|
28422
|
-
"edu.au",
|
|
28423
|
-
"gov.au",
|
|
28424
|
-
"co.nz",
|
|
28425
|
-
"co.jp",
|
|
28426
|
-
"or.jp",
|
|
28427
|
-
"ne.jp",
|
|
28428
|
-
"co.za",
|
|
28429
|
-
"co.kr",
|
|
28430
|
-
"co.il",
|
|
28431
|
-
"co.id",
|
|
28432
|
-
"co.th",
|
|
28433
|
-
"com.br",
|
|
28434
|
-
"com.mx",
|
|
28435
|
-
"com.ar",
|
|
28436
|
-
"com.co",
|
|
28437
|
-
"com.pe",
|
|
28438
|
-
"co.in",
|
|
28439
|
-
"com.sg",
|
|
28440
|
-
"com.tr",
|
|
28441
|
-
"com.cn",
|
|
28442
|
-
"com.hk",
|
|
28443
|
-
"com.tw",
|
|
28444
|
-
"com.my",
|
|
28445
|
-
"com.ph",
|
|
28446
|
-
"com.ua",
|
|
28447
|
-
"com.pl",
|
|
28448
|
-
"com.es",
|
|
28449
|
-
"com.pt",
|
|
28450
|
-
"com.gr"
|
|
28451
|
-
]);
|
|
28452
|
-
function registrableDomain(host) {
|
|
28453
|
-
const parts = host.toLowerCase().replace(/\.$/, "").split(".").filter(Boolean);
|
|
28454
|
-
if (parts.length <= 2) return parts.join(".");
|
|
28455
|
-
const lastTwo = parts.slice(-2).join(".");
|
|
28456
|
-
return MULTI_SUFFIX.has(lastTwo) ? parts.slice(-3).join(".") : lastTwo;
|
|
28457
|
-
}
|
|
28458
|
-
function registrableOf(url) {
|
|
28459
|
-
try {
|
|
28460
|
-
return registrableDomain(new URL(url).hostname);
|
|
28461
|
-
} catch {
|
|
28462
|
-
return "";
|
|
28463
|
-
}
|
|
28464
|
-
}
|
|
28465
|
-
|
|
28466
28840
|
// src/audits/agent-interfaces/mcp-registry-listing-ownership.ts
|
|
28467
28841
|
var REGISTRY = "https://registry.modelcontextprotocol.io/v0.1/servers";
|
|
28468
28842
|
var PROOF_PATH = "/.well-known/mcp-registry-auth";
|
|
@@ -28518,6 +28892,7 @@ var McpRegistryListingOwnershipAudit = class extends Audit {
|
|
|
28518
28892
|
weight: weightForGrade("B", "scored"),
|
|
28519
28893
|
defaultPriority: "medium",
|
|
28520
28894
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-registry-listing-ownership.md",
|
|
28895
|
+
requires: ["origin-reachable"],
|
|
28521
28896
|
guidance: {
|
|
28522
28897
|
impact: 'The registry is the index a client resolves "the MCP server for this domain" against. A domain with no first-party entry is absent from it, so the only path to the server is a URL somebody pastes by hand. A listing under an aggregator\u2019s namespace is worse than absent in one way: the brand cannot update or revoke it, and agents routed through it reach a proxy rather than the origin. The reverse-DNS namespace that fixes this is granted on proof of domain control, and that proof has to keep being served.',
|
|
28523
28898
|
fix: "Publish the server under your own reverse-DNS namespace (`com.example/...`), serve the proof at `/.well-known/mcp-registry-auth` in the exact `v=MCPv1; k=ed25519; p=<base64>` form and keep serving it after DNS migrations, keep the listing\u2019s version in step with what the server reports, and offer a `streamable-http` remote rather than only the deprecated `sse`.",
|
|
@@ -28734,6 +29109,7 @@ var McpToolDescriptionCoverageAudit = class extends Audit {
|
|
|
28734
29109
|
weight: weightForGrade("B", "scored"),
|
|
28735
29110
|
defaultPriority: "medium",
|
|
28736
29111
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-tool-description-coverage.md",
|
|
29112
|
+
requires: ["origin-reachable"],
|
|
28737
29113
|
guidance: {
|
|
28738
29114
|
impact: "A tool description and its parameter descriptions are the only prose a model ever sees about a tool \u2014 they are the whole basis on which it decides whether to call it and what to pass. A required parameter with no description, no enum and no pattern gives the model nothing to derive a legal value from, so it guesses. Guessed values come back as validation errors, and the agent spends retry turns per call until it gives up on the tool.",
|
|
28739
29115
|
fix: "Describe every tool and every parameter, in prose long enough to say what a legal value looks like. Constrain string parameters with `enum`, `format` or `pattern` where the legal set is finite. Declare an `outputSchema` so a client can parse the result rather than re-reading it, give each tool a `title` for the consent prompt, and return top-level `instructions` telling a model how the tools fit together.",
|
|
@@ -28971,6 +29347,7 @@ var OfferSchemaAudit = class extends Audit {
|
|
|
28971
29347
|
evidenceGrade: "A",
|
|
28972
29348
|
tier: "scored",
|
|
28973
29349
|
dossier: "docs/evidence/audits/agentic-commerce/offer-schema.md",
|
|
29350
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
28974
29351
|
applicablePageTypes: ["product"],
|
|
28975
29352
|
defaultPriority: "medium",
|
|
28976
29353
|
guidance: {
|
|
@@ -29089,6 +29466,7 @@ var ProductIdentifiersAudit = class extends Audit {
|
|
|
29089
29466
|
evidenceGrade: "A",
|
|
29090
29467
|
tier: "scored",
|
|
29091
29468
|
dossier: "docs/evidence/audits/agentic-commerce/product-identifiers.md",
|
|
29469
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
29092
29470
|
applicablePageTypes: ["product"],
|
|
29093
29471
|
defaultPriority: "high",
|
|
29094
29472
|
guidance: {
|
|
@@ -29206,6 +29584,7 @@ var ProductTransactionCertaintyAudit = class extends Audit {
|
|
|
29206
29584
|
evidenceGrade: "A",
|
|
29207
29585
|
tier: "scored",
|
|
29208
29586
|
dossier: "docs/evidence/audits/agentic-commerce/product-transaction-certainty.md",
|
|
29587
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
29209
29588
|
applicablePageTypes: ["product"],
|
|
29210
29589
|
defaultPriority: "high",
|
|
29211
29590
|
guidance: {
|
|
@@ -29538,6 +29917,7 @@ var BuyableVariantResolutionAudit = class extends Audit {
|
|
|
29538
29917
|
weight: weightForGrade("B", "scored"),
|
|
29539
29918
|
defaultPriority: "high",
|
|
29540
29919
|
dossier: "docs/evidence/audits/agentic-commerce/buyable-variant-resolution.md",
|
|
29920
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
29541
29921
|
applicablePageTypes: ["product"],
|
|
29542
29922
|
guidance: {
|
|
29543
29923
|
impact: "The agentic-commerce feed models a catalogue variant-first: every sellable thing is a variant with its own id, price and availability. A page that shows five sizes and three colours but publishes one Offer \u2014 or an AggregateOffer with only lowPrice and highPrice \u2014 gives an agent no purchasable unit to name and no single price to quote. The row is dropped at feed validation, or the checkout session comes back with `invalid` on the line item.",
|
|
@@ -29751,6 +30131,7 @@ var CartHandoffReachabilityAudit = class extends Audit {
|
|
|
29751
30131
|
weight: weightForGrade("B", "scored"),
|
|
29752
30132
|
defaultPriority: "high",
|
|
29753
30133
|
dossier: "docs/evidence/audits/agentic-commerce/cart-handoff-reachability.md",
|
|
30134
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
29754
30135
|
guidance: {
|
|
29755
30136
|
impact: "Every upstream signal can be perfect and the purchase still dies at the last click. If the cart 302s to a login form because guest checkout is off, or Turnstile is mounted on the checkout document alone, the agent walks the buyer to a wall it cannot pass. ACP reserves a `requires_sign_in` message code for exactly this case, which is a description of the failure, not a fix for it.",
|
|
29756
30137
|
fix: "Allow guest checkout, or at least let an unauthenticated buyer reach the cart and see the totals. Keep bot challenges off the cart and checkout documents \u2014 challenge the payment submission instead, where a human is present. Allow ChatGPT-User in robots.txt and at the edge on cart paths: blocking GPTBot does not block it, and the two are separately tokened.",
|
|
@@ -29956,6 +30337,7 @@ var OfferTruthConsistencyAudit = class extends Audit {
|
|
|
29956
30337
|
weight: weightForGrade("B", "scored"),
|
|
29957
30338
|
defaultPriority: "high",
|
|
29958
30339
|
dossier: "docs/evidence/audits/agentic-commerce/offer-truth-consistency.md",
|
|
30340
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
29959
30341
|
applicablePageTypes: ["product"],
|
|
29960
30342
|
guidance: {
|
|
29961
30343
|
impact: "An agent quotes from the structured data; the seller recomputes the real amount at checkout. When the two disagree the buyer has already committed, and the session comes back with `invalid` or `out_of_stock` \u2014 the most expensive moment at which a purchase can fail. Google says the same thing from the other side: structured data must be a true representation of the page content. Markup that is present and lying passes every syntax validator on the market.",
|
|
@@ -30305,6 +30687,7 @@ var AcpPolicyLinkSurfaceAudit = class _AcpPolicyLinkSurfaceAudit extends Audit {
|
|
|
30305
30687
|
evidenceGrade: "A",
|
|
30306
30688
|
tier: "scored",
|
|
30307
30689
|
dossier: "docs/evidence/audits/agentic-commerce/acp-policy-link-surface.md",
|
|
30690
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
30308
30691
|
defaultPriority: "high",
|
|
30309
30692
|
guidance: {
|
|
30310
30693
|
impact: "Falsifiable claim: ACP spec 2026-04-17 makes `links` one of the 9 REQUIRED fields on every CheckoutSession response, with type enum {terms_of_use, privacy_policy, return_policy, shipping_policy, contact_us, about_us, faq, support}. Independently, the OpenAI product feed spec makes `seller_privacy_policy` and `seller_tos` HARD-REQUIRED whenever `is_eligible_checkout=true`. Therefore a merchant that cannot produce a resolvable HTTPS URL for terms_of_use and privacy_policy CANNOT set is_eligible_checkout=true and its catalogue is excluded from Instant Checkout no matter how good the feed is. Disproof condition: if a merchant with no reachable ToS URL is observed transacting via ACP Instant Checkout, the check is wrong.",
|
|
@@ -30555,6 +30938,7 @@ var LandedCostAndReturnsAudit = class _LandedCostAndReturnsAudit extends Audit {
|
|
|
30555
30938
|
evidenceGrade: "A",
|
|
30556
30939
|
tier: "scored",
|
|
30557
30940
|
dossier: "docs/evidence/audits/agentic-commerce/landed-cost-and-returns.md",
|
|
30941
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
30558
30942
|
applicablePageTypes: ["product"],
|
|
30559
30943
|
defaultPriority: "high",
|
|
30560
30944
|
guidance: {
|
|
@@ -30683,6 +31067,7 @@ var AgentUaCommerceParityAudit = class extends Audit {
|
|
|
30683
31067
|
evidenceGrade: "A",
|
|
30684
31068
|
tier: "scored",
|
|
30685
31069
|
dossier: "docs/evidence/audits/agentic-commerce/agent-ua-commerce-parity.md",
|
|
31070
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
30686
31071
|
defaultPriority: "critical",
|
|
30687
31072
|
guidance: {
|
|
30688
31073
|
impact: "OpenAI operates four separately-tokened agents with separately published IP ranges: OAI-SearchBot (search indexing), ChatGPT-User (user-initiated fetches \u2014 the shopper's agent), GPTBot (training) and OAI-AdsBot (ad landing-page validation). Falsifiable claim: if a product page returns 403, 429, 503 or a challenge interstitial to ChatGPT-User or OAI-SearchBot while returning 200 to a browser, ChatGPT cannot read live price and availability nor follow the buy link, so the product cannot be surfaced or transacted no matter how good the feed is. That block lives at the WAF or CDN edge, which is why an audit that only parses robots.txt is structurally blind to it. Disproof condition: a site 403ing ChatGPT-User on its product pages that still shows live, accurate prices in ChatGPT.",
|
|
@@ -30826,6 +31211,7 @@ var ContactFormAudit = class _ContactFormAudit extends Audit {
|
|
|
30826
31211
|
evidenceGrade: "C",
|
|
30827
31212
|
tier: "informative",
|
|
30828
31213
|
dossier: "docs/evidence/audits/operability-safety/contact-form.md",
|
|
31214
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
30829
31215
|
defaultPriority: "high",
|
|
30830
31216
|
guidance: {
|
|
30831
31217
|
impact: 'When users ask AI agents to "contact this company for a quote" or "send a message to their support team," the agent needs a machine-submittable form or API endpoint. Without one, the agent cannot complete the request and users turn to competitors.',
|
|
@@ -30934,6 +31320,8 @@ var NoBlockingCaptchaAudit = class _NoBlockingCaptchaAudit extends Audit {
|
|
|
30934
31320
|
evidenceGrade: "A",
|
|
30935
31321
|
tier: "scored",
|
|
30936
31322
|
dossier: "docs/evidence/audits/operability-safety/no-blocking-captcha.md",
|
|
31323
|
+
// Gate exemption: A captcha wall is what this audit reports.
|
|
31324
|
+
requires: ["origin-reachable"],
|
|
30937
31325
|
defaultPriority: "high",
|
|
30938
31326
|
guidance: {
|
|
30939
31327
|
impact: 'Blocking CAPTCHAs completely prevent AI agents from submitting forms on behalf of users. When a user asks an agent to "fill out the contact form," the CAPTCHA blocks the action entirely, forcing the user to do it manually or go to a competitor.',
|
|
@@ -30956,6 +31344,23 @@ var NoBlockingCaptchaAudit = class _NoBlockingCaptchaAudit extends Audit {
|
|
|
30956
31344
|
}
|
|
30957
31345
|
};
|
|
30958
31346
|
audit(ctx) {
|
|
31347
|
+
const waf = ctx.wafProtection;
|
|
31348
|
+
if (waf?.isBlocked && !waf.isRateLimit) {
|
|
31349
|
+
return this.fail(
|
|
31350
|
+
`The site answered the scanner with a bot wall (${waf.name}). An AI agent acting for a user meets the same wall.`,
|
|
31351
|
+
"No bot wall or blocking CAPTCHA between an agent and the page",
|
|
31352
|
+
`${waf.name}: ${waf.reason}`,
|
|
31353
|
+
{ priority: "high", description: _NoBlockingCaptchaAudit.meta.description },
|
|
31354
|
+
ctx.baseUrl
|
|
31355
|
+
);
|
|
31356
|
+
}
|
|
31357
|
+
if (ctx.pages.length === 0) {
|
|
31358
|
+
return this.notApplicable(
|
|
31359
|
+
"No page was fetched, so no form could be inspected for a blocking CAPTCHA.",
|
|
31360
|
+
"No recaptcha, hcaptcha, or turnstile script includes detected",
|
|
31361
|
+
"No page fetched"
|
|
31362
|
+
);
|
|
31363
|
+
}
|
|
30959
31364
|
const detectedCaptchas = [];
|
|
30960
31365
|
for (const page of ctx.pages) {
|
|
30961
31366
|
const html = page.fetchResult.body.toLowerCase();
|
|
@@ -31012,6 +31417,7 @@ var FormsNoJsAudit = class _FormsNoJsAudit extends Audit {
|
|
|
31012
31417
|
evidenceGrade: "C",
|
|
31013
31418
|
tier: "informative",
|
|
31014
31419
|
dossier: "docs/evidence/audits/operability-safety/forms-no-js.md",
|
|
31420
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
31015
31421
|
defaultPriority: "medium",
|
|
31016
31422
|
guidance: {
|
|
31017
31423
|
impact: "Most AI agents do not execute JavaScript. If your forms rely on JS for submission (e.g., React/Vue event handlers with no HTML action), agents cannot submit them at all. This blocks lead capture, contact requests, and any form-based interaction.",
|
|
@@ -31191,6 +31597,7 @@ var FormActionabilityAudit = class extends Audit {
|
|
|
31191
31597
|
evidenceGrade: "A",
|
|
31192
31598
|
tier: "scored",
|
|
31193
31599
|
dossier: "docs/evidence/audits/operability-safety/form-actionability.md",
|
|
31600
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
31194
31601
|
defaultPriority: "high",
|
|
31195
31602
|
guidance: {
|
|
31196
31603
|
impact: "AI agents do not render your page visually. Unlabeled fields, div-based fake inputs, and missing autocomplete attributes mean agents cannot tell which field is the email address or the name, so submissions fail silently or land in the wrong fields \u2014 lost leads, broken signups, and abandoned checkouts.",
|
|
@@ -31344,6 +31751,7 @@ var AriaLandmarksAudit = class extends Audit {
|
|
|
31344
31751
|
evidenceGrade: "A",
|
|
31345
31752
|
tier: "scored",
|
|
31346
31753
|
dossier: "docs/evidence/audits/operability-safety/aria-landmarks.md",
|
|
31754
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
31347
31755
|
defaultPriority: "high",
|
|
31348
31756
|
guidance: {
|
|
31349
31757
|
impact: "Claude computer use and browser agents rely on ARIA landmarks to identify page regions (navigation, main content, footer). Missing landmarks force agents to guess page structure from raw HTML, leading to misclicked elements and incorrect content extraction.",
|
|
@@ -31483,7 +31891,19 @@ function defineA11yAudit(spec) {
|
|
|
31483
31891
|
};
|
|
31484
31892
|
}
|
|
31485
31893
|
var base = {
|
|
31486
|
-
category: "operability-safety"
|
|
31894
|
+
category: "operability-safety",
|
|
31895
|
+
/**
|
|
31896
|
+
* Every audit built on this base reads the sampled pages through
|
|
31897
|
+
* `A11yBackedAudit`, so they all carry the same requirement set. Declared
|
|
31898
|
+
* once here; `scripts/check-requires.mjs` resolves it for each audit that
|
|
31899
|
+
* spreads `base`.
|
|
31900
|
+
*/
|
|
31901
|
+
requires: [
|
|
31902
|
+
"origin-reachable",
|
|
31903
|
+
"unblocked-fetches",
|
|
31904
|
+
"rendered-body",
|
|
31905
|
+
"sample-adequate"
|
|
31906
|
+
]
|
|
31487
31907
|
};
|
|
31488
31908
|
function graded(grade, slug) {
|
|
31489
31909
|
const tier = grade === "A" || grade === "B" ? "scored" : "informative";
|
|
@@ -31593,6 +32013,7 @@ var FormErrorMessagesAudit = class _FormErrorMessagesAudit extends Audit {
|
|
|
31593
32013
|
evidenceGrade: "A",
|
|
31594
32014
|
tier: "scored",
|
|
31595
32015
|
dossier: "docs/evidence/audits/operability-safety/form-error-messages.md",
|
|
32016
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
31596
32017
|
defaultPriority: "medium",
|
|
31597
32018
|
guidance: {
|
|
31598
32019
|
impact: "A field with no aria-errormessage or aria-describedby reference has no message attached to it in the accessibility tree, so an agent that submits a form and gets it back rejected cannot tell which field was wrong or why. It retries the same values or abandons the form.",
|
|
@@ -31998,6 +32419,7 @@ var SecurityHeaderHygieneAudit = class extends Audit {
|
|
|
31998
32419
|
evidenceGrade: "C",
|
|
31999
32420
|
tier: "informative",
|
|
32000
32421
|
dossier: "docs/evidence/audits/operability-safety/security-header-hygiene.md",
|
|
32422
|
+
requires: ["origin-reachable"],
|
|
32001
32423
|
defaultPriority: "low",
|
|
32002
32424
|
guidance: {
|
|
32003
32425
|
impact: "Vulnerability-disclosure hygiene, reported for completeness. A conformant security.txt tells a security researcher who to contact; it is read by researchers and disclosure scanners, not by AI agents. Publishing one changes nothing about how an agent retrieves, parses or cites the site, which is why nothing here moves your score. If you do publish one, an expired or contactless file is worse than none: it advertises a disclosure route that no longer works.",
|
|
@@ -32256,6 +32678,7 @@ var FormAutofillTokenCoverageAudit = class _FormAutofillTokenCoverageAudit exten
|
|
|
32256
32678
|
evidenceGrade: "A",
|
|
32257
32679
|
tier: "scored",
|
|
32258
32680
|
dossier: "docs/evidence/audits/operability-safety/form-autofill-token-coverage.md",
|
|
32681
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
32259
32682
|
defaultPriority: "high",
|
|
32260
32683
|
guidance: {
|
|
32261
32684
|
impact: 'Falsifiable claim: an agent filling a checkout must map each field to a value from user profile data. When the field declares autocomplete="postal-code", that mapping is a table lookup against a ratified vocabulary; when it declares name="field_7" with a visual-only label, the mapping is an inference that fails on ambiguous cases (address-line2 vs address-level2, cc-exp vs bday, tel-national vs tel). WebSuite measures the consequence directly: complex form filling succeeds 12.5% and 0% for the two agents tested, against 85%/76% for simple operational clicks. Test: add correct autocomplete tokens to a failing form and re-run the same fill task.',
|
|
@@ -32427,6 +32850,7 @@ var NativeControlSubstitutionAudit = class _NativeControlSubstitutionAudit exten
|
|
|
32427
32850
|
evidenceGrade: "A",
|
|
32428
32851
|
tier: "scored",
|
|
32429
32852
|
dossier: "docs/evidence/audits/operability-safety/native-control-substitution.md",
|
|
32853
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
32430
32854
|
defaultPriority: "high",
|
|
32431
32855
|
guidance: {
|
|
32432
32856
|
impact: `Falsifiable claim: native <select>, <input type="date">, and <input type="file"> are single-call primitives in every mainstream agent toolkit (selectOption, fill, setInputFiles) and are keyboard-operable, so they succeed in one action with no actionability risk. A custom equivalent requires open \u2192 wait for popup \u2192 scroll the option list into view \u2192 locate the option \u2192 click, where each step is independently subject to Playwright's visible/stable/receives-events gates, and Anthropic documents dropdowns specifically as 'tricky for Claude to manipulate using mouse movements'. Test: instrument the same form with native vs custom controls and count tool calls and retries to reach an identical value.`,
|
|
@@ -32765,6 +33189,7 @@ var InvisibleInstructionScanAudit = class _InvisibleInstructionScanAudit extends
|
|
|
32765
33189
|
evidenceGrade: "A",
|
|
32766
33190
|
tier: "scored",
|
|
32767
33191
|
dossier: "docs/evidence/audits/operability-safety/invisible-instruction-scan.md",
|
|
33192
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
32768
33193
|
defaultPriority: "critical",
|
|
32769
33194
|
guidance: {
|
|
32770
33195
|
impact: "If a page carries text nodes that a sighted human cannot perceive but that survive DOM-to-text serialization, an LLM browsing agent ingests them with the same weight as body copy and can act on them. Brave demonstrated exactly this against Comet (white-on-white text, HTML comments, invisible elements hidden in a Reddit spoiler tag) and confirmed Opera Neon was exploitable through 'hidden HTML elements and other non-rendered markup'. Falsifier: an agent that ingests only visually perceivable, rendered text would be immune \u2014 the disclosed incidents show current agents are not. Google's spam policy independently enumerates the same hiding techniques and their legitimate exceptions, giving the detector a canonical technique list and a false-positive allowlist.",
|
|
@@ -33041,6 +33466,7 @@ var AriaLayerInjectionScanAudit = class _AriaLayerInjectionScanAudit extends Aud
|
|
|
33041
33466
|
evidenceGrade: "A",
|
|
33042
33467
|
tier: "scored",
|
|
33043
33468
|
dossier: "docs/evidence/audits/operability-safety/aria-layer-injection-scan.md",
|
|
33469
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
33044
33470
|
defaultPriority: "critical",
|
|
33045
33471
|
guidance: {
|
|
33046
33472
|
impact: "Computer-use and browser agents drive pages through the DOM and accessibility tree, not pixels, so a11y attributes enter the model context with the same weight as visible text while remaining invisible to a sighted human. Anthropic names the vector explicitly: 'hidden malicious form fields in a webpage's DOM invisible to humans, and other hard-to-catch injections such as through the URL text and tab title that only an agent might see.' The divergence sub-check is a defect in its own right independent of injection: an agent that clicks by accessible name will actuate an aria-label that contradicts the rendered label. Falsifier: if every a11y attribute is short, descriptive, and token-consistent with its element's visible text, this channel carries no payload.",
|
|
@@ -33204,6 +33630,7 @@ var GhostClickableElementRatioAudit = class _GhostClickableElementRatioAudit ext
|
|
|
33204
33630
|
evidenceGrade: "B",
|
|
33205
33631
|
tier: "scored",
|
|
33206
33632
|
dossier: "docs/evidence/audits/operability-safety/ghost-clickable-element-ratio.md",
|
|
33633
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
33207
33634
|
defaultPriority: "high",
|
|
33208
33635
|
guidance: {
|
|
33209
33636
|
impact: "An element whose click behaviour comes only from a JS listener on a non-interactive tag, or from cursor:pointer styling, and which carries no role and no accessible name, is omitted from the serialized accessibility snapshot that agent toolkits send to the model. Playwright MCP's default mode is the accessibility tree, not pixel input: every action tool takes an exact element reference from the snapshot, and coordinate clicking exists only behind the optional vision capability. An element absent from the snapshot is therefore unaddressable by the default toolchain \u2014 the agent cannot emit a valid click and must fail or guess a URL. The accessibility linters cannot warn about it either: axe's button-name and link-name rules only fire on elements that already declare button or link semantics, so a bare unroled div is invisible to them by construction.",
|
|
@@ -33426,6 +33853,7 @@ var StatefulControlIntrospectabilityAudit = class _StatefulControlIntrospectabil
|
|
|
33426
33853
|
evidenceGrade: "B",
|
|
33427
33854
|
tier: "scored",
|
|
33428
33855
|
dossier: "docs/evidence/audits/operability-safety/stateful-control-introspectability.md",
|
|
33856
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
33429
33857
|
defaultPriority: "high",
|
|
33430
33858
|
guidance: {
|
|
33431
33859
|
impact: 'An agent works as observe, act, verify. If a toggle\'s only "on" signal is `class="is-active"` and a colour change, the accessibility snapshot is byte-identical before and after the click, so the agent cannot verify the post-condition: it either clicks again and flips the state back, or reports success with no evidence. The accessibility linters cannot catch this, because `aria-required-attr` fires only once the element already declares `role="switch"` or `role="checkbox"` \u2014 the common class-only toggle declares no role and passes silently. Benchmarks put the cost high: WebSuite measures switch, accordion and dropdown primitives among the worst-performing interactions for web agents, and Operator\'s confirmation design assumes the agent can observe a state transition before acting on it.',
|
|
@@ -33635,6 +34063,7 @@ var HoverOnlyContentAndNavigationAudit = class _HoverOnlyContentAndNavigationAud
|
|
|
33635
34063
|
evidenceGrade: "B",
|
|
33636
34064
|
tier: "scored",
|
|
33637
34065
|
dossier: "docs/evidence/audits/operability-safety/hover-only-content-and-navigation.md",
|
|
34066
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
33638
34067
|
defaultPriority: "high",
|
|
33639
34068
|
guidance: {
|
|
33640
34069
|
impact: "A submenu revealed only by an ancestor `:hover` rule is `display:none` or `visibility:hidden` in the resting DOM, and Playwright's actionability contract defines such an element as not visible \u2014 so every Playwright-derived agent refuses to click it, and the snapshot serializer omits it entirely. The agent never learns those destinations exist: it does not fail loudly, it simply reports that the site has no page for what the user asked. WebSuite measures the information half of the same defect at 0% success for tooltip-based retrieval across both agents it tested. The fix is cheap and it is the same fix keyboard users need, which is why it is worth doing once.",
|
|
@@ -33862,6 +34291,7 @@ var DragAndSliderDependencyAudit = class _DragAndSliderDependencyAudit extends A
|
|
|
33862
34291
|
evidenceGrade: "B",
|
|
33863
34292
|
tier: "scored",
|
|
33864
34293
|
dossier: "docs/evidence/audits/operability-safety/drag-and-slider-dependency.md",
|
|
34294
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
33865
34295
|
defaultPriority: "high",
|
|
33866
34296
|
guidance: {
|
|
33867
34297
|
impact: 'A continuous pointer gesture asks an agent to synthesise a pointerdown, a run of intermediate pointermove events and a pointerup at a computed pixel offset, with no feedback between steps and no way to check the interim value. Every other agent action is discrete and verifiable. WebSuite measures slider interaction at 0% success for both agents it tested \u2014 the worst primitive in its taxonomy \u2014 and Anthropic separately documents scrollbars and dropdowns as unreliable under mouse control, recommending keyboard paths instead. Pair the slider with a numeric input bound to the same value and "set max price to 300" stops being a gesture and becomes a fill.',
|
|
@@ -34110,6 +34540,7 @@ var UrlAddressableStateAndPaginationFallbackAudit = class _UrlAddressableStateAn
|
|
|
34110
34540
|
evidenceGrade: "B",
|
|
34111
34541
|
tier: "scored",
|
|
34112
34542
|
dossier: "docs/evidence/audits/operability-safety/url-addressable-state-and-pagination-fallback.md",
|
|
34543
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
34113
34544
|
defaultPriority: "high",
|
|
34114
34545
|
applicablePageTypes: ["category"],
|
|
34115
34546
|
guidance: {
|
|
@@ -34311,6 +34742,7 @@ var FirstContactConsentGateOperabilityAudit = class _FirstContactConsentGateOper
|
|
|
34311
34742
|
evidenceGrade: "C",
|
|
34312
34743
|
tier: "informative",
|
|
34313
34744
|
dossier: "docs/evidence/audits/operability-safety/first-contact-consent-gate-operability.md",
|
|
34745
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
34314
34746
|
defaultPriority: "low",
|
|
34315
34747
|
guidance: {
|
|
34316
34748
|
impact: "An agent arriving with no cookies spends its first actions on the consent layer, before any step of the actual task. Three properties decide whether it can. A layer rendered inside a cross-origin iframe is invisible to a DOM-text extractor that reads only the top document, so the agent's text and its screenshot disagree and it acts on content it cannot actually see. Accept and reject controls built as unroled, unnamed divs are unaddressable in a snapshot for the same reason a ghost-clickable div is. And main content set `inert` or `aria-hidden=\"true\"` while the layer is open empties every snapshot until the layer is gone \u2014 axe's own guidance is that `aria-hidden` removes the element and all its children from the accessibility API. The evidence here is convention rather than documented consumer behaviour, which is why this audit reports rather than scores.",
|
|
@@ -34562,6 +34994,7 @@ var UnicodeCovertChannelScanAudit = class _UnicodeCovertChannelScanAudit extends
|
|
|
34562
34994
|
evidenceGrade: "B",
|
|
34563
34995
|
tier: "scored",
|
|
34564
34996
|
dossier: "docs/evidence/audits/operability-safety/unicode-covert-channel-scan.md",
|
|
34997
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
34565
34998
|
defaultPriority: "critical",
|
|
34566
34999
|
guidance: {
|
|
34567
35000
|
impact: "Tag-block codepoints mirror ASCII and, per Unicode, render as nothing in tag-unaware implementations \u2014 while modern LLM tokenizers process them normally. A complete instruction can therefore ride inside a product description that no human and no visual QA pass can see. Bidi controls make the rendered order differ from the logical order a text-extracting agent reads, which is the Trojan Source class (CVE-2021-42574). Zero-width characters defeat naive substring matching on both sides at once: the site\u2019s own filters and the agent\u2019s. None of this is visible in a screenshot, a browser, or a review \u2014 only in the bytes.",
|
|
@@ -34801,6 +35234,7 @@ var ThirdPartyDomWriteBlastRadiusAudit = class _ThirdPartyDomWriteBlastRadiusAud
|
|
|
34801
35234
|
evidenceGrade: "B",
|
|
34802
35235
|
tier: "scored",
|
|
34803
35236
|
dossier: "docs/evidence/audits/operability-safety/third-party-dom-write-blast-radius.md",
|
|
35237
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
34804
35238
|
defaultPriority: "high",
|
|
34805
35239
|
guidance: {
|
|
34806
35240
|
impact: 'An agent reads the DOM as one document with one level of trust. It has no way to tell text the site wrote from text a vendor script injected after load, so every third-party origin that can write to the page can write instructions the agent will read as the site\'s own. The count is the risk: eleven uncontrolled origins is eleven independent companies \u2014 and their own supply chains \u2014 with the same authority over what an agent believes about the site. A Content-Security-Policy with a nonce, a hash or `strict-dynamic` is what turns that list from "whoever" into "these, and only these". A policy whose sources include `https:` or `*` is present in the response and constrains nothing.',
|
|
@@ -34996,6 +35430,7 @@ var UnsafeAgentTriggerableAffordancesAudit = class _UnsafeAgentTriggerableAfford
|
|
|
34996
35430
|
evidenceGrade: "B",
|
|
34997
35431
|
tier: "scored",
|
|
34998
35432
|
dossier: "docs/evidence/audits/operability-safety/unsafe-agent-triggerable-affordances.md",
|
|
35433
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
34999
35434
|
defaultPriority: "critical",
|
|
35000
35435
|
guidance: {
|
|
35001
35436
|
impact: "An agent exploring a site follows links, and a link that changes state changes it on the first fetch \u2014 no click, no intent, no confirmation. The same property makes the site a target for indirect prompt injection: text on a page can name the URL, and an agent that reads it as an instruction performs the action with the user's own session. Disallowing the path in robots.txt is only a partial mitigation, because a user-initiated fetch is documented as not necessarily bound by robots.txt. The underlying rule is older than agents: a GET is a safe method, meaning it must not have side effects, and everything here is a violation of that rule that agents simply make expensive.",
|
|
@@ -35133,6 +35568,7 @@ var ReflectedParameterInjectionCanaryAudit = class extends Audit {
|
|
|
35133
35568
|
weight: weightForGrade("B", "scored"),
|
|
35134
35569
|
defaultPriority: "critical",
|
|
35135
35570
|
dossier: "docs/evidence/audits/operability-safety/reflected-parameter-injection-canary.md",
|
|
35571
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
35136
35572
|
guidance: {
|
|
35137
35573
|
impact: "Agents and answer engines weight a source by domain authority, and a reflected-input URL passes human inspection because the hostname is genuine. If attacker-controlled query or path input lands in the page's own title, meta description, canonical link, or JSON-LD strings, the domain becomes a self-serve injection host: the attacker does not need to compromise anything, only to share a link. Reflection into rendered text is the same defect one step down, and it is only contained while the page stays out of an index.",
|
|
35138
35574
|
fix: 'Escape URL-derived input before it reaches any template, and keep it out of `<title>`, `<meta name="description">`, `og:description`, `rel="canonical"` and JSON-LD entirely \u2014 those fields should describe the page, not the request. Where a search page must echo the query back to the visitor, render it as escaped text inside the body and mark the page `noindex`.',
|
|
@@ -35401,6 +35837,7 @@ var UgcTrustBoundaryMarkersAudit = class extends Audit {
|
|
|
35401
35837
|
weight: weightForGrade("B", "scored"),
|
|
35402
35838
|
defaultPriority: "high",
|
|
35403
35839
|
dossier: "docs/evidence/audits/operability-safety/ugc-trust-boundary-markers.md",
|
|
35840
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
35404
35841
|
guidance: {
|
|
35405
35842
|
impact: "Attacker-controllable text sits in the same DOM as first-party copy with no boundary, so anything a visitor types reads, to a fetching agent, as a statement the domain made. Google excludes text inside a `data-nosnippet` span, div or section from snippets across web search, Discover and AI Overviews, and includes everything outside it. The sanitizer arm matters most: if a comment body can carry an inline style or an iframe, hiding an instruction inside visitor text becomes self-serve on this site.",
|
|
35406
35843
|
fix: 'Wrap each visitor-written region in a `<div data-nosnippet>` \u2014 the attribute is honoured on span, div and section only \u2014 and add `rel="ugc"` to links inside it. Strip inline `style`, `iframe`, `script` and remote `img` from submitted markup at render time rather than at submit time, so already-stored content is covered too.',
|
|
@@ -35522,6 +35959,7 @@ var AgentUaContentDivergenceDiffAudit = class extends Audit {
|
|
|
35522
35959
|
weight: weightForGrade("B", "scored"),
|
|
35523
35960
|
defaultPriority: "high",
|
|
35524
35961
|
dossier: "docs/evidence/audits/operability-safety/agent-ua-content-divergence-diff.md",
|
|
35962
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
35525
35963
|
guidance: {
|
|
35526
35964
|
impact: "An agent that reads a different page from the one a human sees cannot be checked by the human it answers to. Where the crawler copy is thinner, the answer engine quotes a page the visitor will never find; where it carries text the browser copy does not, the site is speaking to the model privately \u2014 which is the delivery mechanism for every instruction-injection attack that does not need a compromise. A JSON-LD block that differs between variants is the same problem in the field a machine trusts most.",
|
|
35527
35965
|
fix: "Serve one document to every User-Agent. Where a bot-management rule reduces the page for unknown clients, allow the published AI-crawler UAs through it rather than branching on them, and keep the JSON-LD identical across variants. If a crawler should not read the site at all, block it in robots.txt and at the edge rather than serving it a different story.",
|
|
@@ -35892,6 +36330,7 @@ var C2paManifestSurvivesDeliveryAudit = class extends Audit {
|
|
|
35892
36330
|
weight: weightForGrade("B", "scored"),
|
|
35893
36331
|
defaultPriority: "medium",
|
|
35894
36332
|
dossier: "docs/evidence/audits/operability-safety/c2pa-manifest-survives-delivery.md",
|
|
36333
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
35895
36334
|
guidance: {
|
|
35896
36335
|
impact: "Signing an image at creation proves nothing if the bytes a crawler downloads are unsigned. Image transformation layers discard Content Credentials by default \u2014 Cloudflare states outright that with preservation disabled, existing Content Credentials are always discarded \u2014 so the publisher sees signed assets in their library while every consumer sees stripped ones. The provenance work is done and none of it reaches the reader.",
|
|
35897
36336
|
fix: "Turn on Content Credentials preservation in the image pipeline (Cloudflare Images has an explicit setting; Next.js image optimization and most CDN resizers need the manifest copied through or the asset served unoptimized). Verify by fetching the URL the page actually renders, not the asset in the library.",
|
|
@@ -36041,6 +36480,7 @@ var C2paSignerTrustStatusAudit = class extends Audit {
|
|
|
36041
36480
|
weight: weightForGrade("B", "scored"),
|
|
36042
36481
|
defaultPriority: "medium",
|
|
36043
36482
|
dossier: "docs/evidence/audits/operability-safety/c2pa-signer-trust-status.md",
|
|
36483
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
36044
36484
|
guidance: {
|
|
36045
36485
|
impact: "A manifest that exists is not a manifest that verifies. A conforming C2PA validator resolves the signing certificate against the published Trust List and shows the credential as untrusted when it cannot \u2014 which is what a self-signed certificate always produces, and what an expired one produces the day it lapses. The publisher sees Content Credentials on every asset; the consumer sees a warning, or nothing at all.",
|
|
36046
36486
|
fix: "Sign with a certificate from a CA on the C2PA Trust List rather than a self-signed one, renew before it expires, and include an RFC 3161 timestamp so credentials stay valid past the certificate\u2019s own expiry.",
|
|
@@ -36219,6 +36659,7 @@ var OrganizationIdentifierRegistryResolutionAudit = class extends Audit {
|
|
|
36219
36659
|
weight: weightForGrade("B", "scored"),
|
|
36220
36660
|
defaultPriority: "medium",
|
|
36221
36661
|
dossier: "docs/evidence/audits/operability-safety/organization-identifier-registry-resolution.md",
|
|
36662
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
36222
36663
|
guidance: {
|
|
36223
36664
|
impact: "A shopping or payment agent transacting with an unfamiliar merchant needs one thing no amount of markup can self-assert: a legal identity it can check against an authority. The LEI is the only schema.org organization identifier backed by a free, queryable, authoritative registry, which makes it the only one whose truth an outside party can establish. An identifier that resolves to nothing, or to a lapsed registration, or to a different legal name, is worse than none: it looks like verification and is not.",
|
|
36224
36665
|
fix: 'Publish the LEI as `iso6523Code: "0199:<LEI>"` \u2014 Google documents a preference for the prefixed form over bare `leiCode` \u2014 keep the GLEIF registration renewed so its status stays ISSUED, and make sure the `legalName` in your markup is the name GLEIF has on record, not the trading name.',
|
|
@@ -36432,6 +36873,7 @@ var SyntheticMediaDisclosureValidityAudit = class extends Audit {
|
|
|
36432
36873
|
weight: weightForGrade("B", "scored"),
|
|
36433
36874
|
defaultPriority: "medium",
|
|
36434
36875
|
dossier: "docs/evidence/audits/operability-safety/synthetic-media-disclosure-validity.md",
|
|
36876
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
36435
36877
|
guidance: {
|
|
36436
36878
|
impact: "Disclosure only counts if a machine can read it. IPTC types `DigitalSourceType` as a URI from a controlled vocabulary, so a consumer matching against that vocabulary silently ignores `AI-generated`, a bare `trainedAlgorithmicMedia`, or an `https://` spelling of the `http://` vocabulary URI. The publisher believes the image is disclosed; every machine reader sees an undisclosed image. Worse is an asset whose XMP and C2PA manifest disagree about whether a human took the photo \u2014 two provenance channels, one of them wrong.",
|
|
36437
36879
|
fix: "Write the full vocabulary URI, exactly: `http://cv.iptc.org/newscodes/digitalsourcetype/trainedAlgorithmicMedia`. Keep the `http` scheme the vocabulary itself uses, no trailing slash, no free text, and make sure the value agrees with the digital source type asserted in the asset\u2019s C2PA manifest.",
|
|
@@ -36601,6 +37043,7 @@ var TrustTxtReciprocityCoherenceAudit = class extends Audit {
|
|
|
36601
37043
|
weight: 0,
|
|
36602
37044
|
defaultPriority: "low",
|
|
36603
37045
|
dossier: "docs/evidence/audits/operability-safety/trust-txt-reciprocity-coherence.md",
|
|
37046
|
+
requires: ["origin-reachable"],
|
|
36604
37047
|
guidance: {
|
|
36605
37048
|
impact: "trust.txt association attributes are defined as reciprocal: `belongto=<association>` means something only if that association\u2019s own trust.txt carries `member=<this domain>`. That makes the claim checkable rather than self-asserted, which is the whole point of publishing it. Separately, `datatrainingallowed=no` beside a robots.txt that leaves GPTBot and ClaudeBot free to crawl states two opposite policies, and the channel that actually gates crawlers is the one that says yes. Adoption caveat: no AI engine, answer engine or crawler is documented as reading trust.txt.",
|
|
36606
37049
|
fix: "Ask each association you claim to belong to for a reciprocal `member=` line, drop the ones that will not reciprocate, and make `datatrainingallowed=` say the same thing your robots.txt AI-bot groups say.",
|
|
@@ -36793,6 +37236,7 @@ var WikidataRoundTripVerificationAudit = class extends Audit {
|
|
|
36793
37236
|
weight: weightForGrade("B", "scored"),
|
|
36794
37237
|
defaultPriority: "medium",
|
|
36795
37238
|
dossier: "docs/evidence/audits/operability-safety/wikidata-round-trip-verification.md",
|
|
37239
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
36796
37240
|
guidance: {
|
|
36797
37241
|
impact: "A knowledge-graph consumer that grounds a brand to an entity needs corroboration from the authority side, because `sameAs` carries no reciprocity requirement \u2014 Google documents it as a link to a page with more information, nothing more. Wikidata publishes that corroboration for free as P856. A claim whose entity points at an unrelated domain is either the wrong entity or an unbacked identity claim, and an answer engine that resolves it grounds the brand to somebody else.",
|
|
36798
37242
|
fix: "Claim the entity that really is your organization, and make sure the Wikidata item carries your domain as its official website (P856). If the item has no P856 at all, add one: until it does, the claim cannot be corroborated by anyone.",
|
|
@@ -36997,6 +37441,7 @@ function outcomeOf(check) {
|
|
|
36997
37441
|
const tags = check.tags ?? [];
|
|
36998
37442
|
if (tags.includes(TAG_SCAN_ERROR)) return "error";
|
|
36999
37443
|
if (tags.includes(TAG_SKIPPED_PAGE_TYPE)) return "skipped";
|
|
37444
|
+
if (tags.includes(TAG_SKIPPED_NO_EVIDENCE)) return "gated";
|
|
37000
37445
|
return "ran";
|
|
37001
37446
|
}
|
|
37002
37447
|
function traceFromCheck(check, durationMs) {
|
|
@@ -37058,7 +37503,31 @@ function stubCheck(meta2, tag2, explanation) {
|
|
|
37058
37503
|
tier: meta2.tier
|
|
37059
37504
|
};
|
|
37060
37505
|
}
|
|
37061
|
-
function
|
|
37506
|
+
function unmetRequirements(ctx, meta2) {
|
|
37507
|
+
const required = meta2.requires ?? [];
|
|
37508
|
+
if (required.length === 0) return [];
|
|
37509
|
+
const evidence = ctx.evidence;
|
|
37510
|
+
const unmet = [];
|
|
37511
|
+
for (const key2 of required) {
|
|
37512
|
+
if (key2 === "sample-adequate") {
|
|
37513
|
+
const wanted = meta2.applicablePageTypes?.length ? meta2.applicablePageTypes : ["homepage"];
|
|
37514
|
+
if (!wanted.some((type) => evidence.usablePageTypes.has(type))) unmet.push(key2);
|
|
37515
|
+
continue;
|
|
37516
|
+
}
|
|
37517
|
+
if (!evidence.met[key2]) unmet.push(key2);
|
|
37518
|
+
}
|
|
37519
|
+
return unmet;
|
|
37520
|
+
}
|
|
37521
|
+
function gateExplanation(ctx, meta2, unmet) {
|
|
37522
|
+
const reasons = unmet.map((key2) => ctx.evidence.reasons[key2]).filter(Boolean);
|
|
37523
|
+
if (unmet.includes("sample-adequate") && reasons.length === 0) {
|
|
37524
|
+
const wanted = meta2.applicablePageTypes?.length ? meta2.applicablePageTypes.join("/") : "homepage";
|
|
37525
|
+
return `Not assessed: no scanned ${wanted} page served readable text.`;
|
|
37526
|
+
}
|
|
37527
|
+
const why = reasons.length > 0 ? ` ${reasons.join(" ")}` : "";
|
|
37528
|
+
return `Not assessed: this scan has no ${unmet.join(", ")} evidence.${why}`;
|
|
37529
|
+
}
|
|
37530
|
+
function planAudits(ctx, config, options = {}) {
|
|
37062
37531
|
const scannedPageTypes = new Set(ctx.pages.map((p) => p.pageType));
|
|
37063
37532
|
const runnable = [];
|
|
37064
37533
|
const skipped = [];
|
|
@@ -37078,6 +37547,15 @@ function planAudits(ctx, config) {
|
|
|
37078
37547
|
continue;
|
|
37079
37548
|
}
|
|
37080
37549
|
}
|
|
37550
|
+
if (options.enforceEvidence) {
|
|
37551
|
+
const unmet = unmetRequirements(ctx, reg2.meta);
|
|
37552
|
+
if (unmet.length > 0) {
|
|
37553
|
+
skipped.push(
|
|
37554
|
+
stubCheck(reg2.meta, TAG_SKIPPED_NO_EVIDENCE, gateExplanation(ctx, reg2.meta, unmet))
|
|
37555
|
+
);
|
|
37556
|
+
continue;
|
|
37557
|
+
}
|
|
37558
|
+
}
|
|
37081
37559
|
runnable.push({ reg: reg2, categoryId: cat.id });
|
|
37082
37560
|
}
|
|
37083
37561
|
}
|
|
@@ -37400,6 +37878,18 @@ function detectWafProtection(targetUrl, homepageResult, rootFiles, scannedPagesC
|
|
|
37400
37878
|
|
|
37401
37879
|
// src/orchestrator.ts
|
|
37402
37880
|
var A11Y_MAX_PAGES = Math.max(0, Number(process.env.SCANNER_A11Y_MAX_PAGES ?? 3));
|
|
37881
|
+
var RATE_LIMIT_BACKOFF_MS = 5e3;
|
|
37882
|
+
var MAX_RETRY_AFTER_MS = 3e4;
|
|
37883
|
+
async function fetchHomepage(fetcher, url, signal) {
|
|
37884
|
+
const first5 = await fetcher.fetch({ url, signal });
|
|
37885
|
+
if (first5.status !== 429) return first5;
|
|
37886
|
+
const header = Number(first5.headers["retry-after"]);
|
|
37887
|
+
const waitMs = Number.isFinite(header) && header > 0 ? Math.min(header * 1e3, MAX_RETRY_AFTER_MS) : RATE_LIMIT_BACKOFF_MS;
|
|
37888
|
+
logger.debug({ url, waitMs }, `[orchestrator] Homepage answered 429; retrying once in ${waitMs}ms`);
|
|
37889
|
+
await new Promise((resolve4) => setTimeout(resolve4, waitMs));
|
|
37890
|
+
signal?.throwIfAborted();
|
|
37891
|
+
return fetcher.fetch({ url, signal });
|
|
37892
|
+
}
|
|
37403
37893
|
function discoverPages(homepageUrl, domain, rootFiles, homepage$, exclude, maxAdditional) {
|
|
37404
37894
|
const discovered = /* @__PURE__ */ new Set();
|
|
37405
37895
|
const sitemapBody = rootFiles["/sitemap.xml"]?.status === 200 ? rootFiles["/sitemap.xml"].body : rootFiles["/sitemap-index.xml"]?.status === 200 ? rootFiles["/sitemap-index.xml"].body : "";
|
|
@@ -37567,7 +38057,7 @@ async function runScan(url, options) {
|
|
|
37567
38057
|
signal?.throwIfAborted();
|
|
37568
38058
|
logger.debug("[orchestrator] Phase 2: Fetching pages");
|
|
37569
38059
|
tracker.phaseStart("fetch-pages", 1);
|
|
37570
|
-
const homepageResult = await fetcher
|
|
38060
|
+
const homepageResult = await fetchHomepage(fetcher, url, signal);
|
|
37571
38061
|
tracker.unitDone(displayUrl);
|
|
37572
38062
|
const homepage$ = homepageResult.status === 200 && homepageResult.body ? parseHtml(homepageResult.body) : null;
|
|
37573
38063
|
const discoverLimit = Math.max(0, MAX_PAGES_PER_SCAN - 1 - overrideUrls.length);
|
|
@@ -37626,19 +38116,29 @@ async function runScan(url, options) {
|
|
|
37626
38116
|
signal?.throwIfAborted();
|
|
37627
38117
|
logger.debug("[orchestrator] Phase 3: Running audits");
|
|
37628
38118
|
const wafProtection = detectWafProtection(url, homepageResult, rootFiles, pages.length);
|
|
38119
|
+
const evidence = buildScanEvidence({
|
|
38120
|
+
requestedUrl: url,
|
|
38121
|
+
homepageResult,
|
|
38122
|
+
pages,
|
|
38123
|
+
rootFiles,
|
|
38124
|
+
wafProtection: wafProtection ?? null
|
|
38125
|
+
});
|
|
37629
38126
|
const ctx = {
|
|
37630
38127
|
rootFiles,
|
|
37631
38128
|
pages,
|
|
37632
38129
|
domain,
|
|
37633
38130
|
baseUrl,
|
|
37634
38131
|
fetch: (options2) => fetcher.fetch({ ...options2, signal }),
|
|
37635
|
-
wafProtection: wafProtection ?? void 0
|
|
38132
|
+
wafProtection: wafProtection ?? void 0,
|
|
38133
|
+
evidence
|
|
37636
38134
|
};
|
|
37637
38135
|
const config = filterConfig(defaultConfig, {
|
|
37638
38136
|
categories: options?.categories,
|
|
37639
38137
|
includeExperimental: options?.includeExperimental ?? false
|
|
37640
38138
|
});
|
|
37641
|
-
const auditPlan = planAudits(ctx, config
|
|
38139
|
+
const auditPlan = planAudits(ctx, config, {
|
|
38140
|
+
enforceEvidence: options?.enforceEvidenceGate ?? true
|
|
38141
|
+
});
|
|
37642
38142
|
tracker.phaseStart("audits", auditPlan.runnable.length);
|
|
37643
38143
|
const {
|
|
37644
38144
|
checks: allChecks,
|
|
@@ -37659,7 +38159,7 @@ async function runScan(url, options) {
|
|
|
37659
38159
|
tracker.phaseStart("report", 1);
|
|
37660
38160
|
logger.debug("[orchestrator] Phase 4: Building final report");
|
|
37661
38161
|
const durationMs = Math.round(performance.now() - start);
|
|
37662
|
-
const recommendations = allChecks.filter((c) => c.status
|
|
38162
|
+
const recommendations = allChecks.filter((c) => (c.status === "fail" || c.status === "warn") && !isInformative(c)).slice().sort((a, b) => {
|
|
37663
38163
|
const order = { critical: 0, high: 1, medium: 2, low: 3 };
|
|
37664
38164
|
return (order[a.priority] ?? 3) - (order[b.priority] ?? 3);
|
|
37665
38165
|
});
|
|
@@ -37671,13 +38171,23 @@ async function runScan(url, options) {
|
|
|
37671
38171
|
const readinessScore = Math.round(
|
|
37672
38172
|
readinessVitals.commerce * READINESS_WEIGHTS.commerce + readinessVitals.content * READINESS_WEIGHTS.content + readinessVitals.botAccessibility * READINESS_WEIGHTS.botAccessibility + readinessVitals.technical * READINESS_WEIGHTS.technical
|
|
37673
38173
|
);
|
|
38174
|
+
const gatedShare = gatedMassShare(allChecks);
|
|
38175
|
+
const escalated = gatedShare > GATED_MASS_UNSCORED_THRESHOLD;
|
|
38176
|
+
const unscoredReason = !evidence.judgeable ? Object.values(evidence.reasons).filter(Boolean).join(" ") || "The scan obtained too little evidence to judge this site." : escalated ? `The scan could not feed ${Math.round(gatedShare * 100)}% of the registry's evidence mass, so what remains is not a reading of this site.` : void 0;
|
|
38177
|
+
const scored = unscoredReason === void 0;
|
|
37674
38178
|
const report = {
|
|
37675
38179
|
scanId: "",
|
|
37676
38180
|
// Set by the caller
|
|
37677
38181
|
url: displayUrl,
|
|
37678
38182
|
domain,
|
|
37679
|
-
overallScore,
|
|
37680
|
-
scoreTier: getScoreTier(overallScore),
|
|
38183
|
+
overallScore: scored ? overallScore : null,
|
|
38184
|
+
scoreTier: scored ? getScoreTier(overallScore) : null,
|
|
38185
|
+
scanValidity: {
|
|
38186
|
+
judgeable: evidence.judgeable,
|
|
38187
|
+
evidence: evidence.met,
|
|
38188
|
+
reasons: evidence.reasons,
|
|
38189
|
+
...unscoredReason ? { unscoredReason } : {}
|
|
38190
|
+
},
|
|
37681
38191
|
summary: "",
|
|
37682
38192
|
// Set below
|
|
37683
38193
|
categories,
|
|
@@ -37698,7 +38208,7 @@ async function runScan(url, options) {
|
|
|
37698
38208
|
report.summary = generateScanSummary(report);
|
|
37699
38209
|
tracker.unitDone();
|
|
37700
38210
|
tracker.phaseDone();
|
|
37701
|
-
tracker.scanDone(overallScore);
|
|
38211
|
+
tracker.scanDone(report.overallScore);
|
|
37702
38212
|
logger.debug({ durationMs, score: overallScore }, "[orchestrator] runScan complete");
|
|
37703
38213
|
return report;
|
|
37704
38214
|
}
|
|
@@ -37868,6 +38378,7 @@ function loadConfigFile(customPath) {
|
|
|
37868
38378
|
DEFAULT_SCAN_LIMIT,
|
|
37869
38379
|
DeprecationNoticeSchema,
|
|
37870
38380
|
EvidenceGradeSchema,
|
|
38381
|
+
EvidenceKeySchema,
|
|
37871
38382
|
FixEffortSchema,
|
|
37872
38383
|
MAX_CONCURRENT_REQUESTS,
|
|
37873
38384
|
MAX_PAGES_PER_SCAN,
|
|
@@ -37883,9 +38394,12 @@ function loadConfigFile(customPath) {
|
|
|
37883
38394
|
SCORE_TIER_LABELS,
|
|
37884
38395
|
ScoreDisplayModeSchema,
|
|
37885
38396
|
TAG_SCAN_ERROR,
|
|
38397
|
+
TAG_SKIPPED_NO_EVIDENCE,
|
|
37886
38398
|
TAG_SKIPPED_PAGE_TYPE,
|
|
38399
|
+
allEvidenceMet,
|
|
37887
38400
|
allJsonLdNodes,
|
|
37888
38401
|
buildCategoryResult,
|
|
38402
|
+
buildScanEvidence,
|
|
37889
38403
|
calculateCategoryScore,
|
|
37890
38404
|
calculateOverallScore,
|
|
37891
38405
|
classifyFetch,
|
|
@@ -37916,6 +38430,7 @@ function loadConfigFile(customPath) {
|
|
|
37916
38430
|
formatTrace,
|
|
37917
38431
|
getMainContentText,
|
|
37918
38432
|
getPreset,
|
|
38433
|
+
getRenderedText,
|
|
37919
38434
|
getScoreTier,
|
|
37920
38435
|
getTierColor,
|
|
37921
38436
|
getTierLabel,
|