@forkpoint/agent-lighthouse-core 2.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +224 -10
- package/dist/index.d.ts +224 -10
- package/dist/index.js +1129 -152
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1123 -152
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -8,6 +8,7 @@ var MAX_CONCURRENT_REQUESTS = 10;
|
|
|
8
8
|
var SCANNER_USER_AGENT = "AgentLighthouse/1.0 (+https://github.com/ForkPoint/agent-lighthouse)";
|
|
9
9
|
var TAG_SKIPPED_PAGE_TYPE = "skipped:page-type";
|
|
10
10
|
var TAG_SCAN_ERROR = "scan-error";
|
|
11
|
+
var TAG_SKIPPED_NO_EVIDENCE = "skipped:no-evidence";
|
|
11
12
|
var CATEGORY_NAMES = {
|
|
12
13
|
"access-crawl-control": "Access & Crawl Control",
|
|
13
14
|
"content-extraction": "Content Extraction",
|
|
@@ -181,8 +182,40 @@ async function isSafeUrl(url) {
|
|
|
181
182
|
return false;
|
|
182
183
|
}
|
|
183
184
|
}
|
|
184
|
-
function
|
|
185
|
+
function createGate(limit) {
|
|
186
|
+
let inFlight = 0;
|
|
187
|
+
const waiting = [];
|
|
188
|
+
const release2 = () => {
|
|
189
|
+
inFlight -= 1;
|
|
190
|
+
const next = waiting.shift();
|
|
191
|
+
if (next) next();
|
|
192
|
+
};
|
|
193
|
+
return {
|
|
194
|
+
acquire: async () => {
|
|
195
|
+
if (inFlight >= limit) {
|
|
196
|
+
await new Promise((resolve4) => waiting.push(resolve4));
|
|
197
|
+
}
|
|
198
|
+
inFlight += 1;
|
|
199
|
+
return release2;
|
|
200
|
+
}
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
function boundedDispatcher(connections) {
|
|
204
|
+
return new Agent({ connections });
|
|
205
|
+
}
|
|
206
|
+
function createFetcher(fetcherOptions = {}) {
|
|
207
|
+
const dispatcher = fetcherOptions.dispatcher ?? noRedirectAgent;
|
|
208
|
+
const gate = fetcherOptions.maxConcurrent && fetcherOptions.maxConcurrent > 0 ? createGate(Math.floor(fetcherOptions.maxConcurrent)) : void 0;
|
|
185
209
|
async function fetch(options) {
|
|
210
|
+
if (!gate) return issue(options);
|
|
211
|
+
const release2 = await gate.acquire();
|
|
212
|
+
try {
|
|
213
|
+
return await issue(options);
|
|
214
|
+
} finally {
|
|
215
|
+
release2();
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
async function issue(options) {
|
|
186
219
|
const {
|
|
187
220
|
url,
|
|
188
221
|
timeout = REQUEST_TIMEOUT_MS,
|
|
@@ -222,10 +255,11 @@ function createFetcher() {
|
|
|
222
255
|
headers: reqHeaders,
|
|
223
256
|
body: currentBody,
|
|
224
257
|
signal,
|
|
225
|
-
dispatcher
|
|
258
|
+
dispatcher
|
|
226
259
|
});
|
|
227
260
|
let gateArmed;
|
|
228
261
|
let hops = 0;
|
|
262
|
+
const redirectChain = [];
|
|
229
263
|
while (followRedirects && REDIRECT_STATUS.has(response.statusCode) && response.headers["location"] !== void 0 && hops < MAX_REDIRECTS) {
|
|
230
264
|
const rawLocation = response.headers["location"];
|
|
231
265
|
const location = Array.isArray(rawLocation) ? rawLocation[0] : rawLocation;
|
|
@@ -261,6 +295,7 @@ function createFetcher() {
|
|
|
261
295
|
currentMethod = "GET";
|
|
262
296
|
currentBody = void 0;
|
|
263
297
|
}
|
|
298
|
+
redirectChain.push({ status: response.statusCode, from: currentUrl, to: next });
|
|
264
299
|
currentUrl = next;
|
|
265
300
|
hops += 1;
|
|
266
301
|
response = await request(currentUrl, {
|
|
@@ -268,7 +303,7 @@ function createFetcher() {
|
|
|
268
303
|
headers: reqHeaders,
|
|
269
304
|
body: currentBody,
|
|
270
305
|
signal,
|
|
271
|
-
dispatcher
|
|
306
|
+
dispatcher
|
|
272
307
|
});
|
|
273
308
|
}
|
|
274
309
|
ttfbMs = performance.now() - start;
|
|
@@ -307,7 +342,8 @@ function createFetcher() {
|
|
|
307
342
|
totalMs: Math.round(totalMs),
|
|
308
343
|
contentType: headers["content-type"] ?? "",
|
|
309
344
|
contentLength: bytes ? bytes.byteLength : truncatedBody.length,
|
|
310
|
-
...bytes ? { bytes } : {}
|
|
345
|
+
...bytes ? { bytes } : {},
|
|
346
|
+
...redirectChain.length > 0 ? { redirectChain } : {}
|
|
311
347
|
};
|
|
312
348
|
} catch (err) {
|
|
313
349
|
const totalMs = performance.now() - start;
|
|
@@ -556,12 +592,23 @@ function extractHeadings($) {
|
|
|
556
592
|
});
|
|
557
593
|
return headings;
|
|
558
594
|
}
|
|
559
|
-
function
|
|
560
|
-
const root = $("main").first().length ? $("main").first() : $("body");
|
|
595
|
+
function readableText(root) {
|
|
561
596
|
const clone = root.clone();
|
|
562
597
|
clone.find("script, style, noscript, template").remove();
|
|
563
598
|
return clone.text().replace(/\s+/g, " ").trim();
|
|
564
599
|
}
|
|
600
|
+
function getMainContentText($) {
|
|
601
|
+
let best = "";
|
|
602
|
+
$("body").find("main").each((_, el) => {
|
|
603
|
+
const text3 = readableText($(el));
|
|
604
|
+
if (text3.length > best.length) best = text3;
|
|
605
|
+
});
|
|
606
|
+
if (best) return best;
|
|
607
|
+
return readableText($("body"));
|
|
608
|
+
}
|
|
609
|
+
function getRenderedText($) {
|
|
610
|
+
return readableText($("body"));
|
|
611
|
+
}
|
|
565
612
|
function getWordCount($) {
|
|
566
613
|
const text3 = getMainContentText($);
|
|
567
614
|
return text3.split(/\s+/).filter(Boolean).length;
|
|
@@ -795,6 +842,12 @@ var DeprecationNoticeSchema = z.object({
|
|
|
795
842
|
var EvidenceGradeSchema = z.enum(["A", "B", "C", "D"]);
|
|
796
843
|
var AuditTierSchema = z.enum(["scored", "informative", "experimental"]);
|
|
797
844
|
var AUDIT_ID_PATTERN = /^[a-z-]+\/[a-z0-9-]+$/;
|
|
845
|
+
var EvidenceKeySchema = z.enum([
|
|
846
|
+
"origin-reachable",
|
|
847
|
+
"unblocked-fetches",
|
|
848
|
+
"rendered-body",
|
|
849
|
+
"sample-adequate"
|
|
850
|
+
]);
|
|
798
851
|
var AuditMetaSchema = z.object({
|
|
799
852
|
id: z.string().regex(AUDIT_ID_PATTERN, "audit id must be a `category/slug` path"),
|
|
800
853
|
category: z.string(),
|
|
@@ -813,7 +866,10 @@ var AuditMetaSchema = z.object({
|
|
|
813
866
|
// its weight comes from (grade + tier) and which dossier proves it.
|
|
814
867
|
evidenceGrade: EvidenceGradeSchema,
|
|
815
868
|
tier: AuditTierSchema,
|
|
816
|
-
dossier: z.string().min(1).max(500)
|
|
869
|
+
dossier: z.string().min(1).max(500),
|
|
870
|
+
// What the audit needs the scan to have obtained. Checked against the
|
|
871
|
+
// source by `scripts/check-requires.mjs`, not enforced here beyond shape.
|
|
872
|
+
requires: z.array(EvidenceKeySchema).optional()
|
|
817
873
|
});
|
|
818
874
|
var CheckResultSchema = z.object({
|
|
819
875
|
// v2 ids are `category/slug` paths, which outgrew the old 20-char cap.
|
|
@@ -1038,6 +1094,210 @@ function calculateOverallScore(categories) {
|
|
|
1038
1094
|
if (totalMass === 0) return 0;
|
|
1039
1095
|
return Math.round(weighted / totalMass);
|
|
1040
1096
|
}
|
|
1097
|
+
var GATED_MASS_UNSCORED_THRESHOLD = 0.35;
|
|
1098
|
+
function gatedMassShare(checks2) {
|
|
1099
|
+
let gated = 0;
|
|
1100
|
+
let total = 0;
|
|
1101
|
+
for (const check of checks2) {
|
|
1102
|
+
if (isInformative(check)) continue;
|
|
1103
|
+
const mass = check.weight ?? 0;
|
|
1104
|
+
if (mass <= 0) continue;
|
|
1105
|
+
total += mass;
|
|
1106
|
+
if (check.tags?.includes(TAG_SKIPPED_NO_EVIDENCE)) gated += mass;
|
|
1107
|
+
}
|
|
1108
|
+
return total === 0 ? 0 : gated / total;
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
// src/gatherers/domains.ts
|
|
1112
|
+
var MULTI_SUFFIX = /* @__PURE__ */ new Set([
|
|
1113
|
+
"co.uk",
|
|
1114
|
+
"org.uk",
|
|
1115
|
+
"ac.uk",
|
|
1116
|
+
"gov.uk",
|
|
1117
|
+
"me.uk",
|
|
1118
|
+
"net.uk",
|
|
1119
|
+
"com.au",
|
|
1120
|
+
"net.au",
|
|
1121
|
+
"org.au",
|
|
1122
|
+
"edu.au",
|
|
1123
|
+
"gov.au",
|
|
1124
|
+
"co.nz",
|
|
1125
|
+
"co.jp",
|
|
1126
|
+
"or.jp",
|
|
1127
|
+
"ne.jp",
|
|
1128
|
+
"co.za",
|
|
1129
|
+
"co.kr",
|
|
1130
|
+
"co.il",
|
|
1131
|
+
"co.id",
|
|
1132
|
+
"co.th",
|
|
1133
|
+
"com.br",
|
|
1134
|
+
"com.mx",
|
|
1135
|
+
"com.ar",
|
|
1136
|
+
"com.co",
|
|
1137
|
+
"com.pe",
|
|
1138
|
+
"co.in",
|
|
1139
|
+
"com.sg",
|
|
1140
|
+
"com.tr",
|
|
1141
|
+
"com.cn",
|
|
1142
|
+
"com.hk",
|
|
1143
|
+
"com.tw",
|
|
1144
|
+
"com.my",
|
|
1145
|
+
"com.ph",
|
|
1146
|
+
"com.ua",
|
|
1147
|
+
"com.pl",
|
|
1148
|
+
"com.es",
|
|
1149
|
+
"com.pt",
|
|
1150
|
+
"com.gr"
|
|
1151
|
+
]);
|
|
1152
|
+
function registrableDomain(host) {
|
|
1153
|
+
const parts = host.toLowerCase().replace(/\.$/, "").split(".").filter(Boolean);
|
|
1154
|
+
if (parts.length <= 2) return parts.join(".");
|
|
1155
|
+
const lastTwo = parts.slice(-2).join(".");
|
|
1156
|
+
return MULTI_SUFFIX.has(lastTwo) ? parts.slice(-3).join(".") : lastTwo;
|
|
1157
|
+
}
|
|
1158
|
+
function registrableOf(url) {
|
|
1159
|
+
try {
|
|
1160
|
+
return registrableDomain(new URL(url).hostname);
|
|
1161
|
+
} catch {
|
|
1162
|
+
return "";
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
// src/scan-evidence.ts
|
|
1167
|
+
var ALL_PAGE_TYPES = ["homepage", "category", "product", "content"];
|
|
1168
|
+
var HTML_TYPES = ["text/html", "application/xhtml+xml"];
|
|
1169
|
+
var PERMANENT_REDIRECT = /* @__PURE__ */ new Set([301, 308]);
|
|
1170
|
+
function bareHost(url) {
|
|
1171
|
+
try {
|
|
1172
|
+
return new URL(url).hostname.toLowerCase().replace(/^www\./, "");
|
|
1173
|
+
} catch {
|
|
1174
|
+
return "";
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
function registrableName(url) {
|
|
1178
|
+
const domain = registrableOf(url);
|
|
1179
|
+
if (!domain) return "";
|
|
1180
|
+
const parts = domain.split(".");
|
|
1181
|
+
return parts.length > 1 ? parts.slice(0, -1).join(".") : domain;
|
|
1182
|
+
}
|
|
1183
|
+
function reachedTheRequestedSite(requestedUrl, result) {
|
|
1184
|
+
const requested = bareHost(requestedUrl);
|
|
1185
|
+
const final = bareHost(result.finalUrl || result.url);
|
|
1186
|
+
if (!final) return { ok: false, reason: `The homepage response carried no usable URL.` };
|
|
1187
|
+
if (requested === final) return { ok: true };
|
|
1188
|
+
const requestedDomain = registrableOf(requestedUrl);
|
|
1189
|
+
const finalDomain = registrableOf(result.finalUrl || result.url);
|
|
1190
|
+
if (requestedDomain && requestedDomain === finalDomain) return { ok: true };
|
|
1191
|
+
const requestedName = registrableName(requestedUrl);
|
|
1192
|
+
if (requestedName && requestedName === registrableName(result.finalUrl || result.url)) {
|
|
1193
|
+
return { ok: true };
|
|
1194
|
+
}
|
|
1195
|
+
const chain = result.redirectChain ?? [];
|
|
1196
|
+
const leaving = chain.filter((hop) => registrableOf(hop.from) !== registrableOf(hop.to));
|
|
1197
|
+
if (leaving.length > 0 && leaving.every((hop) => PERMANENT_REDIRECT.has(hop.status))) {
|
|
1198
|
+
return { ok: true };
|
|
1199
|
+
}
|
|
1200
|
+
return {
|
|
1201
|
+
ok: false,
|
|
1202
|
+
reason: `The requested host redirected to ${final}, a different site, without a permanent redirect.`
|
|
1203
|
+
};
|
|
1204
|
+
}
|
|
1205
|
+
function originReachable(requestedUrl, result) {
|
|
1206
|
+
if (result.error) {
|
|
1207
|
+
return { met: false, reason: `The homepage could not be fetched: ${result.error}.` };
|
|
1208
|
+
}
|
|
1209
|
+
if (result.status < 200 || result.status > 299) {
|
|
1210
|
+
return { met: false, reason: `The homepage answered HTTP ${result.status}.` };
|
|
1211
|
+
}
|
|
1212
|
+
const type = (result.contentType || "").toLowerCase();
|
|
1213
|
+
if (!HTML_TYPES.some((html) => type.includes(html))) {
|
|
1214
|
+
return {
|
|
1215
|
+
met: false,
|
|
1216
|
+
reason: `The homepage served ${result.contentType || "no content type"}, not HTML.`
|
|
1217
|
+
};
|
|
1218
|
+
}
|
|
1219
|
+
const reached = reachedTheRequestedSite(requestedUrl, result);
|
|
1220
|
+
return reached.ok ? { met: true } : { met: false, reason: reached.reason };
|
|
1221
|
+
}
|
|
1222
|
+
function unblockedFetches(homepageResult, waf) {
|
|
1223
|
+
if (waf?.isBlocked) {
|
|
1224
|
+
return waf.isRateLimit ? {
|
|
1225
|
+
met: false,
|
|
1226
|
+
reason: `The scan was throttled (${waf.name}): ${waf.reason}.`
|
|
1227
|
+
} : { met: false, reason: `${waf.name} refused the scan: ${waf.reason}.` };
|
|
1228
|
+
}
|
|
1229
|
+
if (homepageResult.status === 429) {
|
|
1230
|
+
return { met: false, reason: "The homepage answered HTTP 429: the scan was throttled." };
|
|
1231
|
+
}
|
|
1232
|
+
return { met: true };
|
|
1233
|
+
}
|
|
1234
|
+
function pageRendersText(page) {
|
|
1235
|
+
const text3 = getRenderedText(page.$);
|
|
1236
|
+
const wordCount2 = text3.split(/\s+/).filter(Boolean).length;
|
|
1237
|
+
return wordCount2 > 50 || text3.length > 200;
|
|
1238
|
+
}
|
|
1239
|
+
function buildScanEvidence(input) {
|
|
1240
|
+
const origin = originReachable(input.requestedUrl, input.homepageResult);
|
|
1241
|
+
const unblocked = unblockedFetches(input.homepageResult, input.wafProtection);
|
|
1242
|
+
const renderedByPage = {};
|
|
1243
|
+
const usablePageTypes = /* @__PURE__ */ new Set();
|
|
1244
|
+
for (const page of input.pages) {
|
|
1245
|
+
const rendered = pageRendersText(page);
|
|
1246
|
+
renderedByPage[page.url] = rendered;
|
|
1247
|
+
if (rendered) usablePageTypes.add(page.pageType);
|
|
1248
|
+
}
|
|
1249
|
+
const renderedCount = Object.values(renderedByPage).filter(Boolean).length;
|
|
1250
|
+
const met = {
|
|
1251
|
+
"origin-reachable": origin.met,
|
|
1252
|
+
"unblocked-fetches": unblocked.met,
|
|
1253
|
+
"rendered-body": renderedCount > 0,
|
|
1254
|
+
"sample-adequate": usablePageTypes.size > 0
|
|
1255
|
+
};
|
|
1256
|
+
const reasons = {};
|
|
1257
|
+
if (origin.reason) reasons["origin-reachable"] = origin.reason;
|
|
1258
|
+
if (unblocked.reason) reasons["unblocked-fetches"] = unblocked.reason;
|
|
1259
|
+
if (!met["rendered-body"]) {
|
|
1260
|
+
reasons["rendered-body"] = input.pages.length === 0 ? "The scan fetched no pages." : `None of the ${input.pages.length} fetched page(s) served readable text.`;
|
|
1261
|
+
}
|
|
1262
|
+
if (!met["sample-adequate"]) {
|
|
1263
|
+
reasons["sample-adequate"] = input.pages.length === 0 ? "The scan fetched no pages." : "No fetched page of any type served readable text.";
|
|
1264
|
+
}
|
|
1265
|
+
return {
|
|
1266
|
+
met,
|
|
1267
|
+
reasons,
|
|
1268
|
+
renderedByPage,
|
|
1269
|
+
usablePageTypes,
|
|
1270
|
+
// A shell site was seen. What it serves is a finding about it, so
|
|
1271
|
+
// `rendered-body` and `sample-adequate` do not clear `judgeable`.
|
|
1272
|
+
judgeable: met["origin-reachable"] && met["unblocked-fetches"]
|
|
1273
|
+
};
|
|
1274
|
+
}
|
|
1275
|
+
function scanReadTheSite(evidence) {
|
|
1276
|
+
return evidence.judgeable;
|
|
1277
|
+
}
|
|
1278
|
+
function unreadSiteReason(evidence) {
|
|
1279
|
+
return evidence.reasons["origin-reachable"] ?? evidence.reasons["unblocked-fetches"] ?? "The scan obtained no response it could attribute to this site.";
|
|
1280
|
+
}
|
|
1281
|
+
function scanReadPageText(evidence) {
|
|
1282
|
+
return evidence.met["rendered-body"];
|
|
1283
|
+
}
|
|
1284
|
+
function unreadPageTextReason(evidence) {
|
|
1285
|
+
return evidence.reasons["rendered-body"] ?? "No fetched page served text a non-JS consumer can read.";
|
|
1286
|
+
}
|
|
1287
|
+
function allEvidenceMet() {
|
|
1288
|
+
return {
|
|
1289
|
+
met: {
|
|
1290
|
+
"origin-reachable": true,
|
|
1291
|
+
"unblocked-fetches": true,
|
|
1292
|
+
"rendered-body": true,
|
|
1293
|
+
"sample-adequate": true
|
|
1294
|
+
},
|
|
1295
|
+
reasons: {},
|
|
1296
|
+
renderedByPage: {},
|
|
1297
|
+
usablePageTypes: new Set(ALL_PAGE_TYPES),
|
|
1298
|
+
judgeable: true
|
|
1299
|
+
};
|
|
1300
|
+
}
|
|
1041
1301
|
|
|
1042
1302
|
// src/audits/access-crawl-control/no-nofollow.ts
|
|
1043
1303
|
var NoNofollowAudit = class _NoNofollowAudit extends Audit {
|
|
@@ -1052,6 +1312,9 @@ var NoNofollowAudit = class _NoNofollowAudit extends Audit {
|
|
|
1052
1312
|
evidenceGrade: "A",
|
|
1053
1313
|
tier: "scored",
|
|
1054
1314
|
dossier: "docs/evidence/audits/access-crawl-control/no-nofollow.md",
|
|
1315
|
+
// Gate exemption: being refused is what this category reports, and the meta tag and
|
|
1316
|
+
// header this audit reads are served by a page whose body renders nothing.
|
|
1317
|
+
requires: ["origin-reachable"],
|
|
1055
1318
|
defaultPriority: "high",
|
|
1056
1319
|
guidance: {
|
|
1057
1320
|
impact: "A nofollow directive prevents AI crawlers from following links on your pages, effectively hiding all linked content from AI indexing. Your deeper pages become invisible to AI search engines, drastically reducing discoverability.",
|
|
@@ -1063,6 +1326,13 @@ var NoNofollowAudit = class _NoNofollowAudit extends Audit {
|
|
|
1063
1326
|
}
|
|
1064
1327
|
};
|
|
1065
1328
|
audit(ctx) {
|
|
1329
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
1330
|
+
return this.notApplicable(
|
|
1331
|
+
"No page here can be attributed to this site, so its nofollow directives were not judged.",
|
|
1332
|
+
"No site-wide nofollow directives",
|
|
1333
|
+
unreadSiteReason(ctx.evidence)
|
|
1334
|
+
);
|
|
1335
|
+
}
|
|
1066
1336
|
if (ctx.pages.length === 0) {
|
|
1067
1337
|
return this.fail(
|
|
1068
1338
|
"No pages scanned.",
|
|
@@ -1133,6 +1403,10 @@ var NoRedirectChainsAudit = class _NoRedirectChainsAudit extends Audit {
|
|
|
1133
1403
|
evidenceGrade: "A",
|
|
1134
1404
|
tier: "scored",
|
|
1135
1405
|
dossier: "docs/evidence/audits/access-crawl-control/no-redirect-chains.md",
|
|
1406
|
+
// Gate exemption: a hop that left the site is this audit's subject, and leaving the
|
|
1407
|
+
// site is exactly what denies `origin-reachable`. It reads request URL against final
|
|
1408
|
+
// URL, which every response carries, and reports "no pages scanned" itself.
|
|
1409
|
+
requires: [],
|
|
1136
1410
|
defaultPriority: "medium",
|
|
1137
1411
|
guidance: {
|
|
1138
1412
|
impact: "Redirect chains slow down AI crawlers and waste their limited crawl budget. Each extra redirect adds latency and increases the chance a crawler gives up before reaching the final page, leaving content unindexed.",
|
|
@@ -1143,17 +1417,6 @@ var NoRedirectChainsAudit = class _NoRedirectChainsAudit extends Audit {
|
|
|
1143
1417
|
}
|
|
1144
1418
|
};
|
|
1145
1419
|
audit(ctx) {
|
|
1146
|
-
if (ctx.pages.length === 0) {
|
|
1147
|
-
return this.fail(
|
|
1148
|
-
"No pages scanned.",
|
|
1149
|
-
"No redirect chains (URL equals finalUrl or single redirect)",
|
|
1150
|
-
"No pages scanned",
|
|
1151
|
-
{
|
|
1152
|
-
priority: "medium",
|
|
1153
|
-
description: _NoRedirectChainsAudit.meta.description
|
|
1154
|
-
}
|
|
1155
|
-
);
|
|
1156
|
-
}
|
|
1157
1420
|
const redirected = [];
|
|
1158
1421
|
for (const page of ctx.pages) {
|
|
1159
1422
|
const requestUrl = page.fetchResult.url;
|
|
@@ -1163,6 +1426,24 @@ var NoRedirectChainsAudit = class _NoRedirectChainsAudit extends Audit {
|
|
|
1163
1426
|
}
|
|
1164
1427
|
}
|
|
1165
1428
|
if (redirected.length === 0) {
|
|
1429
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
1430
|
+
return this.notApplicable(
|
|
1431
|
+
"No page here can be attributed to this site, so its redirect behaviour was not judged.",
|
|
1432
|
+
"No redirect chains",
|
|
1433
|
+
unreadSiteReason(ctx.evidence)
|
|
1434
|
+
);
|
|
1435
|
+
}
|
|
1436
|
+
if (ctx.pages.length === 0) {
|
|
1437
|
+
return this.fail(
|
|
1438
|
+
"No pages scanned.",
|
|
1439
|
+
"No redirect chains (URL equals finalUrl or single redirect)",
|
|
1440
|
+
"No pages scanned",
|
|
1441
|
+
{
|
|
1442
|
+
priority: "medium",
|
|
1443
|
+
description: _NoRedirectChainsAudit.meta.description
|
|
1444
|
+
}
|
|
1445
|
+
);
|
|
1446
|
+
}
|
|
1166
1447
|
return this.pass(
|
|
1167
1448
|
`All ${ctx.pages.length} page(s) resolve without redirects.`,
|
|
1168
1449
|
"No redirect chains",
|
|
@@ -1278,6 +1559,8 @@ var CanonicalLinksAudit = class extends Audit {
|
|
|
1278
1559
|
evidenceGrade: "A",
|
|
1279
1560
|
tier: "scored",
|
|
1280
1561
|
dossier: "docs/evidence/audits/access-crawl-control/canonical.md",
|
|
1562
|
+
// Gate exemption: being refused is what this category reports.
|
|
1563
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
1281
1564
|
defaultPriority: "medium",
|
|
1282
1565
|
guidance: {
|
|
1283
1566
|
impact: "A canonical pointing at the wrong URL is worse than no canonical at all: when every page canonicalizes onto the homepage \u2014 a common CMS and SPA template bug \u2014 the pages consolidate onto one URL and drop out of the index that AI Overviews and AI Mode draw on. A canonical pointing at another domain hands the attribution there.",
|
|
@@ -1290,6 +1573,13 @@ var CanonicalLinksAudit = class extends Audit {
|
|
|
1290
1573
|
};
|
|
1291
1574
|
audit(ctx) {
|
|
1292
1575
|
const expected = "Each page declares a canonical URL that points at itself";
|
|
1576
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
1577
|
+
return this.notApplicable(
|
|
1578
|
+
"No page here can be attributed to this site, so its canonical links were not judged.",
|
|
1579
|
+
expected,
|
|
1580
|
+
unreadSiteReason(ctx.evidence)
|
|
1581
|
+
);
|
|
1582
|
+
}
|
|
1293
1583
|
if (ctx.pages.length === 0) {
|
|
1294
1584
|
return this.notApplicable(
|
|
1295
1585
|
"No pages were scanned, so no canonical links could be read.",
|
|
@@ -1726,6 +2016,8 @@ var GptbotAudit = class extends CrawlerBotAudit {
|
|
|
1726
2016
|
evidenceGrade: "A",
|
|
1727
2017
|
tier: "scored",
|
|
1728
2018
|
dossier: "docs/evidence/audits/access-crawl-control/gptbot.md",
|
|
2019
|
+
// Gate exemption: being refused is what this category reports.
|
|
2020
|
+
requires: ["origin-reachable"],
|
|
1729
2021
|
defaultPriority: "medium",
|
|
1730
2022
|
guidance: {
|
|
1731
2023
|
impact: "Blocking GPTBot prevents your content from being used by OpenAI's models and appearing in ChatGPT responses. Explicitly allowing it signals that your site welcomes AI indexing for the largest AI platform by user base.",
|
|
@@ -1756,6 +2048,8 @@ var GoogleExtendedAudit = class extends CrawlerBotAudit {
|
|
|
1756
2048
|
evidenceGrade: "A",
|
|
1757
2049
|
tier: "scored",
|
|
1758
2050
|
dossier: "docs/evidence/audits/access-crawl-control/google-extended.md",
|
|
2051
|
+
// Gate exemption: being refused is what this category reports.
|
|
2052
|
+
requires: ["origin-reachable"],
|
|
1759
2053
|
defaultPriority: "medium",
|
|
1760
2054
|
guidance: {
|
|
1761
2055
|
impact: "Blocking Google-Extended prevents your content from being used in Google's AI features like Gemini and AI Overviews. Allowing it ensures your site appears in Google's AI-powered search experiences alongside traditional results.",
|
|
@@ -1790,6 +2084,8 @@ var AnthropicAudit = class extends CrawlerBotAudit {
|
|
|
1790
2084
|
evidenceGrade: "A",
|
|
1791
2085
|
tier: "scored",
|
|
1792
2086
|
dossier: "docs/evidence/audits/access-crawl-control/anthropic-ai.md",
|
|
2087
|
+
// Gate exemption: being refused is what this category reports.
|
|
2088
|
+
requires: ["origin-reachable"],
|
|
1793
2089
|
defaultPriority: "medium",
|
|
1794
2090
|
guidance: {
|
|
1795
2091
|
impact: "Disallowing ClaudeBot keeps the site out of the web content Anthropic collects for potential model training. It is an effective, documented control, so it is only a problem where the block was not intended. It buys back very little traffic either way: Cloudflare Radar measures Anthropic's crawl-to-refer ratio at roughly 50,000:1, so the allow-side case is about corpus inclusion rather than referral visibility.",
|
|
@@ -1899,6 +2195,8 @@ var PerplexitybotAudit = class extends CrawlerBotAudit {
|
|
|
1899
2195
|
evidenceGrade: "A",
|
|
1900
2196
|
tier: "scored",
|
|
1901
2197
|
dossier: "docs/evidence/audits/access-crawl-control/perplexitybot.md",
|
|
2198
|
+
// Gate exemption: being refused is what this category reports.
|
|
2199
|
+
requires: ["origin-reachable"],
|
|
1902
2200
|
defaultPriority: "medium",
|
|
1903
2201
|
guidance: {
|
|
1904
2202
|
impact: "Blocking PerplexityBot prevents your content from appearing in Perplexity AI search results, one of the fastest-growing AI answer engines. Allowing it gives your content visibility in AI-native search.",
|
|
@@ -1929,6 +2227,8 @@ var ApplebotExtendedAudit = class extends CrawlerBotAudit {
|
|
|
1929
2227
|
evidenceGrade: "A",
|
|
1930
2228
|
tier: "scored",
|
|
1931
2229
|
dossier: "docs/evidence/audits/access-crawl-control/applebot-extended.md",
|
|
2230
|
+
// Gate exemption: being refused is what this category reports.
|
|
2231
|
+
requires: ["origin-reachable"],
|
|
1932
2232
|
defaultPriority: "medium",
|
|
1933
2233
|
guidance: {
|
|
1934
2234
|
impact: "Blocking Applebot-Extended prevents your content from being used in Apple Intelligence features, Siri AI answers, and Safari Highlights. Allowing it ensures visibility across Apple's AI ecosystem.",
|
|
@@ -1959,6 +2259,8 @@ var CcbotAudit = class extends CrawlerBotAudit {
|
|
|
1959
2259
|
evidenceGrade: "A",
|
|
1960
2260
|
tier: "scored",
|
|
1961
2261
|
dossier: "docs/evidence/audits/access-crawl-control/ccbot.md",
|
|
2262
|
+
// Gate exemption: being refused is what this category reports.
|
|
2263
|
+
requires: ["origin-reachable"],
|
|
1962
2264
|
defaultPriority: "medium",
|
|
1963
2265
|
guidance: {
|
|
1964
2266
|
impact: "Blocking CCBot prevents your content from being included in the Common Crawl dataset, which is a foundational training data source for many AI models. Allowing it broadens your content's reach across multiple AI systems.",
|
|
@@ -1990,6 +2292,8 @@ var MetaExternalAgentAudit = class extends CrawlerBotAudit {
|
|
|
1990
2292
|
evidenceGrade: "A",
|
|
1991
2293
|
tier: "scored",
|
|
1992
2294
|
dossier: "docs/evidence/audits/access-crawl-control/meta-external-agent.md",
|
|
2295
|
+
// Gate exemption: being refused is what this category reports.
|
|
2296
|
+
requires: ["origin-reachable"],
|
|
1993
2297
|
defaultPriority: "medium",
|
|
1994
2298
|
guidance: {
|
|
1995
2299
|
impact: "Disallowing Meta-ExternalAgent keeps the site out of Meta's foundation-model training corpus and out of the direct content indexing that improves Meta products. It is an effective, documented control, so it is only a problem where the block was not intended. It does not by itself govern Meta AI search citations \u2014 Meta documents Meta-WebIndexer as the token behind those.",
|
|
@@ -2082,6 +2386,8 @@ var AmazonbotAudit = class extends CrawlerBotAudit {
|
|
|
2082
2386
|
evidenceGrade: "A",
|
|
2083
2387
|
tier: "scored",
|
|
2084
2388
|
dossier: "docs/evidence/audits/access-crawl-control/amazonbot.md",
|
|
2389
|
+
// Gate exemption: being refused is what this category reports.
|
|
2390
|
+
requires: ["origin-reachable"],
|
|
2085
2391
|
defaultPriority: "medium",
|
|
2086
2392
|
guidance: {
|
|
2087
2393
|
impact: "Blocking Amazonbot prevents your content from appearing in Alexa AI answers and Amazon's AI-powered search features. Allowing it gives your content visibility in Amazon's voice and commerce AI ecosystem.",
|
|
@@ -2156,6 +2462,8 @@ var AiBotDirectivesAudit = class extends Audit {
|
|
|
2156
2462
|
evidenceGrade: "B",
|
|
2157
2463
|
tier: "scored",
|
|
2158
2464
|
dossier: "docs/evidence/audits/access-crawl-control/ai-bot-directives.md",
|
|
2465
|
+
// Gate exemption: being refused is what this category reports.
|
|
2466
|
+
requires: ["origin-reachable"],
|
|
2159
2467
|
defaultPriority: "medium",
|
|
2160
2468
|
guidance: {
|
|
2161
2469
|
impact: "Blocking YouBot removes the site from You.com's live search index; blocking AI2Bot removes it from the Allen Institute's open training corpora while leaving closed commercial crawlers untouched. Leaving either to the wildcard rule means the policy silently flips the day a blanket block is added. The other three tokens carry no comparable consumer, so this audit never penalises blocking them.",
|
|
@@ -2222,6 +2530,8 @@ var ChatgptUserAudit = class extends CrawlerBotAudit {
|
|
|
2222
2530
|
evidenceGrade: "C",
|
|
2223
2531
|
tier: "informative",
|
|
2224
2532
|
dossier: "docs/evidence/audits/access-crawl-control/chatgpt-user.md",
|
|
2533
|
+
// Gate exemption: being refused is what this category reports.
|
|
2534
|
+
requires: ["origin-reachable"],
|
|
2225
2535
|
defaultPriority: "medium",
|
|
2226
2536
|
guidance: {
|
|
2227
2537
|
impact: "Blocking ChatGPT-User prevents ChatGPT from browsing your site in real-time when users ask it to visit your pages. This blocks your content from being cited in ChatGPT Browse conversations, losing a significant source of AI-driven traffic.",
|
|
@@ -2252,6 +2562,8 @@ var ClaudeUserAudit = class extends CrawlerBotAudit {
|
|
|
2252
2562
|
evidenceGrade: "A",
|
|
2253
2563
|
tier: "scored",
|
|
2254
2564
|
dossier: "docs/evidence/audits/access-crawl-control/claude-user.md",
|
|
2565
|
+
// Gate exemption: being refused is what this category reports.
|
|
2566
|
+
requires: ["origin-reachable"],
|
|
2255
2567
|
defaultPriority: "medium",
|
|
2256
2568
|
guidance: {
|
|
2257
2569
|
impact: "Blocking Claude-User prevents Claude from browsing your site in real-time when users ask it to visit your pages. This blocks your content from being cited in Claude conversations with web access enabled.",
|
|
@@ -2281,6 +2593,8 @@ var OaiSearchbotAudit = class extends CrawlerBotAudit {
|
|
|
2281
2593
|
evidenceGrade: "A",
|
|
2282
2594
|
tier: "scored",
|
|
2283
2595
|
dossier: "docs/evidence/audits/access-crawl-control/oai-searchbot.md",
|
|
2596
|
+
// Gate exemption: being refused is what this category reports.
|
|
2597
|
+
requires: ["origin-reachable"],
|
|
2284
2598
|
defaultPriority: "medium",
|
|
2285
2599
|
guidance: {
|
|
2286
2600
|
impact: "Blocking OAI-SearchBot prevents your content from appearing in OpenAI's SearchGPT and ChatGPT web search results. Allowing it ensures your site is discoverable through OpenAI's real-time search features.",
|
|
@@ -2311,6 +2625,8 @@ var MetaExternalFetcherAudit = class extends CrawlerBotAudit {
|
|
|
2311
2625
|
evidenceGrade: "A",
|
|
2312
2626
|
tier: "scored",
|
|
2313
2627
|
dossier: "docs/evidence/audits/access-crawl-control/meta-external-fetcher.md",
|
|
2628
|
+
// Gate exemption: being refused is what this category reports.
|
|
2629
|
+
requires: ["origin-reachable"],
|
|
2314
2630
|
defaultPriority: "medium",
|
|
2315
2631
|
guidance: {
|
|
2316
2632
|
impact: "Blocking Meta-ExternalFetcher prevents Meta's AI from fetching your content in real-time for AI-powered features across Facebook, Instagram, and WhatsApp. Allowing it ensures your content can be surfaced in Meta's real-time AI experiences.",
|
|
@@ -2340,6 +2656,8 @@ var BravebotAudit = class extends CrawlerBotAudit {
|
|
|
2340
2656
|
evidenceGrade: "C",
|
|
2341
2657
|
tier: "informative",
|
|
2342
2658
|
dossier: "docs/evidence/audits/access-crawl-control/bravebot.md",
|
|
2659
|
+
// Gate exemption: being refused is what this category reports.
|
|
2660
|
+
requires: ["origin-reachable"],
|
|
2343
2661
|
defaultPriority: "medium",
|
|
2344
2662
|
guidance: {
|
|
2345
2663
|
impact: "Blocking Bravebot prevents your content from appearing in Brave Search AI answers and Brave Leo AI assistant responses. Allowing it gives your content visibility in the privacy-focused Brave browser ecosystem.",
|
|
@@ -2369,6 +2687,8 @@ var DuckassistbotAudit = class extends CrawlerBotAudit {
|
|
|
2369
2687
|
evidenceGrade: "A",
|
|
2370
2688
|
tier: "scored",
|
|
2371
2689
|
dossier: "docs/evidence/audits/access-crawl-control/duckassistbot.md",
|
|
2690
|
+
// Gate exemption: being refused is what this category reports.
|
|
2691
|
+
requires: ["origin-reachable"],
|
|
2372
2692
|
defaultPriority: "medium",
|
|
2373
2693
|
guidance: {
|
|
2374
2694
|
impact: "Blocking DuckAssistBot prevents your content from appearing in DuckDuckGo's AI-powered DuckAssist feature, which generates instant answers from crawled web pages. Allowing it ensures visibility in this privacy-first AI search experience.",
|
|
@@ -2398,6 +2718,8 @@ var MistralaiUserAudit = class extends CrawlerBotAudit {
|
|
|
2398
2718
|
evidenceGrade: "A",
|
|
2399
2719
|
tier: "scored",
|
|
2400
2720
|
dossier: "docs/evidence/audits/access-crawl-control/mistralai-user.md",
|
|
2721
|
+
// Gate exemption: being refused is what this category reports.
|
|
2722
|
+
requires: ["origin-reachable"],
|
|
2401
2723
|
defaultPriority: "medium",
|
|
2402
2724
|
guidance: {
|
|
2403
2725
|
impact: "Blocking MistralAI-User prevents Mistral AI's Le Chat from browsing your site in real-time when users ask it to visit your pages. Allowing it ensures your content can be cited in Mistral-powered AI conversations.",
|
|
@@ -2427,6 +2749,8 @@ var ClaudeSearchbotAudit = class extends CrawlerBotAudit {
|
|
|
2427
2749
|
evidenceGrade: "A",
|
|
2428
2750
|
tier: "scored",
|
|
2429
2751
|
dossier: "docs/evidence/audits/access-crawl-control/claude-searchbot.md",
|
|
2752
|
+
// Gate exemption: being refused is what this category reports.
|
|
2753
|
+
requires: ["origin-reachable"],
|
|
2430
2754
|
defaultPriority: "medium",
|
|
2431
2755
|
guidance: {
|
|
2432
2756
|
impact: "Blocking Claude-SearchBot prevents your content from appearing in Claude's web search results. Allowing it ensures your site is included when Claude searches the web to answer user questions.",
|
|
@@ -2456,6 +2780,8 @@ var NoBlanketBlockAudit = class extends Audit {
|
|
|
2456
2780
|
evidenceGrade: "B",
|
|
2457
2781
|
tier: "scored",
|
|
2458
2782
|
dossier: "docs/evidence/audits/access-crawl-control/no-blanket-block.md",
|
|
2783
|
+
// Gate exemption: being refused is what this category reports.
|
|
2784
|
+
requires: ["origin-reachable"],
|
|
2459
2785
|
defaultPriority: "critical",
|
|
2460
2786
|
guidance: {
|
|
2461
2787
|
impact: "A blanket Disallow: / under User-agent: * blocks every crawler, including all AI agents. Your site becomes completely invisible to AI search engines, ChatGPT Browse, Perplexity, Claude, and all other AI-powered discovery tools.",
|
|
@@ -2467,6 +2793,13 @@ var NoBlanketBlockAudit = class extends Audit {
|
|
|
2467
2793
|
}
|
|
2468
2794
|
};
|
|
2469
2795
|
audit(ctx) {
|
|
2796
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
2797
|
+
return this.notApplicable(
|
|
2798
|
+
"No response here can be attributed to this site, so its robots.txt was not judged.",
|
|
2799
|
+
"User-agent: * does not Disallow: / entirely",
|
|
2800
|
+
unreadSiteReason(ctx.evidence)
|
|
2801
|
+
);
|
|
2802
|
+
}
|
|
2470
2803
|
const robotsFile = ctx.rootFiles["/robots.txt"];
|
|
2471
2804
|
if (!robotsFile || robotsFile.status !== 200 || !robotsFile.body) {
|
|
2472
2805
|
return this.warn(
|
|
@@ -2609,6 +2942,8 @@ var SensitivePathsAudit = class extends Audit {
|
|
|
2609
2942
|
evidenceGrade: "A",
|
|
2610
2943
|
tier: "scored",
|
|
2611
2944
|
dossier: "docs/evidence/audits/access-crawl-control/sensitive-paths.md",
|
|
2945
|
+
// Gate exemption: being refused is what this category reports.
|
|
2946
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
2612
2947
|
defaultPriority: "low",
|
|
2613
2948
|
guidance: {
|
|
2614
2949
|
impact: 'Cart, checkout, site-search, login and account URLs carry nothing an answer engine can cite, but they are crawled and can surface in AI answers as dead, session-bearing links. Apple documents Applebot and Applebot-Extended honouring "Disallow: /private/", and Meta documents the same for meta-externalagent, so a path-level rule keeps that noise out of AI crawls. Two limits matter: RFC 9309 states the protocol "is not a substitute for valid content security measures" and that listed paths become publicly discoverable, so never use robots.txt to protect anything; and user-initiated fetchers are documented not to obey it \u2014 OpenAI says of ChatGPT-User "Because these actions are initiated by a user, robots.txt rules may not apply", and Perplexity says Perplexity-User "generally ignores robots.txt rules".',
|
|
@@ -2704,6 +3039,8 @@ var CrawlDelayAudit = class extends Audit {
|
|
|
2704
3039
|
evidenceGrade: "C",
|
|
2705
3040
|
tier: "informative",
|
|
2706
3041
|
dossier: "docs/evidence/audits/access-crawl-control/crawl-delay.md",
|
|
3042
|
+
// Gate exemption: being refused is what this category reports.
|
|
3043
|
+
requires: ["origin-reachable"],
|
|
2707
3044
|
defaultPriority: "high",
|
|
2708
3045
|
guidance: {
|
|
2709
3046
|
impact: "Excessive Crawl-delay values (over 10 seconds) dramatically slow AI indexing, meaning your latest content may take days or weeks to appear in AI search results while competitors with lower delays get indexed faster.",
|
|
@@ -2714,6 +3051,13 @@ var CrawlDelayAudit = class extends Audit {
|
|
|
2714
3051
|
}
|
|
2715
3052
|
};
|
|
2716
3053
|
audit(ctx) {
|
|
3054
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
3055
|
+
return this.notApplicable(
|
|
3056
|
+
"No response here can be attributed to this site, so its robots.txt was not judged.",
|
|
3057
|
+
"If Crawl-delay is present, it is <= 10 seconds",
|
|
3058
|
+
unreadSiteReason(ctx.evidence)
|
|
3059
|
+
);
|
|
3060
|
+
}
|
|
2717
3061
|
const robotsFile = ctx.rootFiles["/robots.txt"];
|
|
2718
3062
|
if (!robotsFile || robotsFile.status !== 200 || !robotsFile.body) {
|
|
2719
3063
|
return this.warn(
|
|
@@ -2844,6 +3188,9 @@ var MetaRobotsNotBlockingAudit = class extends Audit {
|
|
|
2844
3188
|
evidenceGrade: "A",
|
|
2845
3189
|
tier: "scored",
|
|
2846
3190
|
dossier: "docs/evidence/audits/access-crawl-control/robots-directives.md",
|
|
3191
|
+
// Gate exemption: being refused is what this category reports, and robots directives
|
|
3192
|
+
// live in the head and the headers, which arrive whether or not the body renders.
|
|
3193
|
+
requires: ["origin-reachable"],
|
|
2847
3194
|
defaultPriority: "high",
|
|
2848
3195
|
guidance: {
|
|
2849
3196
|
impact: 'A content page carrying "noindex" (in a robots meta tag, a per-bot meta tag, or the X-Robots-Tag response header) is dropped from the search index, and Google documents that a page must be indexed to appear in AI Overviews or AI Mode. "nosnippet", "noarchive" and "max-snippet:0" keep the page indexed but stop its text being used as a direct input for AI answers.',
|
|
@@ -2855,6 +3202,13 @@ var MetaRobotsNotBlockingAudit = class extends Audit {
|
|
|
2855
3202
|
}
|
|
2856
3203
|
};
|
|
2857
3204
|
audit(ctx) {
|
|
3205
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
3206
|
+
return this.notApplicable(
|
|
3207
|
+
"No page here can be attributed to this site, so its robots directives were not judged.",
|
|
3208
|
+
"No blocking robots directive on content pages",
|
|
3209
|
+
unreadSiteReason(ctx.evidence)
|
|
3210
|
+
);
|
|
3211
|
+
}
|
|
2858
3212
|
if (!ctx.pages || ctx.pages.length === 0) {
|
|
2859
3213
|
return this.notApplicable(
|
|
2860
3214
|
"No pages were scanned, so no robots directives could be read.",
|
|
@@ -2932,6 +3286,10 @@ var NoBotDetectionAudit = class extends Audit {
|
|
|
2932
3286
|
evidenceGrade: "A",
|
|
2933
3287
|
tier: "scored",
|
|
2934
3288
|
dossier: "docs/evidence/audits/access-crawl-control/no-bot-detection.md",
|
|
3289
|
+
// Gate exemption: being refused is what this category reports, and this audit names
|
|
3290
|
+
// the firewall from `wafProtection` alone. Evidence a wall destroys is not evidence
|
|
3291
|
+
// the wall finding needs.
|
|
3292
|
+
requires: [],
|
|
2935
3293
|
defaultPriority: "high",
|
|
2936
3294
|
guidance: {
|
|
2937
3295
|
impact: "Bot-detection services like Cloudflare Turnstile, DataDome, and reCAPTCHA can block legitimate AI agents from accessing your content. When agents are challenged, they cannot complete page fetches, making your content inaccessible to AI-powered search and assistants.",
|
|
@@ -2961,6 +3319,13 @@ var NoBotDetectionAudit = class extends Audit {
|
|
|
2961
3319
|
}
|
|
2962
3320
|
);
|
|
2963
3321
|
}
|
|
3322
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
3323
|
+
return this.notApplicable(
|
|
3324
|
+
"No page here can be attributed to this site, so its scripts were not judged.",
|
|
3325
|
+
"No JavaScript-based bot challenges that would block legitimate AI agents",
|
|
3326
|
+
unreadSiteReason(ctx.evidence)
|
|
3327
|
+
);
|
|
3328
|
+
}
|
|
2964
3329
|
if (!ctx.pages || ctx.pages.length === 0) {
|
|
2965
3330
|
return this.warn(
|
|
2966
3331
|
"No pages were scanned to check for bot-detection scripts.",
|
|
@@ -2986,6 +3351,13 @@ var NoBotDetectionAudit = class extends Audit {
|
|
|
2986
3351
|
}
|
|
2987
3352
|
}
|
|
2988
3353
|
if (detectedServices.size === 0) {
|
|
3354
|
+
if (!scanReadPageText(ctx.evidence)) {
|
|
3355
|
+
return this.notApplicable(
|
|
3356
|
+
"The scanned page served no readable text, so its scripts were not judged.",
|
|
3357
|
+
"No JavaScript-based bot challenges that would block legitimate AI agents",
|
|
3358
|
+
unreadPageTextReason(ctx.evidence)
|
|
3359
|
+
);
|
|
3360
|
+
}
|
|
2989
3361
|
return this.pass(
|
|
2990
3362
|
"No aggressive bot-detection scripts found on scanned pages.",
|
|
2991
3363
|
"No JavaScript-based bot challenges that would block legitimate AI agents",
|
|
@@ -3093,6 +3465,8 @@ var TdmRepAudit = class extends Audit {
|
|
|
3093
3465
|
evidenceGrade: "C",
|
|
3094
3466
|
tier: "experimental",
|
|
3095
3467
|
dossier: "docs/evidence/audits/access-crawl-control/tdm-rep.md",
|
|
3468
|
+
// Gate exemption: being refused is what this category reports.
|
|
3469
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
3096
3470
|
// Nothing consumes the signal, so nothing here should outrank an item that
|
|
3097
3471
|
// changes what an agent can do.
|
|
3098
3472
|
defaultPriority: "low",
|
|
@@ -3106,6 +3480,13 @@ var TdmRepAudit = class extends Audit {
|
|
|
3106
3480
|
}
|
|
3107
3481
|
};
|
|
3108
3482
|
audit(ctx) {
|
|
3483
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
3484
|
+
return this.notApplicable(
|
|
3485
|
+
"No response here can be attributed to this site, so no TDM-Rep declaration was read.",
|
|
3486
|
+
EXPECTED3,
|
|
3487
|
+
unreadSiteReason(ctx.evidence)
|
|
3488
|
+
);
|
|
3489
|
+
}
|
|
3109
3490
|
const pageUrl = ctx.pages[0]?.url;
|
|
3110
3491
|
const header = readHeader(ctx);
|
|
3111
3492
|
if (header) {
|
|
@@ -3241,6 +3622,8 @@ var AgentGovernanceAudit = class extends Audit {
|
|
|
3241
3622
|
evidenceGrade: "A",
|
|
3242
3623
|
tier: "scored",
|
|
3243
3624
|
dossier: "docs/evidence/audits/access-crawl-control/agent-governance.md",
|
|
3625
|
+
// Gate exemption: being refused is what this category reports.
|
|
3626
|
+
requires: ["origin-reachable"],
|
|
3244
3627
|
defaultPriority: "medium",
|
|
3245
3628
|
guidance: {
|
|
3246
3629
|
impact: "Without separate rules for training crawlers and live conversational agents, you cannot block dataset scraping while still appearing in ChatGPT, Claude, and Perplexity answers. A blanket policy either locks you out of AI-powered discovery entirely or leaves your content open to bulk training crawls you never agreed to.",
|
|
@@ -3376,6 +3759,8 @@ var AiContentDeclarationAudit = class extends Audit {
|
|
|
3376
3759
|
evidenceGrade: "D",
|
|
3377
3760
|
tier: "experimental",
|
|
3378
3761
|
dossier: "docs/evidence/audits/access-crawl-control/ai-content-declaration.md",
|
|
3762
|
+
// Gate exemption: being refused is what this category reports.
|
|
3763
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
3379
3764
|
// Was `medium` on an invented directive; the whole class of signals is
|
|
3380
3765
|
// pre-consumer, so nothing here should outrank an actionable item.
|
|
3381
3766
|
defaultPriority: "low",
|
|
@@ -3389,6 +3774,13 @@ var AiContentDeclarationAudit = class extends Audit {
|
|
|
3389
3774
|
}
|
|
3390
3775
|
};
|
|
3391
3776
|
audit(ctx) {
|
|
3777
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
3778
|
+
return this.notApplicable(
|
|
3779
|
+
"No response here can be attributed to this site, so no AI-usage declaration was read.",
|
|
3780
|
+
EXPECTED4,
|
|
3781
|
+
unreadSiteReason(ctx.evidence)
|
|
3782
|
+
);
|
|
3783
|
+
}
|
|
3392
3784
|
const found = survey(ctx);
|
|
3393
3785
|
if (found.aipref) {
|
|
3394
3786
|
return this.pass(
|
|
@@ -3439,6 +3831,9 @@ var HttpsEnabledAudit = class extends Audit {
|
|
|
3439
3831
|
evidenceGrade: "A",
|
|
3440
3832
|
tier: "scored",
|
|
3441
3833
|
dossier: "docs/evidence/audits/access-crawl-control/https-enabled.md",
|
|
3834
|
+
// Gate exemption: a base URL on plain HTTP is proven by the request, with no response
|
|
3835
|
+
// at all, and that fail is worth reporting on a site whose homepage never answered.
|
|
3836
|
+
requires: [],
|
|
3442
3837
|
defaultPriority: "critical",
|
|
3443
3838
|
guidance: {
|
|
3444
3839
|
impact: "HTTP-only sites are completely excluded from all major AI systems. GPTBot, ClaudeBot, Perplexity, and enterprise RAG pipelines refuse to connect to non-HTTPS origins due to security policies. Your entire site is invisible to AI-generated answers, product recommendations, and agentic workflows.",
|
|
@@ -3453,35 +3848,42 @@ var HttpsEnabledAudit = class extends Audit {
|
|
|
3453
3848
|
const isHttps = ctx.baseUrl.startsWith("https://");
|
|
3454
3849
|
const page = ctx.pages?.[0];
|
|
3455
3850
|
const status200 = page?.fetchResult.status === 200;
|
|
3456
|
-
if (isHttps
|
|
3457
|
-
return this.
|
|
3458
|
-
"Site is served over HTTPS
|
|
3851
|
+
if (!isHttps) {
|
|
3852
|
+
return this.fail(
|
|
3853
|
+
"Site is not served over HTTPS. AI agents require secure connections.",
|
|
3459
3854
|
"Base URL uses https:// and homepage returns 200",
|
|
3460
|
-
|
|
3855
|
+
`Base URL: ${ctx.baseUrl}`,
|
|
3856
|
+
{
|
|
3857
|
+
priority: "critical",
|
|
3858
|
+
description: "Enterprise AI frameworks refuse to interact with non-HTTPS sites due to security policies. GPTBot, ClaudeBot, and enterprise RAG systems all skip HTTP-only sites entirely, making your content invisible to AI-generated answers. Enable HTTPS with a valid TLS certificate.",
|
|
3859
|
+
code: "# For nginx:\nserver {\n listen 443 ssl;\n ssl_certificate /path/to/cert.pem;\n ssl_certificate_key /path/to/key.pem;\n}"
|
|
3860
|
+
},
|
|
3461
3861
|
page?.url
|
|
3462
3862
|
);
|
|
3463
3863
|
}
|
|
3464
|
-
if (
|
|
3465
|
-
return this.
|
|
3466
|
-
|
|
3864
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
3865
|
+
return this.notApplicable(
|
|
3866
|
+
"No homepage here can be attributed to this site, so its transport was not judged.",
|
|
3467
3867
|
"Base URL uses https:// and homepage returns 200",
|
|
3468
|
-
|
|
3469
|
-
|
|
3470
|
-
|
|
3471
|
-
|
|
3472
|
-
|
|
3473
|
-
|
|
3868
|
+
unreadSiteReason(ctx.evidence)
|
|
3869
|
+
);
|
|
3870
|
+
}
|
|
3871
|
+
if (status200) {
|
|
3872
|
+
return this.pass(
|
|
3873
|
+
"Site is served over HTTPS with a valid TLS connection.",
|
|
3874
|
+
"Base URL uses https:// and homepage returns 200",
|
|
3875
|
+
`${ctx.baseUrl} \u2014 status ${page?.fetchResult.status}`,
|
|
3474
3876
|
page?.url
|
|
3475
3877
|
);
|
|
3476
3878
|
}
|
|
3477
|
-
return this.
|
|
3478
|
-
"Site
|
|
3879
|
+
return this.warn(
|
|
3880
|
+
"Site uses HTTPS and the homepage answered, but the response carried no document, so an agent has nothing to read over that connection.",
|
|
3479
3881
|
"Base URL uses https:// and homepage returns 200",
|
|
3480
|
-
|
|
3882
|
+
`${ctx.baseUrl} \u2014 a 2xx response that carried no document`,
|
|
3481
3883
|
{
|
|
3482
|
-
priority: "
|
|
3483
|
-
description: "
|
|
3484
|
-
code: "#
|
|
3884
|
+
priority: "high",
|
|
3885
|
+
description: "The homepage answered over HTTPS and returned no document \u2014 an empty 200 body, or a 2xx status that carries none. An AI agent that follows a link to this origin receives nothing, so nothing about the site can be indexed or quoted. Check the origin, the CDN cache entry and any edge rule that can strip a response body.",
|
|
3886
|
+
code: "# Reproduce with:\ncurl -sSi https://yoursite.com | head -20"
|
|
3485
3887
|
},
|
|
3486
3888
|
page?.url
|
|
3487
3889
|
);
|
|
@@ -3587,6 +3989,10 @@ var RobotsAiGroupShadowingAudit = class _RobotsAiGroupShadowingAudit extends Aud
|
|
|
3587
3989
|
evidenceGrade: "A",
|
|
3588
3990
|
tier: "scored",
|
|
3589
3991
|
dossier: "docs/evidence/audits/access-crawl-control/robots-ai-group-shadowing.md",
|
|
3992
|
+
// Gate exemption: being refused is what this category reports.
|
|
3993
|
+
// Gate exemption: the verdict comes from robots.txt. The scanned pages only widen
|
|
3994
|
+
// the probe path set, so a shell narrows the probe and changes nothing judged.
|
|
3995
|
+
requires: ["origin-reachable"],
|
|
3590
3996
|
defaultPriority: "high",
|
|
3591
3997
|
guidance: {
|
|
3592
3998
|
impact: "RFC 9309 \xA72.2.1 states the wildcard group is consulted only 'if no matching group exists'. Therefore, for any site with a named AI-bot group, the wildcard group's Disallow rules provably do not apply to that bot, and the operator's stated intent (expressed once in `*`) diverges from the enforced policy by exactly the symmetric difference of the two rule sets. Falsifiable by construction: given robots.txt R and token T, the set of paths where R_T and R_star disagree is computable and either empty or not.",
|
|
@@ -3605,6 +4011,13 @@ var RobotsAiGroupShadowingAudit = class _RobotsAiGroupShadowingAudit extends Aud
|
|
|
3605
4011
|
};
|
|
3606
4012
|
}
|
|
3607
4013
|
audit(ctx) {
|
|
4014
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
4015
|
+
return this.notApplicable(
|
|
4016
|
+
"No response here can be attributed to this site, so its robots groups were not judged.",
|
|
4017
|
+
EXPECTED5,
|
|
4018
|
+
unreadSiteReason(ctx.evidence)
|
|
4019
|
+
);
|
|
4020
|
+
}
|
|
3608
4021
|
const robots = ctx.rootFiles["/robots.txt"];
|
|
3609
4022
|
if (!robots || robots.status !== 200 || !robots.body.trim()) {
|
|
3610
4023
|
return this.notApplicable(
|
|
@@ -4045,6 +4458,8 @@ var AiCrawlerEdgeParityAudit = class extends Audit {
|
|
|
4045
4458
|
evidenceGrade: "A",
|
|
4046
4459
|
tier: "scored",
|
|
4047
4460
|
dossier: "docs/evidence/audits/access-crawl-control/ai-crawler-edge-parity.md",
|
|
4461
|
+
// Gate exemption: being refused is what this category reports.
|
|
4462
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
4048
4463
|
defaultPriority: "critical",
|
|
4049
4464
|
guidance: {
|
|
4050
4465
|
impact: 'robots.txt (RFC 9309) is advisory metadata parsed by the crawler; the edge access decision is enforced independently by the WAF. A site can therefore publish "User-agent: PerplexityBot / Allow: /" and return a non-200 to every request carrying that user agent, and the operator \u2014 who reads their own robots.txt \u2014 believes they are open while the crawler never sees a byte. Falsifiable: fetch URL U with a browser UA and with crawler UA C; if robots.txt permits C for U and the C request is not 2xx while the browser request is 200, the two policy layers contradict each other. Cloudflare makes one branch deterministic \u2014 a challenge always carries cf-mitigated: challenge \u2014 and a 200 whose main-content text is under 40% of the baseline is a block wearing a 200.',
|
|
@@ -4260,6 +4675,8 @@ var BotContentDeltaDeclaredAudit = class extends Audit {
|
|
|
4260
4675
|
evidenceGrade: "A",
|
|
4261
4676
|
tier: "scored",
|
|
4262
4677
|
dossier: "docs/evidence/audits/access-crawl-control/bot-content-delta-declared.md",
|
|
4678
|
+
// Gate exemption: being refused is what this category reports.
|
|
4679
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
4263
4680
|
defaultPriority: "high",
|
|
4264
4681
|
guidance: {
|
|
4265
4682
|
impact: "Google states that isAccessibleForFree: false with hasPart/cssSelector markup 'helps Google differentiate paywalled content from the practice of cloaking, which violates spam policies' \u2014 serving a crawler less than a user is sanctioned only when it is declared. The measurement is falsifiable both ways: extract the main text of URL U under a browser UA and under crawler UA C, and if the length ratio falls below 0.6 or the 5-gram shingle similarity below 0.7, the site conditions content on the User-Agent. The declaration is equally checkable, and the declared cssSelector must match a real element in the served HTML \u2014 which is where most implementations silently fail, leaving markup that validates and points at nothing. The second-order cost is not the spam risk: an answer engine that only ever sees the stub cites the stub.",
|
|
@@ -4560,6 +4977,8 @@ var AiUsageSignalCoherenceAcrossChannelsAudit = class extends Audit {
|
|
|
4560
4977
|
weight: weightForGrade("B", "scored"),
|
|
4561
4978
|
defaultPriority: "high",
|
|
4562
4979
|
dossier: "docs/evidence/audits/access-crawl-control/ai-usage-signal-coherence-across-channels.md",
|
|
4980
|
+
// Gate exemption: being refused is what this category reports.
|
|
4981
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
4563
4982
|
guidance: {
|
|
4564
4983
|
impact: "No standard defines precedence between these channels; each specifies only its own parsing. A crawler that reads TDM-Rep and a crawler that reads AIPREF therefore read disjoint inputs, and when those inputs disagree the two reach opposite conclusions about the same page. Whichever one you did not mean to publish is the one some operator will act on. The documented worst case is not even yours to make: Cloudflare\u2019s managed robots.txt prepends its own Content-Signal block above your file, so your stated policy can be contradicted at the edge without you knowing.",
|
|
4565
4984
|
fix: "Decide the policy once, then say the same thing in every channel you publish. If you do not intend to maintain a channel, remove it rather than leaving a stale value \u2014 a contradicted signal is worse than a missing one. Where your CDN prepends its own robots.txt block, either turn that feature off or make your own declarations match it.",
|
|
@@ -4569,6 +4988,13 @@ var AiUsageSignalCoherenceAcrossChannelsAudit = class extends Audit {
|
|
|
4569
4988
|
}
|
|
4570
4989
|
};
|
|
4571
4990
|
audit(ctx) {
|
|
4991
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
4992
|
+
return this.notApplicable(
|
|
4993
|
+
"No response here can be attributed to this site, so its AI-usage channels were not compared.",
|
|
4994
|
+
"Every channel that carries an AI-usage signal says the same thing",
|
|
4995
|
+
unreadSiteReason(ctx.evidence)
|
|
4996
|
+
);
|
|
4997
|
+
}
|
|
4572
4998
|
if (ctx.pages.length === 0 && (ctx.rootFiles["/robots.txt"]?.status ?? 0) !== 200) {
|
|
4573
4999
|
return this.notApplicable(
|
|
4574
5000
|
"The scan read no page and no robots.txt, so no channel could carry a signal.",
|
|
@@ -4772,6 +5198,8 @@ var AiprefContentUsageDeclarationValidityAudit = class extends Audit {
|
|
|
4772
5198
|
weight: weightForGrade("B", "scored"),
|
|
4773
5199
|
defaultPriority: "medium",
|
|
4774
5200
|
dossier: "docs/evidence/audits/access-crawl-control/aipref-content-usage-declaration-validity.md",
|
|
5201
|
+
// Gate exemption: being refused is what this category reports.
|
|
5202
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
4775
5203
|
guidance: {
|
|
4776
5204
|
impact: "AIPREF is the one AI-usage vocabulary on the IETF standards track, so a declaration written in it is the one a future crawler is most likely to read. A crawler that cannot parse the line ignores it, and the site is then treated as having no preference at all \u2014 the same outcome as publishing nothing, after the work of publishing something. The costliest version is invisible: a preference attached to a path robots.txt disallows is discarded by the spec itself, so the line looks right and does nothing.",
|
|
4777
5205
|
fix: "Write `Content-Usage: train-ai=n` \u2014 an RFC 8941 dictionary of `y`/`n` values against the `train-ai` and `search` categories. Use `yes`/`no` only in a Cloudflare `Content-Signal:` line, which is a different directive. Attach preferences to paths a crawler is allowed to fetch, and keep the robots.txt line and the response header saying the same thing for the same path.",
|
|
@@ -4781,6 +5209,13 @@ var AiprefContentUsageDeclarationValidityAudit = class extends Audit {
|
|
|
4781
5209
|
}
|
|
4782
5210
|
};
|
|
4783
5211
|
audit(ctx) {
|
|
5212
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
5213
|
+
return this.notApplicable(
|
|
5214
|
+
"No response here can be attributed to this site, so no Content-Usage declaration was read.",
|
|
5215
|
+
"Every Content-Usage declaration parses as an RFC 8941 dictionary of AIPREF categories, attaches to a crawlable path, and agrees with the other channel",
|
|
5216
|
+
unreadSiteReason(ctx.evidence)
|
|
5217
|
+
);
|
|
5218
|
+
}
|
|
4784
5219
|
const robots = ctx.rootFiles["/robots.txt"];
|
|
4785
5220
|
const robotsBody = robots?.status === 200 ? robots.body : "";
|
|
4786
5221
|
const groups = robotsBody === "" ? [] : parseRobots(robotsBody);
|
|
@@ -4963,6 +5398,8 @@ var RslLicensingTermsConformanceAudit = class extends Audit {
|
|
|
4963
5398
|
weight: weightForGrade("B", "scored"),
|
|
4964
5399
|
defaultPriority: "medium",
|
|
4965
5400
|
dossier: "docs/evidence/audits/access-crawl-control/rsl-licensing-terms-conformance.md",
|
|
5401
|
+
// Gate exemption: being refused is what this category reports.
|
|
5402
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
4966
5403
|
guidance: {
|
|
4967
5404
|
impact: 'RSL is the machine-readable form of "here are my terms". A crawler that cannot find the document applies its own defaults instead, and a document it finds but cannot parse is worth no more than one it never found. The specification mandates no default location, so a licence reachable only at a guessed path is one no crawler is obliged to look for. The quiet failure is a `<content url>` prefix that does not cover the pages the licence was written for: the terms load, parse, and apply to nothing.',
|
|
4968
5405
|
fix: 'Point at the licence from robots.txt with an absolute `License:` URI, and add the `Link: <...>; rel="license"; type="application/rsl+xml"` response header so a crawler that never reads robots.txt still finds it. Serve the document as `application/rsl+xml`, keep the `https://rslstandard.org/rsl` namespace on the root element, and make every `<content url>` prefix cover the paths it licenses.',
|
|
@@ -5270,6 +5707,8 @@ var MachineActionable402PaidAccessAudit = class extends Audit {
|
|
|
5270
5707
|
weight: weightForGrade("B", "scored"),
|
|
5271
5708
|
defaultPriority: "medium",
|
|
5272
5709
|
dossier: "docs/evidence/audits/access-crawl-control/machine-actionable-402-paid-access.md",
|
|
5710
|
+
// Gate exemption: being refused is what this category reports.
|
|
5711
|
+
requires: ["origin-reachable", "rendered-body", "sample-adequate"],
|
|
5273
5712
|
guidance: {
|
|
5274
5713
|
impact: "Charging for crawler access is a legitimate choice, and 402 is the status code for it. But a crawler is a program: it can pay only what it can parse. A 402 whose body is an HTML page explaining your licensing terms reads, to the client, as an unexplained refusal \u2014 the same outcome as a 403, after you built a paywall meant to earn revenue. A 402 that a shared cache is allowed to store is worse: the next crawler gets a stored refusal even after paying.",
|
|
5275
5714
|
fix: 'Send one of the machine-readable forms with the 402: Cloudflare\u2019s `crawler-price: USD 0.01`, an x402 `PAYMENT-REQUIRED` challenge listing what you accept, or a `Link: rel=license` pointing at an RSL document whose `<payment type="crawl">` covers the path. Mark the response `Cache-Control: no-store` so a proxy cannot hand your 402 to a crawler that already paid.',
|
|
@@ -5474,6 +5913,8 @@ var WebBotAuthRequestToleranceAudit = class _WebBotAuthRequestToleranceAudit ext
|
|
|
5474
5913
|
weight: weightForGrade("B", "scored"),
|
|
5475
5914
|
defaultPriority: "medium",
|
|
5476
5915
|
dossier: "docs/evidence/audits/access-crawl-control/web-bot-auth-request-tolerance.md",
|
|
5916
|
+
// Gate exemption: being refused is what this category reports.
|
|
5917
|
+
requires: ["origin-reachable"],
|
|
5477
5918
|
guidance: {
|
|
5478
5919
|
impact: "Web Bot Auth is how an agent says who it is in a way an origin can check, and the operators building it are the ones whose traffic you would most want to identify. An edge that answers a signed request with 400 or 403 turns that identification into a reason for refusal: the agents willing to declare themselves are the ones you turn away, and the ones that lie carry no signature headers at all and sail through. A 431 is the same outcome from a different cause \u2014 a header-size limit \u2014 and it is fixed differently.",
|
|
5479
5920
|
fix: "Let unknown request headers through: `Signature`, `Signature-Input` and `Signature-Agent` are additive and safe to ignore. If your edge enforces a header-size budget, raise it enough for an Ed25519 signature. If you do vary behaviour on those headers, list them in `Vary` so a shared cache cannot serve the rejected variant to everyone.",
|
|
@@ -5662,6 +6103,9 @@ var ServerResponsivenessAudit = class extends Audit {
|
|
|
5662
6103
|
evidenceGrade: "B",
|
|
5663
6104
|
tier: "scored",
|
|
5664
6105
|
dossier: "docs/evidence/audits/content-extraction/server-responsiveness.md",
|
|
6106
|
+
// Gate exemption: TTFB is measured from the response, and a shell answers as fast
|
|
6107
|
+
// or as slow as anything else the origin serves.
|
|
6108
|
+
requires: ["origin-reachable", "unblocked-fetches"],
|
|
5665
6109
|
defaultPriority: "medium",
|
|
5666
6110
|
guidance: {
|
|
5667
6111
|
impact: 'Google documents that crawl capacity falls when a host slows down ("if the site slows down\u2026 the limit goes down and Google crawls less"), and slow origins are where logged HTTP 499 client-closed-request clusters from AI fetchers appear. A slow origin therefore gets less of its content into the indexes AI answers are drawn from.',
|
|
@@ -5680,6 +6124,13 @@ var ServerResponsivenessAudit = class extends Audit {
|
|
|
5680
6124
|
`Blocked by ${ctx.wafProtection.name}`
|
|
5681
6125
|
);
|
|
5682
6126
|
}
|
|
6127
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
6128
|
+
return this.notApplicable(
|
|
6129
|
+
"No page here can be attributed to this site, so its response time was not judged.",
|
|
6130
|
+
EXPECTED8,
|
|
6131
|
+
unreadSiteReason(ctx.evidence)
|
|
6132
|
+
);
|
|
6133
|
+
}
|
|
5683
6134
|
const measured = ctx.pages.filter((p) => !p.fetchResult.error && p.fetchResult.status !== 0);
|
|
5684
6135
|
const unmeasured = ctx.pages.length - measured.length;
|
|
5685
6136
|
if (measured.length === 0) {
|
|
@@ -5740,6 +6191,8 @@ var LanguageAttributeAudit = class extends Audit {
|
|
|
5740
6191
|
evidenceGrade: "A",
|
|
5741
6192
|
tier: "scored",
|
|
5742
6193
|
dossier: "docs/evidence/audits/content-extraction/language-attribute.md",
|
|
6194
|
+
// Gate exemption: `<html lang>` is served before any body renders.
|
|
6195
|
+
requires: ["origin-reachable", "unblocked-fetches"],
|
|
5743
6196
|
defaultPriority: "high",
|
|
5744
6197
|
guidance: {
|
|
5745
6198
|
impact: "AI agents use the lang attribute to select the correct language model and tokenizer when processing your content. Without it, agents may misinterpret content language, leading to poor translations or incorrect answers in multilingual AI systems.",
|
|
@@ -5751,6 +6204,13 @@ var LanguageAttributeAudit = class extends Audit {
|
|
|
5751
6204
|
}
|
|
5752
6205
|
};
|
|
5753
6206
|
audit(ctx) {
|
|
6207
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
6208
|
+
return this.notApplicable(
|
|
6209
|
+
"No page here can be attributed to this site, so its language attribute was not judged.",
|
|
6210
|
+
'<html lang="..."> with a non-empty language code',
|
|
6211
|
+
unreadSiteReason(ctx.evidence)
|
|
6212
|
+
);
|
|
6213
|
+
}
|
|
5754
6214
|
const page = ctx.pages[0];
|
|
5755
6215
|
const $ = page?.$;
|
|
5756
6216
|
const lang = $?.("html").attr("lang") ?? "";
|
|
@@ -5889,6 +6349,7 @@ var MarkdownAlternateAudit = class extends Audit {
|
|
|
5889
6349
|
weight: weightForGrade("A", "scored"),
|
|
5890
6350
|
defaultPriority: "medium",
|
|
5891
6351
|
dossier: "docs/evidence/audits/content-extraction/markdown-alternate.md",
|
|
6352
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
5892
6353
|
guidance: {
|
|
5893
6354
|
impact: "A markdown alternate is a promise that an agent can read the page cheaply and get the same answer. A stale or partial alternate breaks that promise silently: the agent gets a document that looks authoritative, costs less, and says less than the page it claims to mirror. Serving it as `text/plain` or `text/html` is the same failure one level down \u2014 the client that negotiated for markdown cannot tell it got any. The consumers this is graded on are interactive coding agents \u2014 Claude Code, Cursor, Copilot Chat and CLI, Codex CLI \u2014 and GPTBot, measured taking markdown on 34.8% of fetches where a `.md` URL exists.",
|
|
5894
6355
|
fix: 'Serve the alternate from the same source as the HTML, so headings and prose cannot drift, with `Content-Type: text/markdown` (a `charset` parameter is fine). Publish it on the page URL plus `.md`, or answer `Accept: text/markdown` on the page URL itself \u2014 those are the two routes with documented consumers. Declaring it with `<link rel="alternate" type="text/markdown" href="...">` saves an agent a guess, but the link relation itself has one single-sourced consumer, so this audit reports it rather than scoring it.',
|
|
@@ -6119,6 +6580,7 @@ var JsonLdDuplicationMassAudit = class extends Audit {
|
|
|
6119
6580
|
weight: weightForGrade("C", "informative"),
|
|
6120
6581
|
defaultPriority: "low",
|
|
6121
6582
|
dossier: "docs/evidence/audits/content-extraction/json-ld-duplication-mass.md",
|
|
6583
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6122
6584
|
guidance: {
|
|
6123
6585
|
impact: "A non-rendering agent tokenizes the whole document, JSON-LD included. Where a block repeats the article body the DOM already carries, the page ships that text twice and the agent pays for both copies out of one context window. The same holds for a node declared identically in two blocks: the second copy adds tokens and no facts.",
|
|
6124
6586
|
fix: "Keep JSON-LD to the facts a parser needs \u2014 identifiers, prices, dates, relationships \u2014 and let the prose live in the DOM. Where a schema property genuinely needs body text, a summary is usually enough. Merge blocks that declare the same `@id` into one.",
|
|
@@ -6228,6 +6690,7 @@ var SingleH1Audit = class extends Audit {
|
|
|
6228
6690
|
evidenceGrade: "B",
|
|
6229
6691
|
tier: "scored",
|
|
6230
6692
|
dossier: "docs/evidence/audits/content-extraction/single-h1.md",
|
|
6693
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6231
6694
|
defaultPriority: "high",
|
|
6232
6695
|
guidance: {
|
|
6233
6696
|
impact: "AI agents use the single <h1> as the authoritative page title for content indexing and answer generation. Multiple <h1> elements create ambiguity about the page's primary topic, causing agents to misidentify or conflate subjects when generating answers.",
|
|
@@ -6239,6 +6702,13 @@ var SingleH1Audit = class extends Audit {
|
|
|
6239
6702
|
}
|
|
6240
6703
|
};
|
|
6241
6704
|
audit(ctx) {
|
|
6705
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
6706
|
+
return this.notApplicable(
|
|
6707
|
+
"No page here can be attributed to this site, so its headings were not judged.",
|
|
6708
|
+
"Exactly one <h1> on the homepage",
|
|
6709
|
+
unreadSiteReason(ctx.evidence)
|
|
6710
|
+
);
|
|
6711
|
+
}
|
|
6242
6712
|
const homepage = ctx.pages[0];
|
|
6243
6713
|
if (!homepage) {
|
|
6244
6714
|
return this.fail(
|
|
@@ -6290,6 +6760,7 @@ var SequentialHeadingsAudit = class extends Audit {
|
|
|
6290
6760
|
evidenceGrade: "B",
|
|
6291
6761
|
tier: "scored",
|
|
6292
6762
|
dossier: "docs/evidence/audits/content-extraction/sequential-headings.md",
|
|
6763
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6293
6764
|
defaultPriority: "high",
|
|
6294
6765
|
guidance: {
|
|
6295
6766
|
impact: "AI systems build content outlines from heading levels to understand document hierarchy. Skipped levels (e.g., h1 directly to h3) break this hierarchy, causing agents to misinterpret section nesting and produce inaccurate content summaries with wrong parent-child relationships.",
|
|
@@ -6379,6 +6850,7 @@ var MainElementAudit = class extends Audit {
|
|
|
6379
6850
|
evidenceGrade: "A",
|
|
6380
6851
|
tier: "scored",
|
|
6381
6852
|
dossier: "docs/evidence/audits/content-extraction/main-element.md",
|
|
6853
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6382
6854
|
defaultPriority: "high",
|
|
6383
6855
|
guidance: {
|
|
6384
6856
|
impact: "Without a <main> element, AI scrapers cannot distinguish primary content from navigation, sidebars, and footer boilerplate. This causes agents to ingest menus, disclaimers, and repeated chrome into their context window, increasing hallucination risk and reducing answer relevance.",
|
|
@@ -6390,6 +6862,13 @@ var MainElementAudit = class extends Audit {
|
|
|
6390
6862
|
}
|
|
6391
6863
|
};
|
|
6392
6864
|
audit(ctx) {
|
|
6865
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
6866
|
+
return this.notApplicable(
|
|
6867
|
+
"No page here can be attributed to this site, so its landmarks were not judged.",
|
|
6868
|
+
"<main> element present on all pages",
|
|
6869
|
+
unreadSiteReason(ctx.evidence)
|
|
6870
|
+
);
|
|
6871
|
+
}
|
|
6393
6872
|
let pagesWithMain = 0;
|
|
6394
6873
|
for (const page of ctx.pages) {
|
|
6395
6874
|
if (page.$("main").length > 0) pagesWithMain++;
|
|
@@ -6436,6 +6915,7 @@ var ArticleElementAudit = class extends Audit {
|
|
|
6436
6915
|
evidenceGrade: "A",
|
|
6437
6916
|
tier: "scored",
|
|
6438
6917
|
dossier: "docs/evidence/audits/content-extraction/article-element.md",
|
|
6918
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6439
6919
|
applicablePageTypes: ["content"],
|
|
6440
6920
|
defaultPriority: "medium",
|
|
6441
6921
|
guidance: {
|
|
@@ -6448,6 +6928,13 @@ var ArticleElementAudit = class extends Audit {
|
|
|
6448
6928
|
}
|
|
6449
6929
|
};
|
|
6450
6930
|
audit(ctx) {
|
|
6931
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
6932
|
+
return this.notApplicable(
|
|
6933
|
+
"No page here can be attributed to this site, so its landmarks were not judged.",
|
|
6934
|
+
"<article> elements on content pages",
|
|
6935
|
+
unreadSiteReason(ctx.evidence)
|
|
6936
|
+
);
|
|
6937
|
+
}
|
|
6451
6938
|
let pagesWithArticle = 0;
|
|
6452
6939
|
for (const page of ctx.pages) {
|
|
6453
6940
|
if (page.$("article").length > 0) pagesWithArticle++;
|
|
@@ -6494,6 +6981,7 @@ var HeaderFooterAudit = class extends Audit {
|
|
|
6494
6981
|
evidenceGrade: "A",
|
|
6495
6982
|
tier: "scored",
|
|
6496
6983
|
dossier: "docs/evidence/audits/content-extraction/header-footer.md",
|
|
6984
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6497
6985
|
defaultPriority: "medium",
|
|
6498
6986
|
guidance: {
|
|
6499
6987
|
impact: "AI agents use <header> and <footer> landmarks to identify and exclude boilerplate content (navigation menus, copyright notices, legal links) from primary content extraction. Without these landmarks, agents may include footer disclaimers or nav menus in their content summaries, reducing answer accuracy.",
|
|
@@ -6505,6 +6993,13 @@ var HeaderFooterAudit = class extends Audit {
|
|
|
6505
6993
|
}
|
|
6506
6994
|
};
|
|
6507
6995
|
audit(ctx) {
|
|
6996
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
6997
|
+
return this.notApplicable(
|
|
6998
|
+
"No page here can be attributed to this site, so its landmarks were not judged.",
|
|
6999
|
+
"Both <header> and <footer> present on all pages",
|
|
7000
|
+
unreadSiteReason(ctx.evidence)
|
|
7001
|
+
);
|
|
7002
|
+
}
|
|
6508
7003
|
let pagesWithBoth = 0;
|
|
6509
7004
|
let pagesWithHeader = 0;
|
|
6510
7005
|
let pagesWithFooter = 0;
|
|
@@ -6585,6 +7080,7 @@ var AsideElementAudit = class extends Audit {
|
|
|
6585
7080
|
evidenceGrade: "B",
|
|
6586
7081
|
tier: "scored",
|
|
6587
7082
|
dossier: "docs/evidence/audits/content-extraction/aside-element.md",
|
|
7083
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6588
7084
|
applicablePageTypes: ["content"],
|
|
6589
7085
|
defaultPriority: "low",
|
|
6590
7086
|
guidance: {
|
|
@@ -6667,6 +7163,7 @@ var SectionHeadingsAudit = class extends Audit {
|
|
|
6667
7163
|
evidenceGrade: "B",
|
|
6668
7164
|
tier: "scored",
|
|
6669
7165
|
dossier: "docs/evidence/audits/content-extraction/section-headings.md",
|
|
7166
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6670
7167
|
defaultPriority: "medium",
|
|
6671
7168
|
guidance: {
|
|
6672
7169
|
impact: "AI agents use section headings to build a topic map of your page for retrieval-augmented generation. Unlabeled <section> elements are opaque to AI chunking systems, preventing them from indexing and retrieving your content by topic, which reduces your visibility in AI-generated answers.",
|
|
@@ -6845,6 +7342,7 @@ var SemanticListsAudit = class extends Audit {
|
|
|
6845
7342
|
evidenceGrade: "B",
|
|
6846
7343
|
tier: "scored",
|
|
6847
7344
|
dossier: "docs/evidence/audits/content-extraction/semantic-lists.md",
|
|
7345
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6848
7346
|
defaultPriority: "medium",
|
|
6849
7347
|
guidance: {
|
|
6850
7348
|
impact: 'AI agents recognize <ul>, <ol>, and <dl> as structured lists and extract them as bullet points, numbered steps or term/definition pairs. Content formatted as styled <div> elements \u2014 or as paragraphs that start with "1.", "2." \u2014 collapses into undelimited prose when the page is converted to markdown or an accessibility tree, so the agent has to re-infer where each item begins.',
|
|
@@ -6916,6 +7414,7 @@ var DataTablesAudit = class extends Audit {
|
|
|
6916
7414
|
evidenceGrade: "B",
|
|
6917
7415
|
tier: "scored",
|
|
6918
7416
|
dossier: "docs/evidence/audits/content-extraction/data-tables.md",
|
|
7417
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6919
7418
|
defaultPriority: "medium",
|
|
6920
7419
|
guidance: {
|
|
6921
7420
|
impact: "AI agents rely on <thead> and <th> elements to understand column headers and map cell values to their meanings. Without proper table structure, agents cannot interpret tabular data correctly, leading to garbled comparisons and inaccurate data extraction in AI-generated summaries.",
|
|
@@ -6927,6 +7426,13 @@ var DataTablesAudit = class extends Audit {
|
|
|
6927
7426
|
}
|
|
6928
7427
|
};
|
|
6929
7428
|
audit(ctx) {
|
|
7429
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
7430
|
+
return this.notApplicable(
|
|
7431
|
+
"No page here can be attributed to this site, so its tables were not judged.",
|
|
7432
|
+
"Tables have <thead> and <th> elements",
|
|
7433
|
+
unreadSiteReason(ctx.evidence)
|
|
7434
|
+
);
|
|
7435
|
+
}
|
|
6930
7436
|
let totalTables = 0;
|
|
6931
7437
|
let properTables = 0;
|
|
6932
7438
|
for (const page of ctx.pages) {
|
|
@@ -6939,6 +7445,13 @@ var DataTablesAudit = class extends Audit {
|
|
|
6939
7445
|
});
|
|
6940
7446
|
}
|
|
6941
7447
|
if (totalTables === 0) {
|
|
7448
|
+
if (!scanReadPageText(ctx.evidence)) {
|
|
7449
|
+
return this.notApplicable(
|
|
7450
|
+
"The scanned page served no readable text, so it held no tables to judge.",
|
|
7451
|
+
"Tables have <thead> and <th> elements",
|
|
7452
|
+
unreadPageTextReason(ctx.evidence)
|
|
7453
|
+
);
|
|
7454
|
+
}
|
|
6942
7455
|
return this.pass(
|
|
6943
7456
|
"No data tables found \u2014 check not applicable.",
|
|
6944
7457
|
"Tables have <thead> and <th> elements",
|
|
@@ -6992,6 +7505,7 @@ var CodeLanguageAudit = class extends Audit {
|
|
|
6992
7505
|
evidenceGrade: "C",
|
|
6993
7506
|
tier: "informative",
|
|
6994
7507
|
dossier: "docs/evidence/audits/content-extraction/code-language.md",
|
|
7508
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
6995
7509
|
applicablePageTypes: ["content"],
|
|
6996
7510
|
defaultPriority: "low",
|
|
6997
7511
|
guidance: {
|
|
@@ -7073,6 +7587,7 @@ var TimeElementAudit = class extends Audit {
|
|
|
7073
7587
|
evidenceGrade: "C",
|
|
7074
7588
|
tier: "informative",
|
|
7075
7589
|
dossier: "docs/evidence/audits/content-extraction/time-element.md",
|
|
7590
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7076
7591
|
applicablePageTypes: ["content"],
|
|
7077
7592
|
defaultPriority: "medium",
|
|
7078
7593
|
guidance: {
|
|
@@ -7123,6 +7638,7 @@ var ContentDepthAudit = class extends Audit {
|
|
|
7123
7638
|
evidenceGrade: "B",
|
|
7124
7639
|
tier: "scored",
|
|
7125
7640
|
dossier: "docs/evidence/audits/content-extraction/content-depth.md",
|
|
7641
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7126
7642
|
defaultPriority: "medium",
|
|
7127
7643
|
guidance: {
|
|
7128
7644
|
impact: "Pages with fewer than 300 words provide too little context for AI RAG systems to generate accurate, detailed answers. Thin content produces weak vector embeddings that rank poorly in retrieval, causing your pages to be excluded from AI-generated responses entirely.",
|
|
@@ -7132,6 +7648,13 @@ var ContentDepthAudit = class extends Audit {
|
|
|
7132
7648
|
}
|
|
7133
7649
|
};
|
|
7134
7650
|
audit(ctx) {
|
|
7651
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
7652
|
+
return this.notApplicable(
|
|
7653
|
+
"No page here can be attributed to this site, so its content depth was not judged.",
|
|
7654
|
+
"More than 300 words of content per page",
|
|
7655
|
+
unreadSiteReason(ctx.evidence)
|
|
7656
|
+
);
|
|
7657
|
+
}
|
|
7135
7658
|
let pagesAboveThreshold = 0;
|
|
7136
7659
|
const wordCounts = [];
|
|
7137
7660
|
for (const page of ctx.pages) {
|
|
@@ -7205,6 +7728,7 @@ var ImageAltTextAudit = class extends Audit {
|
|
|
7205
7728
|
evidenceGrade: "A",
|
|
7206
7729
|
tier: "scored",
|
|
7207
7730
|
dossier: "docs/evidence/audits/content-extraction/image-alt-text.md",
|
|
7731
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7208
7732
|
defaultPriority: "high",
|
|
7209
7733
|
guidance: {
|
|
7210
7734
|
impact: "An image with no text alternative has no accessible name, so it is an unnamed node in the accessibility-tree snapshots agent toolkits send to a model \u2014 Playwright MCP, Claude-in-Chrome read_page, Chrome DevTools take_snapshot \u2014 and it carries no subject matter for Google Images, which states it uses alt text to understand what an image shows. A multimodal agent that fetches the image bytes can caption it without one; a text-only crawler or a snapshot-driven agent cannot.",
|
|
@@ -7291,6 +7815,7 @@ var FigureFigcaptionAudit = class extends Audit {
|
|
|
7291
7815
|
evidenceGrade: "C",
|
|
7292
7816
|
tier: "informative",
|
|
7293
7817
|
dossier: "docs/evidence/audits/content-extraction/figure-figcaption.md",
|
|
7818
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7294
7819
|
defaultPriority: "medium",
|
|
7295
7820
|
guidance: {
|
|
7296
7821
|
impact: "AI agents use <figcaption> to understand the purpose and context of visual content beyond what alt text provides. Without captions, figures are treated as opaque image containers, and your charts, diagrams, and illustrations cannot be meaningfully cited in AI-generated answers.",
|
|
@@ -7302,6 +7827,13 @@ var FigureFigcaptionAudit = class extends Audit {
|
|
|
7302
7827
|
}
|
|
7303
7828
|
};
|
|
7304
7829
|
audit(ctx) {
|
|
7830
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
7831
|
+
return this.notApplicable(
|
|
7832
|
+
"No page here can be attributed to this site, so its figures were not judged.",
|
|
7833
|
+
"Images with context wrapped in <figure> with <figcaption>",
|
|
7834
|
+
unreadSiteReason(ctx.evidence)
|
|
7835
|
+
);
|
|
7836
|
+
}
|
|
7305
7837
|
let totalFigures = 0;
|
|
7306
7838
|
let figuresWithCaption = 0;
|
|
7307
7839
|
for (const page of ctx.pages) {
|
|
@@ -7330,6 +7862,13 @@ var FigureFigcaptionAudit = class extends Audit {
|
|
|
7330
7862
|
}
|
|
7331
7863
|
);
|
|
7332
7864
|
}
|
|
7865
|
+
if (!scanReadPageText(ctx.evidence)) {
|
|
7866
|
+
return this.notApplicable(
|
|
7867
|
+
"The scanned page served no readable text, so it held no images or figures to judge.",
|
|
7868
|
+
"Images with context wrapped in <figure> with <figcaption>",
|
|
7869
|
+
unreadPageTextReason(ctx.evidence)
|
|
7870
|
+
);
|
|
7871
|
+
}
|
|
7333
7872
|
return this.pass(
|
|
7334
7873
|
"No images or <figure> elements found \u2014 check not applicable.",
|
|
7335
7874
|
"Images with context wrapped in <figure> with <figcaption>",
|
|
@@ -7413,6 +7952,7 @@ var SvgBloatAudit = class extends Audit {
|
|
|
7413
7952
|
evidenceGrade: "B",
|
|
7414
7953
|
tier: "scored",
|
|
7415
7954
|
dossier: "docs/evidence/audits/content-extraction/svg-bloat.md",
|
|
7955
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
7416
7956
|
defaultPriority: "medium",
|
|
7417
7957
|
guidance: {
|
|
7418
7958
|
impact: "Large inline SVGs are inlined verbatim as path-data tokens when an LLM converts your page to Markdown. A single 10KB icon or chart can consume thousands of tokens of agent context per page load, inflating agent cost and pushing real content out of the context window \u2014 reducing the quality of what agents extract and say about your site.",
|
|
@@ -12823,6 +13363,7 @@ var TokenRatioAudit = class extends Audit {
|
|
|
12823
13363
|
evidenceGrade: "B",
|
|
12824
13364
|
tier: "scored",
|
|
12825
13365
|
dossier: "docs/evidence/audits/content-extraction/token-ratio.md",
|
|
13366
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
12826
13367
|
defaultPriority: "high",
|
|
12827
13368
|
guidance: {
|
|
12828
13369
|
impact: "When less than 15% of your HTML is actual content, AI agents burn most of their context window and token budget on markup noise: inline scripts, CSS, SVG sprites, tracking tags, and deeply nested divs. The useful text that remains gets weaker attention from the model, and pages with extreme bloat may be truncated before the real content is even read.",
|
|
@@ -12833,6 +13374,13 @@ var TokenRatioAudit = class extends Audit {
|
|
|
12833
13374
|
}
|
|
12834
13375
|
};
|
|
12835
13376
|
audit(ctx) {
|
|
13377
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
13378
|
+
return this.notApplicable(
|
|
13379
|
+
"No page here can be attributed to this site, so its token mix was not measured.",
|
|
13380
|
+
"A homepage from this site whose token mix can be measured",
|
|
13381
|
+
unreadSiteReason(ctx.evidence)
|
|
13382
|
+
);
|
|
13383
|
+
}
|
|
12836
13384
|
const page = ctx.pages[0];
|
|
12837
13385
|
const rawHtml = page?.fetchResult.body ?? "";
|
|
12838
13386
|
if (!page || rawHtml.trim().length === 0) {
|
|
@@ -12954,6 +13502,7 @@ var FakeHeadingsAudit = class extends Audit {
|
|
|
12954
13502
|
evidenceGrade: "B",
|
|
12955
13503
|
tier: "scored",
|
|
12956
13504
|
dossier: "docs/evidence/audits/content-extraction/fake-headings.md",
|
|
13505
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
12957
13506
|
defaultPriority: "medium",
|
|
12958
13507
|
guidance: {
|
|
12959
13508
|
impact: "AI agents build content outlines exclusively from <h1>\u2013<h6> elements. Text that only looks like a heading is treated as ordinary body copy, so agents miss your section structure entirely \u2014 summaries flatten into a wall of text, section-level citations become impossible, and chunking for retrieval splits content at arbitrary points instead of at your intended section boundaries.",
|
|
@@ -12965,6 +13514,13 @@ var FakeHeadingsAudit = class extends Audit {
|
|
|
12965
13514
|
}
|
|
12966
13515
|
};
|
|
12967
13516
|
audit(ctx) {
|
|
13517
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
13518
|
+
return this.notApplicable(
|
|
13519
|
+
"No page here can be attributed to this site, so its headings were not judged.",
|
|
13520
|
+
"All heading-like text uses semantic <h1>-<h6> elements",
|
|
13521
|
+
unreadSiteReason(ctx.evidence)
|
|
13522
|
+
);
|
|
13523
|
+
}
|
|
12968
13524
|
const found = [];
|
|
12969
13525
|
for (const page of ctx.pages) {
|
|
12970
13526
|
const $ = page.$;
|
|
@@ -12988,6 +13544,13 @@ var FakeHeadingsAudit = class extends Audit {
|
|
|
12988
13544
|
const foundSummary = found.slice(0, 5).map((f) => `${f.url}: ${describe5(f.heading)}`).join("; ");
|
|
12989
13545
|
const expected = "All heading-like text uses semantic <h1>-<h6> elements";
|
|
12990
13546
|
if (found.length === 0) {
|
|
13547
|
+
if (!scanReadPageText(ctx.evidence)) {
|
|
13548
|
+
return this.notApplicable(
|
|
13549
|
+
"The scanned page served no readable text, so it held no headings to judge.",
|
|
13550
|
+
expected,
|
|
13551
|
+
unreadPageTextReason(ctx.evidence)
|
|
13552
|
+
);
|
|
13553
|
+
}
|
|
12991
13554
|
return this.pass(
|
|
12992
13555
|
"No fake headings detected \u2014 heading-like text uses semantic heading elements.",
|
|
12993
13556
|
expected,
|
|
@@ -13017,6 +13580,9 @@ var FakeHeadingsAudit = class extends Audit {
|
|
|
13017
13580
|
};
|
|
13018
13581
|
|
|
13019
13582
|
// src/audits/content-extraction/server-rendered.ts
|
|
13583
|
+
function withDetails(result, details) {
|
|
13584
|
+
return { ...result, details: { ...result.details ?? {}, ...details } };
|
|
13585
|
+
}
|
|
13020
13586
|
var ServerRenderedAudit = class extends Audit {
|
|
13021
13587
|
static meta = {
|
|
13022
13588
|
id: "content-extraction/server-rendered",
|
|
@@ -13029,6 +13595,8 @@ var ServerRenderedAudit = class extends Audit {
|
|
|
13029
13595
|
evidenceGrade: "B",
|
|
13030
13596
|
tier: "scored",
|
|
13031
13597
|
dossier: "docs/evidence/audits/content-extraction/server-rendered.md",
|
|
13598
|
+
// Gate exemption: A shell is what this audit reports. Gating it would delete the finding.
|
|
13599
|
+
requires: ["origin-reachable", "unblocked-fetches"],
|
|
13032
13600
|
defaultPriority: "critical",
|
|
13033
13601
|
guidance: {
|
|
13034
13602
|
impact: "AI crawlers (GPTBot, ClaudeBot, PerplexityBot) do not execute JavaScript. If your content is only rendered client-side, these crawlers see an empty or near-empty page. Your products, articles, and brand information are completely absent from AI knowledge bases, meaning AI-generated answers never reference your site.",
|
|
@@ -13040,37 +13608,64 @@ var ServerRenderedAudit = class extends Audit {
|
|
|
13040
13608
|
}
|
|
13041
13609
|
};
|
|
13042
13610
|
audit(ctx) {
|
|
13043
|
-
|
|
13044
|
-
|
|
13045
|
-
|
|
13046
|
-
"
|
|
13047
|
-
|
|
13048
|
-
"No homepage fetched",
|
|
13049
|
-
void 0,
|
|
13050
|
-
void 0
|
|
13611
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
13612
|
+
return this.notApplicable(
|
|
13613
|
+
"No page here can be attributed to this site, so its served HTML was not judged.",
|
|
13614
|
+
"Every fetched page serves > 50 words or > 200 characters of readable text",
|
|
13615
|
+
unreadSiteReason(ctx.evidence)
|
|
13051
13616
|
);
|
|
13052
13617
|
}
|
|
13053
|
-
const
|
|
13054
|
-
|
|
13055
|
-
|
|
13056
|
-
|
|
13057
|
-
|
|
13058
|
-
|
|
13059
|
-
|
|
13060
|
-
|
|
13061
|
-
|
|
13618
|
+
const pages = ctx.pages ?? [];
|
|
13619
|
+
if (pages.length === 0) {
|
|
13620
|
+
return this.notApplicable(
|
|
13621
|
+
"The scan fetched no page, so there is no served HTML to judge.",
|
|
13622
|
+
"Every fetched page serves > 50 words or > 200 characters of readable text",
|
|
13623
|
+
"No page fetched"
|
|
13624
|
+
);
|
|
13625
|
+
}
|
|
13626
|
+
const rendered = ctx.evidence.renderedByPage;
|
|
13627
|
+
const emptyPages = pages.filter((page) => !(rendered[page.url] ?? pageRendersText(page))).map((page) => page.url);
|
|
13628
|
+
const total = pages.length;
|
|
13629
|
+
const renderedCount = total - emptyPages.length;
|
|
13630
|
+
const expected = "Every fetched page serves > 50 words or > 200 characters of readable text";
|
|
13631
|
+
const found = `${renderedCount} of ${total} page(s) served readable text`;
|
|
13632
|
+
if (emptyPages.length === 0) {
|
|
13633
|
+
return withDetails(
|
|
13634
|
+
this.pass(
|
|
13635
|
+
`All ${total} fetched page(s) serve their content in the HTML response.`,
|
|
13636
|
+
expected,
|
|
13637
|
+
found,
|
|
13638
|
+
pages[0].url
|
|
13639
|
+
),
|
|
13640
|
+
{ pagesChecked: total, renderedPages: renderedCount }
|
|
13062
13641
|
);
|
|
13063
13642
|
}
|
|
13064
|
-
|
|
13065
|
-
|
|
13066
|
-
"
|
|
13067
|
-
|
|
13068
|
-
|
|
13069
|
-
|
|
13070
|
-
|
|
13071
|
-
|
|
13072
|
-
|
|
13073
|
-
|
|
13643
|
+
const failGuidance = {
|
|
13644
|
+
priority: "critical",
|
|
13645
|
+
description: "AI crawlers like GPTBot and ClaudeBot do not execute JavaScript. Content only visible after JS execution is completely invisible to them, meaning your site effectively has no content in AI knowledge bases. Use SSR (server-side rendering) or SSG (static site generation) to serve content in the initial HTML response.",
|
|
13646
|
+
code: "// Next.js SSR example:\nexport async function getServerSideProps() {\n const data = await fetchData();\n return { props: { data } };\n}"
|
|
13647
|
+
};
|
|
13648
|
+
if (renderedCount === 0) {
|
|
13649
|
+
return withDetails(
|
|
13650
|
+
this.fail(
|
|
13651
|
+
`None of the ${total} fetched page(s) serve readable content in the HTML response. AI agents cannot read client-side-only rendered content.`,
|
|
13652
|
+
expected,
|
|
13653
|
+
found,
|
|
13654
|
+
failGuidance,
|
|
13655
|
+
pages[0].url
|
|
13656
|
+
),
|
|
13657
|
+
{ pagesChecked: total, renderedPages: 0, emptyPages }
|
|
13658
|
+
);
|
|
13659
|
+
}
|
|
13660
|
+
return withDetails(
|
|
13661
|
+
this.warn(
|
|
13662
|
+
`${emptyPages.length} of ${total} fetched page(s) serve no readable content in the HTML response. AI agents read nothing on those pages.`,
|
|
13663
|
+
expected,
|
|
13664
|
+
found,
|
|
13665
|
+
failGuidance,
|
|
13666
|
+
emptyPages[0]
|
|
13667
|
+
),
|
|
13668
|
+
{ pagesChecked: total, renderedPages: renderedCount, emptyPages }
|
|
13074
13669
|
);
|
|
13075
13670
|
}
|
|
13076
13671
|
};
|
|
@@ -13315,6 +13910,7 @@ var CssHiddenGhostContentAudit = class _CssHiddenGhostContentAudit extends Audit
|
|
|
13315
13910
|
evidenceGrade: "A",
|
|
13316
13911
|
tier: "scored",
|
|
13317
13912
|
dossier: "docs/evidence/audits/content-extraction/css-hidden-ghost-content.md",
|
|
13913
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
13318
13914
|
defaultPriority: "medium",
|
|
13319
13915
|
guidance: {
|
|
13320
13916
|
impact: "This is provable from source, not inferred. Readability's visibility test consults only node.style.display, node.style.visibility, the hidden attribute and aria-hidden \u2014 it explicitly does not evaluate class-based CSS rules from stylesheets. AI crawlers do not render, so no cascade is ever computed. Therefore any subtree hidden by `.mobile-only{display:none}`, `.tab-panel:not(.active){display:none}` or `[data-state=closed]{display:none}` reaches the model as ordinary body text with full weight. Consequence is not just cost: the agent sees three parallel copies of a nav, both the collapsed and expanded FAQ answers, and often stale price text from a hidden variant block, and irrelevant/contradictory context measurably degrades answers.",
|
|
@@ -13333,6 +13929,13 @@ var CssHiddenGhostContentAudit = class _CssHiddenGhostContentAudit extends Audit
|
|
|
13333
13929
|
};
|
|
13334
13930
|
}
|
|
13335
13931
|
async audit(ctx) {
|
|
13932
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
13933
|
+
return this.notApplicable(
|
|
13934
|
+
"No page here can be attributed to this site, so its hidden text was not measured.",
|
|
13935
|
+
EXPECTED11,
|
|
13936
|
+
unreadSiteReason(ctx.evidence)
|
|
13937
|
+
);
|
|
13938
|
+
}
|
|
13336
13939
|
const s = await survey2(ctx);
|
|
13337
13940
|
const partial = s.crossOrigin > 0 ? `; ${s.crossOrigin} cross-origin stylesheet not fetched` : "";
|
|
13338
13941
|
if (s.totalChars === 0) {
|
|
@@ -13493,6 +14096,7 @@ var HydrationPayloadShareAudit = class _HydrationPayloadShareAudit extends Audit
|
|
|
13493
14096
|
evidenceGrade: "A",
|
|
13494
14097
|
tier: "scored",
|
|
13495
14098
|
dossier: "docs/evidence/audits/content-extraction/hydration-payload-share.md",
|
|
14099
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
13496
14100
|
defaultPriority: "medium",
|
|
13497
14101
|
guidance: {
|
|
13498
14102
|
impact: "These blobs are inlined into every HTML response by design, and the framework vendor itself flags > 128 kB as a defect. A browser parses them and throws them away after hydration; a non-rendering AI crawler cannot \u2014 it tokenizes the JSON verbatim, including escaped HTML, CDN image variants, GraphQL type metadata and the full body text a second time. The causal claim is falsifiable per page: strip these script nodes, re-tokenize, and the delta is the exact context cost that carries zero incremental information, since duplicate #3 is byte-identical content the agent already has.",
|
|
@@ -13647,6 +14251,7 @@ var PreambleTaxTokensBeforeTheFirstContentTokenAudit = class extends Audit {
|
|
|
13647
14251
|
weight: weightForGrade("B", "scored"),
|
|
13648
14252
|
defaultPriority: "medium",
|
|
13649
14253
|
dossier: "docs/evidence/audits/content-extraction/preamble-tax.md",
|
|
14254
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
13650
14255
|
guidance: {
|
|
13651
14256
|
impact: "A non-rendering agent ingests the document as a linear stream, so DOM order is context order. A page that inlines a critical-CSS block and a serialized state blob ahead of its content does two things at once: it pushes the answer into the middle of the context window, where retrieval is measurably weakest, and it guarantees the answer is what gets cut when the fetching harness truncates to a byte or token cap.",
|
|
13652
14257
|
fix: "Move inline `<style>` and `<script>` blocks below the main content or into external files, and put `<main>` as early in the body as the layout allows. Where critical CSS must be inline, keep it to the rules that paint the first screen rather than the whole stylesheet.",
|
|
@@ -13667,6 +14272,13 @@ var PreambleTaxTokensBeforeTheFirstContentTokenAudit = class extends Audit {
|
|
|
13667
14272
|
}
|
|
13668
14273
|
};
|
|
13669
14274
|
audit(ctx) {
|
|
14275
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
14276
|
+
return this.notApplicable(
|
|
14277
|
+
"No page here can be attributed to this site, so its preamble was not measured.",
|
|
14278
|
+
"A page from this site whose preamble can be measured",
|
|
14279
|
+
unreadSiteReason(ctx.evidence)
|
|
14280
|
+
);
|
|
14281
|
+
}
|
|
13670
14282
|
const page = ctx.pages[0];
|
|
13671
14283
|
if (!page) {
|
|
13672
14284
|
return this.notApplicable(
|
|
@@ -13791,6 +14403,7 @@ var BoilerplateTaxAudit = class extends Audit {
|
|
|
13791
14403
|
weight: weightForGrade("B", "scored"),
|
|
13792
14404
|
defaultPriority: "medium",
|
|
13793
14405
|
dossier: "docs/evidence/audits/content-extraction/boilerplate-tax.md",
|
|
14406
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
13794
14407
|
guidance: {
|
|
13795
14408
|
impact: "An agent answering a question about a site fetches several of its pages. If each fetch delivers the same navigation, the same promotional header and the same footer around a thin body, the agent pays for those tokens once per fetch and learns nothing new from them. The cost compounds with every page, and the distinct content it came for competes for what is left of the context window.",
|
|
13796
14409
|
fix: "Cut repeated chrome down to what a reader needs on every page: collapse mega-menus to a short nav, move legal and marketing boilerplate to the pages that are about it, and let each page carry more of its own content. Where the chrome must stay for humans, keeping it out of `<main>` at least lets an extractor drop it.",
|
|
@@ -13916,6 +14529,7 @@ var ExtractionDeterminismAudit = class extends Audit {
|
|
|
13916
14529
|
weight: weightForGrade("B", "scored"),
|
|
13917
14530
|
defaultPriority: "high",
|
|
13918
14531
|
dossier: "docs/evidence/audits/content-extraction/extraction-determinism.md",
|
|
14532
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
13919
14533
|
guidance: {
|
|
13920
14534
|
impact: "Every agent pipeline strips a page down before a model reads it, and they do not all strip the same way. When the extractors disagree, the same URL yields different answers depending on which tool fetched it \u2014 and the page cannot be tested, because there is no single thing it says. When readability declines a page outright, the most widely deployed extractor of the three hands an agent nothing at all.",
|
|
13921
14535
|
fix: "Put the article in one container \u2014 `<main>` or `<article>` \u2014 with the chrome outside it, and keep the largest block of prose on the page the one you want quoted. Readability keys on paragraph density and link density, so a body split across many small wrappers, or padded with link-heavy blocks, is what makes the three disagree.",
|
|
@@ -13925,6 +14539,13 @@ var ExtractionDeterminismAudit = class extends Audit {
|
|
|
13925
14539
|
}
|
|
13926
14540
|
};
|
|
13927
14541
|
audit(ctx) {
|
|
14542
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
14543
|
+
return this.notApplicable(
|
|
14544
|
+
"No page here can be attributed to this site, so no extraction could be compared.",
|
|
14545
|
+
"A page from this site whose extraction can be compared",
|
|
14546
|
+
unreadSiteReason(ctx.evidence)
|
|
14547
|
+
);
|
|
14548
|
+
}
|
|
13928
14549
|
const page = ctx.pages[0];
|
|
13929
14550
|
if (!page) {
|
|
13930
14551
|
return this.notApplicable(
|
|
@@ -14093,6 +14714,7 @@ var LlmsTxtExistsAudit = class extends Audit {
|
|
|
14093
14714
|
evidenceGrade: "C",
|
|
14094
14715
|
tier: "informative",
|
|
14095
14716
|
dossier: "docs/evidence/audits/machine-discovery/llms-txt-exists.md",
|
|
14717
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
14096
14718
|
defaultPriority: "low",
|
|
14097
14719
|
guidance: {
|
|
14098
14720
|
impact: "Thousands of sites publish an llms.txt, including every major AI lab, but as publishers rather than readers. No vendor documentation names an agent that fetches it, and Google Search Central states Search ignores it. Publishing one is cheap and harmless; it is not a documented path to any AI answer.",
|
|
@@ -14173,6 +14795,7 @@ var LlmsTxtStructureAudit = class extends Audit {
|
|
|
14173
14795
|
evidenceGrade: "C",
|
|
14174
14796
|
tier: "informative",
|
|
14175
14797
|
dossier: "docs/evidence/audits/machine-discovery/llms-txt-structure.md",
|
|
14798
|
+
requires: ["origin-reachable"],
|
|
14176
14799
|
defaultPriority: "low",
|
|
14177
14800
|
guidance: {
|
|
14178
14801
|
impact: "The reference llms.txt parser extracts the blockquote as a `summary` field and the H2 headings as a `sections` map, so a file that carries both is machine-navigable: an agent can read the summary and pick a section instead of consuming the whole file. No vendor documents an agent behaving differently when either element is absent, so this is reported, not scored.",
|
|
@@ -14236,6 +14859,7 @@ var LlmsTxtLinkDescriptionsAudit = class extends Audit {
|
|
|
14236
14859
|
evidenceGrade: "C",
|
|
14237
14860
|
tier: "informative",
|
|
14238
14861
|
dossier: "docs/evidence/audits/machine-discovery/llms-txt-link-descriptions.md",
|
|
14862
|
+
requires: ["origin-reachable"],
|
|
14239
14863
|
defaultPriority: "medium",
|
|
14240
14864
|
guidance: {
|
|
14241
14865
|
impact: "Links without descriptions force AI agents to visit every page to understand its content, wasting crawl budget and slowing down response generation. Described links let agents filter relevant pages instantly.",
|
|
@@ -14331,6 +14955,7 @@ var LlmsTxtLinksValidAudit = class extends Audit {
|
|
|
14331
14955
|
evidenceGrade: "C",
|
|
14332
14956
|
tier: "informative",
|
|
14333
14957
|
dossier: "docs/evidence/audits/machine-discovery/llms-txt-links-valid.md",
|
|
14958
|
+
requires: ["origin-reachable"],
|
|
14334
14959
|
defaultPriority: "low",
|
|
14335
14960
|
guidance: {
|
|
14336
14961
|
impact: "A broken link inside llms.txt points at nothing, the same as a broken link anywhere else. No documented agent consumer reads the file, so the cost is to any human or tool that follows it, not to a measured AI outcome.",
|
|
@@ -14412,6 +15037,7 @@ var LlmsFullTxtAudit = class extends Audit {
|
|
|
14412
15037
|
evidenceGrade: "C",
|
|
14413
15038
|
tier: "informative",
|
|
14414
15039
|
dossier: "docs/evidence/audits/machine-discovery/llms-full-txt.md",
|
|
15040
|
+
requires: ["origin-reachable"],
|
|
14415
15041
|
defaultPriority: "high",
|
|
14416
15042
|
guidance: {
|
|
14417
15043
|
impact: "Without llms-full.txt, AI agents must crawl your site page by page, which is slow and often incomplete. This means AI assistants give shallow or outdated answers about your products and services.",
|
|
@@ -14423,6 +15049,13 @@ var LlmsFullTxtAudit = class extends Audit {
|
|
|
14423
15049
|
}
|
|
14424
15050
|
};
|
|
14425
15051
|
audit(ctx) {
|
|
15052
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
15053
|
+
return this.notApplicable(
|
|
15054
|
+
"No response here can be attributed to this site, so no llms-full.txt was judged.",
|
|
15055
|
+
"GET /llms-full.txt returns 200",
|
|
15056
|
+
unreadSiteReason(ctx.evidence)
|
|
15057
|
+
);
|
|
15058
|
+
}
|
|
14426
15059
|
const result = ctx.rootFiles["/llms-full.txt"];
|
|
14427
15060
|
if (!result || !isOk5(result)) {
|
|
14428
15061
|
return this.fail(
|
|
@@ -14480,6 +15113,7 @@ var SitemapExistsAudit = class extends Audit {
|
|
|
14480
15113
|
evidenceGrade: "A",
|
|
14481
15114
|
tier: "scored",
|
|
14482
15115
|
dossier: "docs/evidence/audits/machine-discovery/sitemap-exists.md",
|
|
15116
|
+
requires: ["origin-reachable"],
|
|
14483
15117
|
defaultPriority: "critical",
|
|
14484
15118
|
guidance: {
|
|
14485
15119
|
impact: "Without a sitemap, AI crawlers must discover your pages solely through link-following, which is slow and incomplete. Pages deep in your site hierarchy may never be found, meaning AI search engines like Perplexity and ChatGPT Browse cannot surface your full content.",
|
|
@@ -14619,6 +15253,7 @@ var DiscoveryIndexCoverageAudit = class extends Audit {
|
|
|
14619
15253
|
evidenceGrade: "B",
|
|
14620
15254
|
tier: "scored",
|
|
14621
15255
|
dossier: "docs/evidence/audits/machine-discovery/discovery-index-coverage.md",
|
|
15256
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
14622
15257
|
defaultPriority: "medium",
|
|
14623
15258
|
guidance: {
|
|
14624
15259
|
impact: "A page listed in no discovery index is reachable only through the link graph, and the major AI crawlers do not execute JavaScript \u2014 so a page missing from both the sitemap and llms.txt can stay invisible to AI search even though it exists on your site.",
|
|
@@ -14751,6 +15386,7 @@ var SitemapAbsoluteUrlsAudit = class extends Audit {
|
|
|
14751
15386
|
evidenceGrade: "B",
|
|
14752
15387
|
tier: "scored",
|
|
14753
15388
|
dossier: "docs/evidence/audits/machine-discovery/sitemap-absolute-urls.md",
|
|
15389
|
+
requires: ["origin-reachable"],
|
|
14754
15390
|
defaultPriority: "high",
|
|
14755
15391
|
guidance: {
|
|
14756
15392
|
impact: "Relative URLs in your sitemap cannot be resolved by AI crawlers, causing them to silently skip those pages. Any page listed with a relative URL is effectively invisible to AI search engines.",
|
|
@@ -14855,6 +15491,7 @@ var SitemapLastmodAudit = class extends Audit {
|
|
|
14855
15491
|
evidenceGrade: "A",
|
|
14856
15492
|
tier: "scored",
|
|
14857
15493
|
dossier: "docs/evidence/audits/machine-discovery/sitemap-lastmod.md",
|
|
15494
|
+
requires: ["origin-reachable"],
|
|
14858
15495
|
defaultPriority: "medium",
|
|
14859
15496
|
guidance: {
|
|
14860
15497
|
impact: "Without <lastmod> dates, AI crawlers must re-fetch every page on every visit because they cannot tell which pages have changed. This wastes crawl budget and delays indexing of your freshest content.",
|
|
@@ -14991,6 +15628,7 @@ var RssFeedAudit = class extends Audit {
|
|
|
14991
15628
|
evidenceGrade: "B",
|
|
14992
15629
|
tier: "scored",
|
|
14993
15630
|
dossier: "docs/evidence/audits/machine-discovery/rss-feed.md",
|
|
15631
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
14994
15632
|
defaultPriority: "medium",
|
|
14995
15633
|
guidance: {
|
|
14996
15634
|
impact: "Without an RSS/Atom feed, AI agents have no efficient way to track new and updated content on your site. They must re-crawl your entire site to find changes, which means your latest posts and pages may take much longer to appear in AI search results.",
|
|
@@ -15001,6 +15639,13 @@ var RssFeedAudit = class extends Audit {
|
|
|
15001
15639
|
}
|
|
15002
15640
|
};
|
|
15003
15641
|
async audit(ctx) {
|
|
15642
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
15643
|
+
return this.notApplicable(
|
|
15644
|
+
"No response here can be attributed to this site, so no feed was judged.",
|
|
15645
|
+
"Feed returns HTTP 200",
|
|
15646
|
+
unreadSiteReason(ctx.evidence)
|
|
15647
|
+
);
|
|
15648
|
+
}
|
|
15004
15649
|
const links = autodiscoveryLinks(ctx);
|
|
15005
15650
|
const feed = await findFeedResult(ctx, links);
|
|
15006
15651
|
const linkNote = links.length > 0 ? `autodiscovery <link> present (${links[0].url})` : "no autodiscovery <link> in <head>";
|
|
@@ -15079,6 +15724,7 @@ var RssFeedContentAudit = class extends Audit {
|
|
|
15079
15724
|
evidenceGrade: "C",
|
|
15080
15725
|
tier: "informative",
|
|
15081
15726
|
dossier: "docs/evidence/audits/machine-discovery/rss-feed-content.md",
|
|
15727
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15082
15728
|
defaultPriority: "medium",
|
|
15083
15729
|
guidance: {
|
|
15084
15730
|
impact: "Truncated RSS feed items force AI agents to visit each page individually, increasing crawl time and often resulting in incomplete indexing. Full-content feeds let agents ingest all your articles in a single request, producing richer AI-generated answers.",
|
|
@@ -15234,6 +15880,7 @@ var InContentLinksAudit = class extends Audit {
|
|
|
15234
15880
|
evidenceGrade: "A",
|
|
15235
15881
|
tier: "scored",
|
|
15236
15882
|
dossier: "docs/evidence/audits/machine-discovery/in-content-links.md",
|
|
15883
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15237
15884
|
defaultPriority: "medium",
|
|
15238
15885
|
guidance: {
|
|
15239
15886
|
impact: "Google can only crawl a link that is an <a> element with an href, and the measured behaviour of GPTBot and ClaudeBot is that they do not execute JavaScript \u2014 so a page whose only links are in a client-rendered nav is a dead end for them. Links inside the body copy also tell an agent which pages belong together, which template chrome (identical on every page) cannot.",
|
|
@@ -15310,6 +15957,7 @@ var NoBrokenLinksAudit = class _NoBrokenLinksAudit extends Audit {
|
|
|
15310
15957
|
evidenceGrade: "A",
|
|
15311
15958
|
tier: "scored",
|
|
15312
15959
|
dossier: "docs/evidence/audits/machine-discovery/no-broken-links.md",
|
|
15960
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15313
15961
|
defaultPriority: "high",
|
|
15314
15962
|
guidance: {
|
|
15315
15963
|
impact: "Broken internal links waste AI crawlers' limited crawl budget by sending them to dead ends. This means fewer of your pages get indexed, and users asking AI about your site may encounter errors or missing information.",
|
|
@@ -15410,6 +16058,7 @@ var CorsAiFilesAudit = class extends Audit {
|
|
|
15410
16058
|
evidenceGrade: "C",
|
|
15411
16059
|
tier: "informative",
|
|
15412
16060
|
dossier: "docs/evidence/audits/machine-discovery/cors-ai-files.md",
|
|
16061
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15413
16062
|
defaultPriority: "medium",
|
|
15414
16063
|
guidance: {
|
|
15415
16064
|
impact: "Browser-based AI tools, ChatGPT plugins, and MCP clients all run in browser contexts governed by the same-origin policy. Without CORS headers on your llms.txt and AI catalog, these agents receive a network error instead of your content \u2014 making your AI-facing files completely invisible to the fastest-growing category of AI consumers.",
|
|
@@ -15531,6 +16180,7 @@ var AiFileDeliveryAudit = class extends Audit {
|
|
|
15531
16180
|
evidenceGrade: "B",
|
|
15532
16181
|
tier: "informative",
|
|
15533
16182
|
dossier: "docs/evidence/audits/machine-discovery/ai-file-delivery.md",
|
|
16183
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15534
16184
|
defaultPriority: "medium",
|
|
15535
16185
|
guidance: {
|
|
15536
16186
|
impact: "Incorrect Content-Type headers cause AI agents to misparse your files: JSON served as text/html breaks structured-data extraction, an XML sitemap served as text/plain hides it from crawl discovery, and llms.txt served as application/octet-stream triggers a download instead of a read. Missing caching headers make every agent re-download the full file on each visit rather than revalidating it.",
|
|
@@ -15624,6 +16274,7 @@ var NoBrokenAiEndpointsAudit = class extends Audit {
|
|
|
15624
16274
|
evidenceGrade: "A",
|
|
15625
16275
|
tier: "scored",
|
|
15626
16276
|
dossier: "docs/evidence/audits/machine-discovery/no-broken-ai-endpoints.md",
|
|
16277
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15627
16278
|
defaultPriority: "high",
|
|
15628
16279
|
guidance: {
|
|
15629
16280
|
impact: "Broken URLs in your AI manifest files (ai-catalog.json, llms.txt, navigation.json) cause agents to lose trust in your entire manifest. After encountering broken links, AI systems may stop following any of your listed endpoints, effectively making all your AI-facing resources undiscoverable.",
|
|
@@ -15702,6 +16353,15 @@ var NoBrokenAiEndpointsAudit = class extends Audit {
|
|
|
15702
16353
|
for (const url of allUrls) {
|
|
15703
16354
|
if (await isSafeUrl(url)) urls.push(url);
|
|
15704
16355
|
}
|
|
16356
|
+
if (urls.length === 0) {
|
|
16357
|
+
return this.warn(
|
|
16358
|
+
`${allUrls.length} AI endpoint URL(s) are listed, and none of them could be requested: each names localhost, a private address, or a host that does not resolve.`,
|
|
16359
|
+
"All URLs from AI-related files return 200",
|
|
16360
|
+
`${allUrls.length} URL(s) listed, 0 reachable to check`,
|
|
16361
|
+
void 0,
|
|
16362
|
+
page?.url
|
|
16363
|
+
);
|
|
16364
|
+
}
|
|
15705
16365
|
const results = await Promise.all(
|
|
15706
16366
|
urls.map(async (url) => {
|
|
15707
16367
|
try {
|
|
@@ -15893,6 +16553,7 @@ var AiCrawlerSurfaceReachabilityAudit = class extends Audit {
|
|
|
15893
16553
|
evidenceGrade: "A",
|
|
15894
16554
|
tier: "scored",
|
|
15895
16555
|
dossier: "docs/evidence/audits/machine-discovery/ai-crawler-surface-reachability.md",
|
|
16556
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
15896
16557
|
defaultPriority: "high",
|
|
15897
16558
|
guidance: {
|
|
15898
16559
|
impact: "The Sitemap: directive is host-global and user-agent independent (RFC 9309 \xA72.2.3), but the sitemap file, the feed files and every URL they list obey per-crawler rules \u2014 and under \xA72.2.1 a crawler with a named group ignores the '*' group entirely. OpenAI documents the consequence at the extreme: 'Sites that are opted out of OAI-SearchBot will not be shown in ChatGPT search answers.' So for any crawler whose named group disallows the advertised sitemap or feed path, or a majority of the URLs the sitemap lists, the site's whole pull-indexing surface is unreachable to that agent no matter how good the sitemap is. The common trigger is a bot-blocking plugin adding a broad pattern (Disallow: /*.xml$, Disallow: /feed/, Disallow: /) to an AI-bot group while the site keeps advertising those exact paths.",
|
|
@@ -16078,6 +16739,7 @@ var SitemapLastmodVerifiabilityAudit = class extends Audit {
|
|
|
16078
16739
|
evidenceGrade: "A",
|
|
16079
16740
|
tier: "scored",
|
|
16080
16741
|
dossier: "docs/evidence/audits/machine-discovery/sitemap-lastmod-verifiability.md",
|
|
16742
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
16081
16743
|
defaultPriority: "medium",
|
|
16082
16744
|
guidance: {
|
|
16083
16745
|
impact: `Google states it uses <lastmod> "if it's consistently and verifiably (for example by comparing to the last modification of the page) accurate". lastmod is therefore a conditional signal an engine silently discards on divergence \u2014 and it is the only freshness hint a pull-based AI crawler gets from a sitemap. If sampled values disagree with every available page-level signal for a material share of URLs, the freshness channel is inert and re-crawl scheduling degrades to organic rediscovery. Two specific pathologies are detectable without guessing: over 90% of URLs sharing one lastmod equal to the last deploy date \u2014 a build stamp, exactly the pattern Google's "copyright date is not significant" rule disqualifies \u2014 and a lastmod in the future relative to the scan, which is never valid.`,
|
|
@@ -16459,6 +17121,7 @@ var CheckoutOfferFieldMappingAudit = class _CheckoutOfferFieldMappingAudit exten
|
|
|
16459
17121
|
evidenceGrade: "A",
|
|
16460
17122
|
tier: "scored",
|
|
16461
17123
|
dossier: "docs/evidence/audits/agentic-commerce/checkout-offer-field-mapping.md",
|
|
17124
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
16462
17125
|
applicablePageTypes: ["product"],
|
|
16463
17126
|
defaultPriority: "high",
|
|
16464
17127
|
guidance: {
|
|
@@ -16655,6 +17318,7 @@ var AgentCommerceFeedParityAudit = class extends Audit {
|
|
|
16655
17318
|
evidenceGrade: "A",
|
|
16656
17319
|
tier: "scored",
|
|
16657
17320
|
dossier: "docs/evidence/audits/machine-discovery/agent-commerce-feed-parity.md",
|
|
17321
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
16658
17322
|
defaultPriority: "high",
|
|
16659
17323
|
guidance: {
|
|
16660
17324
|
impact: `Google's automatic item updates repair feed/page discrepancies "using the structured data markup the crawlers find on your website", and state that where extractors cannot determine price, availability and condition, "your products will be subject to item-level disapprovals". Merchant Center separately requires that feed availability match the landing page and that price match the landing page and checkout. OpenAI's Product Feed Spec requires a strictly larger per-item set than Google's rich-result minimum: a stable item_id (<=100 chars), brand (<=70), seller_name, target_countries as ISO 3166-1 alpha-2, a plain-text description under 5000 characters, availability from a fixed enum, and price with an ISO 4217 currency. Falsifiable claim: a PDP missing brand, seller, itemCondition-as-URL, a stable SKU or a country signal passes every Google rich-result test yet cannot be reconciled by automatic item updates, so feed rejections are silent and unattributable. Second claim, sharper: where the JSON-LD price disagrees with the price the page renders, automatic item updates overwrite the feed with one value while an agent reading the page quotes the other.`,
|
|
@@ -17195,6 +17859,7 @@ var ConditionalRequestSupportAudit = class extends Audit {
|
|
|
17195
17859
|
weight: weightForGrade("B", "scored"),
|
|
17196
17860
|
defaultPriority: "medium",
|
|
17197
17861
|
dossier: "docs/evidence/audits/machine-discovery/conditional-request-support.md",
|
|
17862
|
+
requires: ["origin-reachable"],
|
|
17198
17863
|
guidance: {
|
|
17199
17864
|
impact: 'A crawler that wants to know what changed re-reads your sitemap and your feed on a schedule. If those responses carry no `ETag` and no `Last-Modified`, it cannot ask "has this changed?" \u2014 it can only download the file again, every time, forever. The cost is yours as much as theirs: bandwidth you serve for no new information, and a crawl budget spent re-reading a list instead of fetching the pages on it. A validator that changes on every build is the same cost wearing a correct-looking header.',
|
|
17200
17865
|
fix: "Emit a strong `ETag` derived from the file\u2019s content, not from the build, and a `Last-Modified` that moves only when the content does. Answer `If-None-Match` and `If-Modified-Since` with 304 and an empty body. Keep `no-store` and `private` off public discovery surfaces \u2014 they tell a crawler not to keep the copy it just paid for.",
|
|
@@ -17348,6 +18013,7 @@ var FeedEntryIdentityAndCanonicalIntegrityAudit = class extends Audit {
|
|
|
17348
18013
|
weight: weightForGrade("B", "scored"),
|
|
17349
18014
|
defaultPriority: "medium",
|
|
17350
18015
|
dossier: "docs/evidence/audits/machine-discovery/feed-entry-identity-and-canonical-integrity.md",
|
|
18016
|
+
requires: ["origin-reachable"],
|
|
17351
18017
|
guidance: {
|
|
17352
18018
|
impact: 'A feed is how a consumer tracks what changed without re-crawling the site, and identity is what makes that possible: the id says "this is the same item you saw last time". An entry with no id, or with an id that repeats, forces the consumer to guess \u2014 usually by URL, which is exactly the thing that changes. A link that carries `utm_` parameters or redirects somewhere else creates a second address for one page, so the item the consumer stores is not the page the site considers canonical.',
|
|
17353
18019
|
fix: "Give every entry a stable id \u2014 an `atom:id` that never changes, or an RSS `<guid>` that is an absolute URL when `isPermaLink` is true \u2014 and never reuse one. Point item links at the canonical URL itself, with no tracking parameters and no redirect in between. Serve the feed as its registered media type, with no byte-order mark before the first element.",
|
|
@@ -17527,6 +18193,7 @@ var RootTextFileResolutionIntegrityAudit = class extends Audit {
|
|
|
17527
18193
|
weight: weightForGrade("B", "scored"),
|
|
17528
18194
|
defaultPriority: "medium",
|
|
17529
18195
|
dossier: "docs/evidence/audits/machine-discovery/root-text-file-resolution-integrity.md",
|
|
18196
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
17530
18197
|
guidance: {
|
|
17531
18198
|
impact: "IndexNow proves ownership by fetching `https://host/{key}.txt` and byte-comparing the body to the key, and six engines discard the submission when that comparison fails. The same property decides whether any other root `.txt` file means anything: if an origin answers 200 for a path that does not exist, then a 200 for `/llms.txt` is not evidence the file is there. A catch-all rewrite ahead of static file serving turns every one of those signals into noise, with no visible symptom on the site itself.",
|
|
17532
18199
|
fix: "Serve root-level `.txt` paths from static files and let a missing one answer 404. Order the static-file handler ahead of any SPA or catch-all rewrite, and make sure the rewrite does not cover `*.txt`. Serve `/robots.txt` as `text/plain`, not as `text/html` or `application/octet-stream`.",
|
|
@@ -17702,6 +18369,7 @@ var ThreeWayFreshnessLagAudit = class extends Audit {
|
|
|
17702
18369
|
weight: weightForGrade("B", "scored"),
|
|
17703
18370
|
defaultPriority: "medium",
|
|
17704
18371
|
dossier: "docs/evidence/audits/machine-discovery/three-way-freshness-lag.md",
|
|
18372
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
17705
18373
|
guidance: {
|
|
17706
18374
|
impact: "A pull-based crawler fetches the sitemap and the feed on a schedule and reads nothing else. When those two surfaces trail the site, everything published in between is discoverable only by link-following, which is the slow path the site published a sitemap to avoid. A feed whose `lastBuildDate` is older than its own newest item is worse than stale: consumers that poll conditionally on that timestamp skip the feed entirely, so the new items are never read at all.",
|
|
17707
18375
|
fix: "Regenerate the sitemap and the feed when content changes, not on a nightly cron that can fail silently. Stamp `<lastBuildDate>` (or the Atom feed-level `<updated>`) from the newest item at generation time. Order feed items newest-first, since many consumers read only the head. Remove sitemap entries whose URLs 404 or are noindex.",
|
|
@@ -17856,6 +18524,7 @@ var WebsubHubAdvertisementAudit = class extends Audit {
|
|
|
17856
18524
|
weight: 0,
|
|
17857
18525
|
defaultPriority: "low",
|
|
17858
18526
|
dossier: "docs/evidence/audits/machine-discovery/websub-hub-advertisement.md",
|
|
18527
|
+
requires: ["origin-reachable"],
|
|
17859
18528
|
guidance: {
|
|
17860
18529
|
impact: "A hub subscription is verified against the feed\u2019s own `rel=self`. When that link is missing, relative, or points at a different URL than the one the feed is served from, verification cannot complete, and the push path degrades to whatever polling cadence subscribers happen to use. The publisher sees a hub that looks configured and no error anywhere. The benefit side is unproven: WebSub is a W3C Recommendation, but no AI answer engine is documented as a subscriber, which is why this audit reports and does not score.",
|
|
17861
18530
|
fix: "Advertise the hub and the canonical topic URL in the feed\u2019s `Link:` response headers, which is where a subscriber looks first. Emit exactly one `rel=self` with an absolute URL identical to the address the feed is served from, and at least one `rel=hub` over HTTPS. If you run no hub, a hosted one (Google\u2019s pubsubhubbub, Superfeedr, websub.rocks) needs only the two link relations.",
|
|
@@ -17999,6 +18668,7 @@ var JsonLdPresentAudit = class extends Audit {
|
|
|
17999
18668
|
evidenceGrade: "A",
|
|
18000
18669
|
tier: "scored",
|
|
18001
18670
|
dossier: "docs/evidence/audits/structured-data/json-ld-present.md",
|
|
18671
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18002
18672
|
defaultPriority: "critical",
|
|
18003
18673
|
guidance: {
|
|
18004
18674
|
impact: "Without any JSON-LD structured data, AI agents like ChatGPT and Perplexity treat your site as unstructured text with no machine-readable identity. Your brand, products, and services become invisible to AI-powered discovery, search, and recommendation systems.",
|
|
@@ -18062,6 +18732,7 @@ var SchemaValidationAudit = class extends Audit {
|
|
|
18062
18732
|
evidenceGrade: "A",
|
|
18063
18733
|
tier: "scored",
|
|
18064
18734
|
dossier: "docs/evidence/audits/structured-data/schema-validation.md",
|
|
18735
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18065
18736
|
defaultPriority: "critical",
|
|
18066
18737
|
guidance: {
|
|
18067
18738
|
impact: "JSON-LD blocks missing @context or @type are silently ignored by every schema consumer, including Google, ChatGPT plugins, and RAG pipelines. Even if you have structured data on the page, invalid blocks provide zero value to AI agents.",
|
|
@@ -18188,6 +18859,7 @@ var OrganizationSchemaAudit = class extends Audit {
|
|
|
18188
18859
|
evidenceGrade: "A",
|
|
18189
18860
|
tier: "scored",
|
|
18190
18861
|
dossier: "docs/evidence/audits/structured-data/organization-schema.md",
|
|
18862
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18191
18863
|
applicablePageTypes: ["homepage"],
|
|
18192
18864
|
defaultPriority: "high",
|
|
18193
18865
|
guidance: {
|
|
@@ -18291,6 +18963,7 @@ var BreadcrumbSchemaAudit = class extends Audit {
|
|
|
18291
18963
|
evidenceGrade: "A",
|
|
18292
18964
|
tier: "scored",
|
|
18293
18965
|
dossier: "docs/evidence/audits/structured-data/breadcrumb-schema.md",
|
|
18966
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18294
18967
|
applicablePageTypes: ["category", "product", "content"],
|
|
18295
18968
|
defaultPriority: "medium",
|
|
18296
18969
|
guidance: {
|
|
@@ -18413,6 +19086,7 @@ var ArticleSchemaAudit = class extends Audit {
|
|
|
18413
19086
|
evidenceGrade: "A",
|
|
18414
19087
|
tier: "scored",
|
|
18415
19088
|
dossier: "docs/evidence/audits/structured-data/article-schema.md",
|
|
19089
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18416
19090
|
applicablePageTypes: ["content"],
|
|
18417
19091
|
defaultPriority: "high",
|
|
18418
19092
|
guidance: {
|
|
@@ -18539,6 +19213,7 @@ var FaqPageSchemaAudit = class extends Audit {
|
|
|
18539
19213
|
evidenceGrade: "C",
|
|
18540
19214
|
tier: "informative",
|
|
18541
19215
|
dossier: "docs/evidence/audits/structured-data/faqpage-schema.md",
|
|
19216
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18542
19217
|
defaultPriority: "medium",
|
|
18543
19218
|
guidance: {
|
|
18544
19219
|
impact: "AI answer engines like Perplexity and Google SGE give priority to FAQ-structured content for direct answers. Without FAQPage schema, your Q&A content is treated as unstructured text and is less likely to be surfaced as a featured answer in AI-generated responses.",
|
|
@@ -18692,6 +19367,7 @@ var ServiceSchemaAudit = class _ServiceSchemaAudit extends Audit {
|
|
|
18692
19367
|
evidenceGrade: "A",
|
|
18693
19368
|
tier: "scored",
|
|
18694
19369
|
dossier: "docs/evidence/audits/structured-data/service-schema.md",
|
|
19370
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18695
19371
|
// Where a service business publishes its offerings. NOT ['product'] —
|
|
18696
19372
|
// that was inherited from the pre-split audit and inverted this check:
|
|
18697
19373
|
// it skipped every service site (no product page in the scan) and ran only
|
|
@@ -18835,6 +19511,7 @@ var SpeakableSchemaAudit = class _SpeakableSchemaAudit extends Audit {
|
|
|
18835
19511
|
evidenceGrade: "B",
|
|
18836
19512
|
tier: "scored",
|
|
18837
19513
|
dossier: "docs/evidence/audits/structured-data/speakable-schema.md",
|
|
19514
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18838
19515
|
// News and article publishing is the whole documented scope of the
|
|
18839
19516
|
// feature, so a scan with no content page never runs this audit at all.
|
|
18840
19517
|
// The runtime guard below repeats the precondition for the pages that
|
|
@@ -18933,6 +19610,7 @@ var HowToSchemaAudit = class extends Audit {
|
|
|
18933
19610
|
evidenceGrade: "C",
|
|
18934
19611
|
tier: "informative",
|
|
18935
19612
|
dossier: "docs/evidence/audits/structured-data/howto-schema.md",
|
|
19613
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
18936
19614
|
applicablePageTypes: ["content"],
|
|
18937
19615
|
defaultPriority: "low",
|
|
18938
19616
|
guidance: {
|
|
@@ -19081,6 +19759,7 @@ var LocalBusinessSchemaAudit = class extends Audit {
|
|
|
19081
19759
|
evidenceGrade: "A",
|
|
19082
19760
|
tier: "scored",
|
|
19083
19761
|
dossier: "docs/evidence/audits/structured-data/local-business-schema.md",
|
|
19762
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19084
19763
|
applicablePageTypes: ["homepage"],
|
|
19085
19764
|
defaultPriority: "medium",
|
|
19086
19765
|
guidance: {
|
|
@@ -19246,6 +19925,7 @@ var ReviewSchemaAudit = class extends Audit {
|
|
|
19246
19925
|
evidenceGrade: "A",
|
|
19247
19926
|
tier: "scored",
|
|
19248
19927
|
dossier: "docs/evidence/audits/structured-data/review-schema.md",
|
|
19928
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19249
19929
|
applicablePageTypes: ["homepage", "product"],
|
|
19250
19930
|
defaultPriority: "medium",
|
|
19251
19931
|
guidance: {
|
|
@@ -19363,6 +20043,7 @@ var AuthorSchemaAudit = class extends Audit {
|
|
|
19363
20043
|
evidenceGrade: "C",
|
|
19364
20044
|
tier: "informative",
|
|
19365
20045
|
dossier: "docs/evidence/audits/structured-data/author-schema.md",
|
|
20046
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19366
20047
|
applicablePageTypes: ["content"],
|
|
19367
20048
|
defaultPriority: "medium",
|
|
19368
20049
|
guidance: {
|
|
@@ -19485,6 +20166,7 @@ var ProductDetailsAudit = class extends Audit {
|
|
|
19485
20166
|
evidenceGrade: "A",
|
|
19486
20167
|
tier: "scored",
|
|
19487
20168
|
dossier: "docs/evidence/audits/structured-data/advanced-product-details.md",
|
|
20169
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19488
20170
|
applicablePageTypes: ["product"],
|
|
19489
20171
|
defaultPriority: "medium",
|
|
19490
20172
|
guidance: {
|
|
@@ -19638,6 +20320,7 @@ var ClaimreviewAdvisoryAudit = class extends Audit {
|
|
|
19638
20320
|
evidenceGrade: "A",
|
|
19639
20321
|
tier: "informative",
|
|
19640
20322
|
dossier: "docs/evidence/audits/structured-data/claimreview-advisory.md",
|
|
20323
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19641
20324
|
defaultPriority: "low",
|
|
19642
20325
|
guidance: {
|
|
19643
20326
|
impact: "Google's fact check documentation states plainly: 'We're phasing out support for ClaimReview markup in Google Search', with no deprecation date, and notes only one ClaimReview element per page qualifies for rich results. A check that scored ClaimReview coverage as an AI-readiness win would therefore push publishers to invest in a channel its largest documented consumer is actively withdrawing from. FALSIFIABLE and grade A on the evidence, but it measures the state of an external product, not the quality of the site \u2014 which is exactly why it must not contribute to a score.",
|
|
@@ -19793,6 +20476,7 @@ var MetaDescriptionAudit = class extends Audit {
|
|
|
19793
20476
|
evidenceGrade: "B",
|
|
19794
20477
|
tier: "scored",
|
|
19795
20478
|
dossier: "docs/evidence/audits/answer-readiness/meta-description.md",
|
|
20479
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19796
20480
|
defaultPriority: "high",
|
|
19797
20481
|
guidance: {
|
|
19798
20482
|
impact: "Google sometimes uses the meta description as the search snippet when it describes the page more accurately than the body text, and AI Overviews and AI Mode inherit that snippet pipeline. A missing, keyword-stuffed or off-topic description means the summary shown alongside your page is written by someone else.",
|
|
@@ -19887,6 +20571,7 @@ var MetaAuthorAudit = class extends Audit {
|
|
|
19887
20571
|
evidenceGrade: "C",
|
|
19888
20572
|
tier: "informative",
|
|
19889
20573
|
dossier: "docs/evidence/audits/answer-readiness/meta-author.md",
|
|
20574
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19890
20575
|
applicablePageTypes: ["content"],
|
|
19891
20576
|
defaultPriority: "medium",
|
|
19892
20577
|
guidance: {
|
|
@@ -19935,6 +20620,7 @@ var UniqueMetaAudit = class extends Audit {
|
|
|
19935
20620
|
evidenceGrade: "C",
|
|
19936
20621
|
tier: "informative",
|
|
19937
20622
|
dossier: "docs/evidence/audits/answer-readiness/unique-meta.md",
|
|
20623
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
19938
20624
|
defaultPriority: "high",
|
|
19939
20625
|
guidance: {
|
|
19940
20626
|
impact: "AI crawlers use title and description pairs to distinguish between pages. Duplicate meta across pages causes agents to merge or skip content, meaning some pages become invisible in AI-generated answers.",
|
|
@@ -19945,6 +20631,13 @@ var UniqueMetaAudit = class extends Audit {
|
|
|
19945
20631
|
}
|
|
19946
20632
|
};
|
|
19947
20633
|
audit(ctx) {
|
|
20634
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
20635
|
+
return this.notApplicable(
|
|
20636
|
+
"No page here can be attributed to this site, so its metadata was not judged.",
|
|
20637
|
+
"Each page has a unique title + description combination",
|
|
20638
|
+
unreadSiteReason(ctx.evidence)
|
|
20639
|
+
);
|
|
20640
|
+
}
|
|
19948
20641
|
const canonicalGroups = /* @__PURE__ */ new Map();
|
|
19949
20642
|
for (const page of ctx.pages) {
|
|
19950
20643
|
let canon = (page.meta?.["canonical"] || page.url).trim();
|
|
@@ -19961,7 +20654,7 @@ var UniqueMetaAudit = class extends Audit {
|
|
|
19961
20654
|
}
|
|
19962
20655
|
const uniquePages = Array.from(canonicalGroups.values());
|
|
19963
20656
|
if (uniquePages.length < 2) {
|
|
19964
|
-
return this.
|
|
20657
|
+
return this.notApplicable(
|
|
19965
20658
|
"Only one distinct canonical page scanned; uniqueness check not applicable.",
|
|
19966
20659
|
"Each page has a unique title + description combination",
|
|
19967
20660
|
"1 distinct page scanned"
|
|
@@ -20048,6 +20741,7 @@ var CoreOpenGraphAudit = class extends Audit {
|
|
|
20048
20741
|
evidenceGrade: "A",
|
|
20049
20742
|
tier: "scored",
|
|
20050
20743
|
dossier: "docs/evidence/audits/answer-readiness/core-open-graph.md",
|
|
20744
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20051
20745
|
defaultPriority: "high",
|
|
20052
20746
|
guidance: {
|
|
20053
20747
|
impact: "Link-preview crawlers use Open Graph tags to build the card shown wherever your page is shared, and Google uses og:title and og:site_name as inputs to the title link and site name on a result \u2014 the same labels that carry into AI Overviews source cards. Without them the crawler falls back to guessing.",
|
|
@@ -20131,6 +20825,7 @@ var OgTypeAudit = class extends Audit {
|
|
|
20131
20825
|
evidenceGrade: "B",
|
|
20132
20826
|
tier: "scored",
|
|
20133
20827
|
dossier: "docs/evidence/audits/answer-readiness/og-type.md",
|
|
20828
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20134
20829
|
defaultPriority: "medium",
|
|
20135
20830
|
guidance: {
|
|
20136
20831
|
impact: "AI agents use og:type to classify page content for type-specific handling. Without it, agents treat every page as generic content, missing opportunities for article freshness scoring or product-specific handling.",
|
|
@@ -20194,6 +20889,7 @@ var OgImageAltAudit = class extends Audit {
|
|
|
20194
20889
|
evidenceGrade: "C",
|
|
20195
20890
|
tier: "informative",
|
|
20196
20891
|
dossier: "docs/evidence/audits/answer-readiness/og-image-alt.md",
|
|
20892
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20197
20893
|
defaultPriority: "medium",
|
|
20198
20894
|
guidance: {
|
|
20199
20895
|
impact: "AI agents cannot process images directly and rely on og:image:alt text to understand your page's visual content. Without alt text, the OG image is invisible to text-based AI systems generating answers about your page.",
|
|
@@ -20265,6 +20961,7 @@ var FaqSectionsAudit = class _FaqSectionsAudit extends Audit {
|
|
|
20265
20961
|
evidenceGrade: "C",
|
|
20266
20962
|
tier: "informative",
|
|
20267
20963
|
dossier: "docs/evidence/audits/answer-readiness/faq-sections.md",
|
|
20964
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20268
20965
|
defaultPriority: "medium",
|
|
20269
20966
|
guidance: {
|
|
20270
20967
|
impact: 'FAQ sections with clear question headings are the highest-priority extraction target for AI-generated answers and "People Also Ask" results. Without them, your content misses the most direct path to appearing in AI answer snippets.',
|
|
@@ -20359,6 +21056,7 @@ var QuestionHeadingsAudit = class _QuestionHeadingsAudit extends Audit {
|
|
|
20359
21056
|
evidenceGrade: "C",
|
|
20360
21057
|
tier: "informative",
|
|
20361
21058
|
dossier: "docs/evidence/audits/answer-readiness/question-headings.md",
|
|
21059
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20362
21060
|
defaultPriority: "medium",
|
|
20363
21061
|
guidance: {
|
|
20364
21062
|
impact: "AI answer engines directly match user questions to heading text. Question-formatted headings are the primary signal for identifying which section answers a specific query. Without them, agents must guess which section is relevant, reducing your content's match rate.",
|
|
@@ -20527,6 +21225,7 @@ var DatesOnContentAudit = class extends Audit {
|
|
|
20527
21225
|
evidenceGrade: "A",
|
|
20528
21226
|
tier: "scored",
|
|
20529
21227
|
dossier: "docs/evidence/audits/answer-readiness/dates-on-content.md",
|
|
21228
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20530
21229
|
applicablePageTypes: ["content"],
|
|
20531
21230
|
defaultPriority: "medium",
|
|
20532
21231
|
guidance: {
|
|
@@ -20618,6 +21317,7 @@ var FirstParagraphAnswersAudit = class extends Audit {
|
|
|
20618
21317
|
evidenceGrade: "C",
|
|
20619
21318
|
tier: "informative",
|
|
20620
21319
|
dossier: "docs/evidence/audits/answer-readiness/first-paragraph-answers.md",
|
|
21320
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20621
21321
|
applicablePageTypes: ["content"],
|
|
20622
21322
|
defaultPriority: "high",
|
|
20623
21323
|
guidance: {
|
|
@@ -20783,6 +21483,7 @@ var DirectDefinitionsAudit = class extends Audit {
|
|
|
20783
21483
|
evidenceGrade: "C",
|
|
20784
21484
|
tier: "informative",
|
|
20785
21485
|
dossier: "docs/evidence/audits/answer-readiness/direct-definitions.md",
|
|
21486
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20786
21487
|
applicablePageTypes: ["content"],
|
|
20787
21488
|
// Never a defect, so never above the actionable items.
|
|
20788
21489
|
defaultPriority: "low",
|
|
@@ -20841,6 +21542,7 @@ var ComparisonTablesAudit = class _ComparisonTablesAudit extends Audit {
|
|
|
20841
21542
|
evidenceGrade: "C",
|
|
20842
21543
|
tier: "informative",
|
|
20843
21544
|
dossier: "docs/evidence/audits/answer-readiness/comparison-tables.md",
|
|
21545
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20844
21546
|
applicablePageTypes: ["category", "product", "content"],
|
|
20845
21547
|
defaultPriority: "low",
|
|
20846
21548
|
guidance: {
|
|
@@ -20916,6 +21618,7 @@ var SpecificNumbersAudit = class _SpecificNumbersAudit extends Audit {
|
|
|
20916
21618
|
evidenceGrade: "B",
|
|
20917
21619
|
tier: "scored",
|
|
20918
21620
|
dossier: "docs/evidence/audits/answer-readiness/specific-numbers.md",
|
|
21621
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
20919
21622
|
defaultPriority: "medium",
|
|
20920
21623
|
guidance: {
|
|
20921
21624
|
impact: "AI answer engines strongly prefer content with concrete data points over vague claims. Pages with specific numbers, percentages, and metrics are ranked higher for data-driven queries because agents can cite exact figures in generated answers.",
|
|
@@ -20974,17 +21677,6 @@ var SpecificNumbersAudit = class _SpecificNumbersAudit extends Audit {
|
|
|
20974
21677
|
};
|
|
20975
21678
|
|
|
20976
21679
|
// src/audits/answer-readiness/content-without-clickthrough.ts
|
|
20977
|
-
function contentWordCount($) {
|
|
20978
|
-
const main = $("main").first();
|
|
20979
|
-
const extract = (sel) => {
|
|
20980
|
-
const clone = sel.clone();
|
|
20981
|
-
clone.find("script, style, noscript, template").remove();
|
|
20982
|
-
return clone.text().replace(/\s+/g, " ").trim();
|
|
20983
|
-
};
|
|
20984
|
-
let text3 = main.length ? extract(main) : "";
|
|
20985
|
-
if (!text3) text3 = extract($("body"));
|
|
20986
|
-
return text3.split(/\s+/).filter(Boolean).length;
|
|
20987
|
-
}
|
|
20988
21680
|
var TEASER_PATTERNS = [
|
|
20989
21681
|
/click\s+(here\s+)?to\s+read\s+more/i,
|
|
20990
21682
|
/contact\s+us\s+to\s+learn/i,
|
|
@@ -21007,6 +21699,7 @@ var ContentWithoutClickthroughAudit = class _ContentWithoutClickthroughAudit ext
|
|
|
21007
21699
|
evidenceGrade: "B",
|
|
21008
21700
|
tier: "scored",
|
|
21009
21701
|
dossier: "docs/evidence/audits/answer-readiness/content-without-clickthrough.md",
|
|
21702
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21010
21703
|
defaultPriority: "high",
|
|
21011
21704
|
guidance: {
|
|
21012
21705
|
impact: 'AI answer engines skip pages dominated by teaser content ("click to read more", "sign up to access"). These pages provide no extractable answers, so agents will never surface your content in AI-generated responses, costing you visibility in AI search.',
|
|
@@ -21017,6 +21710,13 @@ var ContentWithoutClickthroughAudit = class _ContentWithoutClickthroughAudit ext
|
|
|
21017
21710
|
}
|
|
21018
21711
|
};
|
|
21019
21712
|
audit(ctx) {
|
|
21713
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
21714
|
+
return this.notApplicable(
|
|
21715
|
+
"No page here can be attributed to this site, so its teasers were not judged.",
|
|
21716
|
+
'No "click to read more" or "contact us to learn" teasers dominating the page',
|
|
21717
|
+
unreadSiteReason(ctx.evidence)
|
|
21718
|
+
);
|
|
21719
|
+
}
|
|
21020
21720
|
const page = ctx.pages[0];
|
|
21021
21721
|
if (!page) {
|
|
21022
21722
|
return this.fail(
|
|
@@ -21058,7 +21758,7 @@ var ContentWithoutClickthroughAudit = class _ContentWithoutClickthroughAudit ext
|
|
|
21058
21758
|
return !head2.startsWith("<?xml");
|
|
21059
21759
|
});
|
|
21060
21760
|
if (checkPage) {
|
|
21061
|
-
const wordCount2 =
|
|
21761
|
+
const wordCount2 = getWordCount(checkPage.$);
|
|
21062
21762
|
if (wordCount2 < 50) {
|
|
21063
21763
|
return this.warn(
|
|
21064
21764
|
"Insufficient content to evaluate.",
|
|
@@ -21073,6 +21773,13 @@ var ContentWithoutClickthroughAudit = class _ContentWithoutClickthroughAudit ext
|
|
|
21073
21773
|
);
|
|
21074
21774
|
}
|
|
21075
21775
|
}
|
|
21776
|
+
if (!scanReadPageText(ctx.evidence)) {
|
|
21777
|
+
return this.notApplicable(
|
|
21778
|
+
"The scanned page served no readable text, so there was no content to judge for teasers.",
|
|
21779
|
+
'No "click to read more" or "contact us to learn" teasers dominating the page',
|
|
21780
|
+
unreadPageTextReason(ctx.evidence)
|
|
21781
|
+
);
|
|
21782
|
+
}
|
|
21076
21783
|
return this.pass(
|
|
21077
21784
|
"No excessive click-through teasers found.",
|
|
21078
21785
|
'No "click to read more" or "contact us to learn" teasers dominating the page',
|
|
@@ -21149,6 +21856,7 @@ var NamedAuthorAudit = class extends Audit {
|
|
|
21149
21856
|
evidenceGrade: "C",
|
|
21150
21857
|
tier: "informative",
|
|
21151
21858
|
dossier: "docs/evidence/audits/answer-readiness/named-author.md",
|
|
21859
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21152
21860
|
applicablePageTypes: ["content"],
|
|
21153
21861
|
defaultPriority: "high",
|
|
21154
21862
|
guidance: {
|
|
@@ -21275,6 +21983,7 @@ var AuthorSameAsAudit = class extends Audit {
|
|
|
21275
21983
|
evidenceGrade: "C",
|
|
21276
21984
|
tier: "informative",
|
|
21277
21985
|
dossier: "docs/evidence/audits/answer-readiness/author-same-as.md",
|
|
21986
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21278
21987
|
applicablePageTypes: ["content"],
|
|
21279
21988
|
defaultPriority: "medium",
|
|
21280
21989
|
guidance: {
|
|
@@ -21394,6 +22103,7 @@ var AuthorPageAudit = class extends Audit {
|
|
|
21394
22103
|
evidenceGrade: "C",
|
|
21395
22104
|
tier: "informative",
|
|
21396
22105
|
dossier: "docs/evidence/audits/answer-readiness/author-page.md",
|
|
22106
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21397
22107
|
applicablePageTypes: ["content"],
|
|
21398
22108
|
defaultPriority: "medium",
|
|
21399
22109
|
guidance: {
|
|
@@ -21528,6 +22238,7 @@ var AboutCredentialsAudit = class extends Audit {
|
|
|
21528
22238
|
evidenceGrade: "C",
|
|
21529
22239
|
tier: "informative",
|
|
21530
22240
|
dossier: "docs/evidence/audits/answer-readiness/about-credentials.md",
|
|
22241
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21531
22242
|
defaultPriority: "medium",
|
|
21532
22243
|
guidance: {
|
|
21533
22244
|
impact: "AI engines crawl your about page to build an organizational authority profile. Without credential-rich content (team bios, expertise areas, certifications), agents cannot assess your organization's authority, reducing your content's trust score in AI-generated recommendations.",
|
|
@@ -21637,6 +22348,7 @@ var ExternalCitationsAudit = class extends Audit {
|
|
|
21637
22348
|
evidenceGrade: "B",
|
|
21638
22349
|
tier: "scored",
|
|
21639
22350
|
dossier: "docs/evidence/audits/answer-readiness/external-citations.md",
|
|
22351
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21640
22352
|
applicablePageTypes: ["content"],
|
|
21641
22353
|
defaultPriority: "medium",
|
|
21642
22354
|
guidance: {
|
|
@@ -21751,6 +22463,7 @@ var BrandNameAudit = class extends Audit {
|
|
|
21751
22463
|
evidenceGrade: "C",
|
|
21752
22464
|
tier: "informative",
|
|
21753
22465
|
dossier: "docs/evidence/audits/answer-readiness/brand-name.md",
|
|
22466
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21754
22467
|
defaultPriority: "medium",
|
|
21755
22468
|
guidance: {
|
|
21756
22469
|
impact: "AI engines build entity graphs by matching Organization schema names to in-content mentions. If your brand name only appears in schema but not body text, agents cannot associate your content with your entity, weakening brand recognition in AI responses.",
|
|
@@ -21845,7 +22558,7 @@ function statesZeroReviews(record3) {
|
|
|
21845
22558
|
}
|
|
21846
22559
|
return false;
|
|
21847
22560
|
}
|
|
21848
|
-
function
|
|
22561
|
+
function readableText2(page) {
|
|
21849
22562
|
const body = page.$("body").clone();
|
|
21850
22563
|
body.find("script, style, noscript, template").remove();
|
|
21851
22564
|
return body.text().replace(/\s+/g, " ").trim();
|
|
@@ -21946,6 +22659,7 @@ var ReviewSignalsAudit = class extends Audit {
|
|
|
21946
22659
|
evidenceGrade: "B",
|
|
21947
22660
|
tier: "scored",
|
|
21948
22661
|
dossier: "docs/evidence/audits/answer-readiness/review-signals.md",
|
|
22662
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
21949
22663
|
applicablePageTypes: ["homepage", "product"],
|
|
21950
22664
|
defaultPriority: "medium",
|
|
21951
22665
|
guidance: {
|
|
@@ -21993,7 +22707,7 @@ var ReviewSignalsAudit = class extends Audit {
|
|
|
21993
22707
|
).toArray().filter((el) => p.$(el).text().trim() !== "" || p.$(el).children().length > 0);
|
|
21994
22708
|
if (widget.length > 0) {
|
|
21995
22709
|
noteWeak("review widget markup", p.url);
|
|
21996
|
-
} else if (/\b\d[\d,]*\s+reviews?\b/i.test(
|
|
22710
|
+
} else if (/\b\d[\d,]*\s+reviews?\b/i.test(readableText2(p))) {
|
|
21997
22711
|
noteWeak('"N reviews" text', p.url);
|
|
21998
22712
|
}
|
|
21999
22713
|
}
|
|
@@ -22053,7 +22767,7 @@ function isNonEnglish(page) {
|
|
|
22053
22767
|
const lang = (page.$("html").attr("lang") ?? "").trim().toLowerCase();
|
|
22054
22768
|
return lang !== "" && !lang.startsWith("en");
|
|
22055
22769
|
}
|
|
22056
|
-
function
|
|
22770
|
+
function readableText3(page) {
|
|
22057
22771
|
const body = page.$("body").clone();
|
|
22058
22772
|
body.find("script, style, noscript, template").remove();
|
|
22059
22773
|
return body.text().replace(/\s+/g, " ").trim();
|
|
@@ -22105,6 +22819,7 @@ var TrustSignalsAudit = class _TrustSignalsAudit extends Audit {
|
|
|
22105
22819
|
evidenceGrade: "B",
|
|
22106
22820
|
tier: "scored",
|
|
22107
22821
|
dossier: "docs/evidence/audits/answer-readiness/trust-signals.md",
|
|
22822
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22108
22823
|
applicablePageTypes: ["homepage"],
|
|
22109
22824
|
defaultPriority: "low",
|
|
22110
22825
|
guidance: {
|
|
@@ -22132,7 +22847,7 @@ var TrustSignalsAudit = class _TrustSignalsAudit extends Audit {
|
|
|
22132
22847
|
"Non-English homepage \u2014 detector not applicable"
|
|
22133
22848
|
);
|
|
22134
22849
|
}
|
|
22135
|
-
const text3 =
|
|
22850
|
+
const text3 = readableText3(page);
|
|
22136
22851
|
const satisfied = [];
|
|
22137
22852
|
const missing = [];
|
|
22138
22853
|
let counted = 0;
|
|
@@ -22242,6 +22957,7 @@ var PublicationDateAudit = class extends Audit {
|
|
|
22242
22957
|
evidenceGrade: "B",
|
|
22243
22958
|
tier: "scored",
|
|
22244
22959
|
dossier: "docs/evidence/audits/answer-readiness/publication-date.md",
|
|
22960
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22245
22961
|
applicablePageTypes: ["content"],
|
|
22246
22962
|
defaultPriority: "medium",
|
|
22247
22963
|
guidance: {
|
|
@@ -22338,6 +23054,7 @@ var LastModifiedSchemaAudit = class extends Audit {
|
|
|
22338
23054
|
evidenceGrade: "B",
|
|
22339
23055
|
tier: "scored",
|
|
22340
23056
|
dossier: "docs/evidence/audits/answer-readiness/last-modified-schema.md",
|
|
23057
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22341
23058
|
applicablePageTypes: ["content"],
|
|
22342
23059
|
defaultPriority: "medium",
|
|
22343
23060
|
guidance: {
|
|
@@ -22424,6 +23141,7 @@ var UniqueDataAudit = class extends Audit {
|
|
|
22424
23141
|
evidenceGrade: "B",
|
|
22425
23142
|
tier: "scored",
|
|
22426
23143
|
dossier: "docs/evidence/audits/answer-readiness/unique-data.md",
|
|
23144
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22427
23145
|
defaultPriority: "medium",
|
|
22428
23146
|
guidance: {
|
|
22429
23147
|
impact: "AI generative engines prioritize content with unique, citable data points because agents can quote exact figures in generated answers. Content without specific numbers reads as opinion rather than evidence, reducing its chances of being cited.",
|
|
@@ -22521,6 +23239,8 @@ var DescriptiveUrlsAudit = class extends Audit {
|
|
|
22521
23239
|
evidenceGrade: "C",
|
|
22522
23240
|
tier: "informative",
|
|
22523
23241
|
dossier: "docs/evidence/audits/answer-readiness/descriptive-urls.md",
|
|
23242
|
+
// Gate exemption: a URL is readable whether or not the page behind it rendered text.
|
|
23243
|
+
requires: ["origin-reachable", "unblocked-fetches"],
|
|
22524
23244
|
defaultPriority: "high",
|
|
22525
23245
|
guidance: {
|
|
22526
23246
|
impact: "AI engines use URL text as a pre-fetch topic signal and display URLs in generated citations. Non-descriptive slugs with UUIDs or numeric IDs provide no topical context, reducing your content's relevance score before the page is even crawled.",
|
|
@@ -22531,6 +23251,13 @@ var DescriptiveUrlsAudit = class extends Audit {
|
|
|
22531
23251
|
}
|
|
22532
23252
|
};
|
|
22533
23253
|
audit(ctx) {
|
|
23254
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
23255
|
+
return this.notApplicable(
|
|
23256
|
+
"No page here can be attributed to this site, so its URLs were not judged.",
|
|
23257
|
+
"Page URLs use readable slugs (no UUIDs, no /post-123/, no encoded params)",
|
|
23258
|
+
unreadSiteReason(ctx.evidence)
|
|
23259
|
+
);
|
|
23260
|
+
}
|
|
22534
23261
|
const page = ctx.pages[0];
|
|
22535
23262
|
if (!page) {
|
|
22536
23263
|
return this.fail(
|
|
@@ -22776,6 +23503,7 @@ var SnippetGateCoverageAudit = class _SnippetGateCoverageAudit extends Audit {
|
|
|
22776
23503
|
evidenceGrade: "A",
|
|
22777
23504
|
tier: "scored",
|
|
22778
23505
|
dossier: "docs/evidence/audits/answer-readiness/snippet-gate-coverage.md",
|
|
23506
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22779
23507
|
defaultPriority: "high",
|
|
22780
23508
|
guidance: {
|
|
22781
23509
|
impact: "Google states the eligibility gate directly: to appear as a supporting link a page 'must be indexed and eligible to be shown in Google Search with a snippet', and names nosnippet, data-nosnippet, max-snippet and noindex as the controls that limit what AI Overviews and AI Mode can show. This makes the causal chain fully documented rather than inferred: a max-snippet value shorter than the answer sentence truncates the answer below usefulness, and data-nosnippet wrapping the answer removes it from AI surfaces entirely while leaving it visible to humans \u2014 an invisible failure that page-level SEO reports do not surface because the directive itself is technically 'valid'.",
|
|
@@ -22794,6 +23522,13 @@ var SnippetGateCoverageAudit = class _SnippetGateCoverageAudit extends Audit {
|
|
|
22794
23522
|
};
|
|
22795
23523
|
}
|
|
22796
23524
|
audit(ctx) {
|
|
23525
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
23526
|
+
return this.notApplicable(
|
|
23527
|
+
"No page here can be attributed to this site, so snippet permissions were not resolved.",
|
|
23528
|
+
EXPECTED22,
|
|
23529
|
+
unreadSiteReason(ctx.evidence)
|
|
23530
|
+
);
|
|
23531
|
+
}
|
|
22797
23532
|
const page = ctx.pages[0];
|
|
22798
23533
|
if (!page) {
|
|
22799
23534
|
return this.notApplicable(
|
|
@@ -22988,6 +23723,7 @@ var TextFragmentAddressabilityAudit = class _TextFragmentAddressabilityAudit ext
|
|
|
22988
23723
|
evidenceGrade: "A",
|
|
22989
23724
|
tier: "scored",
|
|
22990
23725
|
dossier: "docs/evidence/audits/answer-readiness/text-fragment-addressability.md",
|
|
23726
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
22991
23727
|
defaultPriority: "medium",
|
|
22992
23728
|
guidance: {
|
|
22993
23729
|
impact: "Google Search auto-generates text-fragment URLs to land users on the exact featured-snippet text, and the spec requires each of prefix/start/end/suffix to match within a single block-level element. When an answer sentence is fragmented across block boundaries, or the header opt-out is set, the fragment silently fails and the link degrades to page-top. Falsifiable and directly testable: take the citing surface\u2019s own generated URL, load it, and observe whether the browser scrolls and highlights. Two failure classes are binary and deterministic \u2014 the opt-out header, and a start string that straddles two blocks.",
|
|
@@ -23135,6 +23871,7 @@ var ChunkBoundaryReferentIntegrityAudit = class extends Audit {
|
|
|
23135
23871
|
weight: weightForGrade("B", "scored"),
|
|
23136
23872
|
defaultPriority: "high",
|
|
23137
23873
|
dossier: "docs/evidence/audits/answer-readiness/chunk-boundary-referent-integrity.md",
|
|
23874
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
23138
23875
|
guidance: {
|
|
23139
23876
|
impact: 'An answer engine retrieves a passage, not a page. A section that opens "This means you should..." and never names its subject is unusable on arrival: the model either drops it or attributes it to whatever else is in the window. The fix is per-sentence and cheap, and it is invisible to a reader of the whole page \u2014 which is why it survives editing.',
|
|
23140
23877
|
fix: 'Open each section with its subject rather than a pronoun, name the product or topic once in every section over about forty words, and replace "as described above" with the name of the thing described.',
|
|
@@ -23266,6 +24003,13 @@ function chainOf($, el) {
|
|
|
23266
24003
|
const parents = $(el).parents().toArray().filter((parent) => !["html", "body"].includes(parent.tagName)).reverse().map(describe3);
|
|
23267
24004
|
return [...parents, describe3(el)].join(" > ");
|
|
23268
24005
|
}
|
|
24006
|
+
function lastElementContaining($, text3) {
|
|
24007
|
+
const all = $("*").toArray();
|
|
24008
|
+
for (let i = all.length - 1; i >= 0; i--) {
|
|
24009
|
+
if ($(all[i]).text().includes(text3)) return all[i];
|
|
24010
|
+
}
|
|
24011
|
+
return void 0;
|
|
24012
|
+
}
|
|
23269
24013
|
function needleOf(text3) {
|
|
23270
24014
|
return normalizeText(text3).split(" ").slice(0, SPAN_WORDS).join(" ");
|
|
23271
24015
|
}
|
|
@@ -23298,8 +24042,8 @@ function keySpans(page) {
|
|
|
23298
24042
|
if (typeof value === "string") {
|
|
23299
24043
|
const needle = needleOf(value);
|
|
23300
24044
|
if (needle.split(" ").length >= 3 && bodyText.includes(needle)) {
|
|
23301
|
-
const host =
|
|
23302
|
-
push("json-ld", host
|
|
24045
|
+
const host = lastElementContaining($, value.slice(0, 40));
|
|
24046
|
+
push("json-ld", host ?? $("body")[0], value);
|
|
23303
24047
|
}
|
|
23304
24048
|
return;
|
|
23305
24049
|
}
|
|
@@ -23333,6 +24077,7 @@ var ExtractorSurvivalRecallAudit = class extends Audit {
|
|
|
23333
24077
|
weight: weightForGrade("B", "scored"),
|
|
23334
24078
|
defaultPriority: "high",
|
|
23335
24079
|
dossier: "docs/evidence/audits/answer-readiness/extractor-survival-recall.md",
|
|
24080
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
23336
24081
|
guidance: {
|
|
23337
24082
|
impact: 'An answer engine never sees the page; it sees whatever its extractor kept. A specification table inside `<aside class="related-specs">` is invisible to every pipeline that strips asides, and the answer about that product gets written without it. The loss is silent: the page looks complete to its author and to every human reviewer.',
|
|
23338
24083
|
fix: 'Put facts inside the main content container, not in an aside, a footer, or a block whose class says "related" or "promo". Where a table must sit outside the article, repeat its facts in the prose so at least one copy survives.',
|
|
@@ -23342,6 +24087,13 @@ var ExtractorSurvivalRecallAudit = class extends Audit {
|
|
|
23342
24087
|
}
|
|
23343
24088
|
};
|
|
23344
24089
|
audit(ctx) {
|
|
24090
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
24091
|
+
return this.notApplicable(
|
|
24092
|
+
"No page here can be attributed to this site, so no key span was measured.",
|
|
24093
|
+
"A page from this site whose key spans can be measured",
|
|
24094
|
+
unreadSiteReason(ctx.evidence)
|
|
24095
|
+
);
|
|
24096
|
+
}
|
|
23345
24097
|
const page = ctx.pages[0];
|
|
23346
24098
|
if (!page) {
|
|
23347
24099
|
return this.notApplicable(
|
|
@@ -23465,6 +24217,7 @@ var SectionSplitRiskProfileAudit = class extends Audit {
|
|
|
23465
24217
|
weight: weightForGrade("B", "scored"),
|
|
23466
24218
|
defaultPriority: "medium",
|
|
23467
24219
|
dossier: "docs/evidence/audits/answer-readiness/section-split-risk-profile.md",
|
|
24220
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
23468
24221
|
guidance: {
|
|
23469
24222
|
impact: "Retrieval pipelines cut pages into fixed windows. A section longer than the window becomes one chunk carrying the heading and one or more tail chunks carrying none \u2014 and a tail chunk is text with no subject, which retrieves badly and cites worse. A page with no headings at all is cut at arbitrary offsets throughout.",
|
|
23470
24223
|
fix: "Add an `h2` or `h3` roughly every 400 tokens of prose, and split a specification table that runs past the window into per-topic tables so the header row stays with its rows.",
|
|
@@ -23654,6 +24407,7 @@ var SiteWidePassageUniquenessRatioAudit = class extends Audit {
|
|
|
23654
24407
|
weight: weightForGrade("B", "scored"),
|
|
23655
24408
|
defaultPriority: "medium",
|
|
23656
24409
|
dossier: "docs/evidence/audits/answer-readiness/site-wide-passage-uniqueness-ratio.md",
|
|
24410
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
23657
24411
|
guidance: {
|
|
23658
24412
|
impact: "A search engine clusters duplicate and near-duplicate URLs and elects one canonical; the losers have their signals folded into the winner. A cluster of near-duplicate pages that each name themselves canonical therefore competes against itself, and at most one member stays citable however good the others are. Separately, a page whose sentences are mostly site-wide template produces chunks whose embeddings encode the template rather than the page, so every page built from that template lands in the same place in vector space and none is a distinctive match for any question.",
|
|
23659
24413
|
fix: 'Merge near-duplicate pages into one, or point the weaker members at the strongest with rel="canonical" so the election has an answer. For pages that stay, raise the share of text that is theirs alone: cut the repeated intro, the repeated legal paragraph and the repeated call to action, and let each page carry the sentences only it can carry.',
|
|
@@ -23887,6 +24641,7 @@ var TableMarkdownRoundTripLossAudit = class extends Audit {
|
|
|
23887
24641
|
weight: weightForGrade("B", "scored"),
|
|
23888
24642
|
defaultPriority: "medium",
|
|
23889
24643
|
dossier: "docs/evidence/audits/answer-readiness/table-markdown-round-trip-loss.md",
|
|
24644
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
23890
24645
|
guidance: {
|
|
23891
24646
|
impact: "A model does not read your table markup. Something converts it to markdown first, and GFM markdown has no merged cells, no second header row and no lists inside a cell. A header spanning two columns arrives heading one of them; the other column of numbers arrives with no header at all. The model still answers the question \u2014 with a number read from the wrong column, stated as confidently as a right one.",
|
|
23892
24647
|
fix: "Flatten spanned headers into one header row of plain `th` cells, repeating the text where a span used to cover two columns. Put the unit or currency in the header cell rather than in the caption. Take paragraphs and lists out of cells. Where a table is genuinely two tables, publish it as two.",
|
|
@@ -24154,6 +24909,7 @@ var OpenApiExistsAudit = class _OpenApiExistsAudit extends Audit {
|
|
|
24154
24909
|
evidenceGrade: "B",
|
|
24155
24910
|
tier: "informative",
|
|
24156
24911
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-exists.md",
|
|
24912
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
24157
24913
|
defaultPriority: "medium",
|
|
24158
24914
|
guidance: {
|
|
24159
24915
|
impact: "An agent that cannot find your API description cannot call it. Note that every documented consumer today (GPT Actions, Microsoft 365 Copilot API plugins) receives the document from a developer rather than fetching it from your site, so this check is informative and unscored.",
|
|
@@ -24294,6 +25050,7 @@ var OpenApiEndpointsAudit = class _OpenApiEndpointsAudit extends Audit {
|
|
|
24294
25050
|
evidenceGrade: "B",
|
|
24295
25051
|
tier: "scored",
|
|
24296
25052
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-endpoints.md",
|
|
25053
|
+
requires: ["origin-reachable"],
|
|
24297
25054
|
defaultPriority: "high",
|
|
24298
25055
|
guidance: {
|
|
24299
25056
|
impact: "An OpenAPI spec without endpoints is unusable -- AI agents see a spec file but have zero actions they can perform. Your site remains a passive document that agents cannot interact with programmatically.",
|
|
@@ -24421,6 +25178,7 @@ var OpenApiOperationIdsAudit = class _OpenApiOperationIdsAudit extends Audit {
|
|
|
24421
25178
|
evidenceGrade: "B",
|
|
24422
25179
|
tier: "scored",
|
|
24423
25180
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-operation-ids.md",
|
|
25181
|
+
requires: ["origin-reachable"],
|
|
24424
25182
|
defaultPriority: "medium",
|
|
24425
25183
|
guidance: {
|
|
24426
25184
|
impact: "AI agents use operationIds as stable function names when calling your API. Without unique operationIds, agents must infer endpoint names from URL paths, leading to ambiguous calls, naming collisions, and broken integrations.",
|
|
@@ -24600,6 +25358,7 @@ var OpenApiServersAudit = class _OpenApiServersAudit extends Audit {
|
|
|
24600
25358
|
evidenceGrade: "B",
|
|
24601
25359
|
tier: "scored",
|
|
24602
25360
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-servers.md",
|
|
25361
|
+
requires: ["origin-reachable"],
|
|
24603
25362
|
defaultPriority: "high",
|
|
24604
25363
|
guidance: {
|
|
24605
25364
|
impact: "Without a servers array, AI agents cannot determine the base URL for your API. Even if your endpoints are perfectly documented, agents cannot construct valid request URLs, rendering the entire OpenAPI spec unusable.",
|
|
@@ -24764,6 +25523,7 @@ var OpenApiSchemasAudit = class _OpenApiSchemasAudit extends Audit {
|
|
|
24764
25523
|
evidenceGrade: "B",
|
|
24765
25524
|
tier: "scored",
|
|
24766
25525
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-schemas.md",
|
|
25526
|
+
requires: ["origin-reachable"],
|
|
24767
25527
|
defaultPriority: "medium",
|
|
24768
25528
|
guidance: {
|
|
24769
25529
|
impact: "Without request/response schemas, AI agents must guess what data to send and what to expect back. This leads to malformed requests, failed API calls, and agents that cannot reliably use your endpoints.",
|
|
@@ -25168,6 +25928,7 @@ var AiCatalogExistsAudit = class _AiCatalogExistsAudit extends Audit {
|
|
|
25168
25928
|
evidenceGrade: "C",
|
|
25169
25929
|
tier: "informative",
|
|
25170
25930
|
dossier: "docs/evidence/audits/agent-interfaces/ai-catalog-exists.md",
|
|
25931
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
25171
25932
|
defaultPriority: "medium",
|
|
25172
25933
|
guidance: {
|
|
25173
25934
|
impact: "Without an AI catalog, agents must probe multiple endpoints to discover your services. This wastes time, increases error rates, and often results in agents skipping your site entirely in favor of competitors with a machine-readable capability manifest.",
|
|
@@ -25288,6 +26049,7 @@ var AiCatalogMetadataAudit = class _AiCatalogMetadataAudit extends Audit {
|
|
|
25288
26049
|
evidenceGrade: "B",
|
|
25289
26050
|
tier: "scored",
|
|
25290
26051
|
dossier: "docs/evidence/audits/agent-interfaces/ai-catalog-metadata.md",
|
|
26052
|
+
requires: ["origin-reachable"],
|
|
25291
26053
|
defaultPriority: "medium",
|
|
25292
26054
|
guidance: {
|
|
25293
26055
|
impact: "A thin catalog entry is a catalog entry nobody finds. Consumers match a user query against the entry text, so entries with no description, tags, capabilities or representative queries lose to better-described alternatives even when your service is the better answer.",
|
|
@@ -25423,6 +26185,7 @@ var AiCatalogUrlsAudit = class _AiCatalogUrlsAudit extends Audit {
|
|
|
25423
26185
|
evidenceGrade: "B",
|
|
25424
26186
|
tier: "scored",
|
|
25425
26187
|
dossier: "docs/evidence/audits/agent-interfaces/ai-catalog-urls.md",
|
|
26188
|
+
requires: ["origin-reachable"],
|
|
25426
26189
|
defaultPriority: "medium",
|
|
25427
26190
|
guidance: {
|
|
25428
26191
|
impact: "A broken entry url makes an agent fail mid-task: it read your manifest, followed the link you published, and got nothing. Entries whose url points at a nested catalog or registry cut off everything behind them as well.",
|
|
@@ -25550,6 +26313,7 @@ var AgentsJsonAudit = class extends Audit {
|
|
|
25550
26313
|
evidenceGrade: "C",
|
|
25551
26314
|
tier: "informative",
|
|
25552
26315
|
dossier: "docs/evidence/audits/agent-interfaces/agents-json.md",
|
|
26316
|
+
requires: ["origin-reachable"],
|
|
25553
26317
|
defaultPriority: "low",
|
|
25554
26318
|
guidance: {
|
|
25555
26319
|
impact: "Publishing agents.json is not known to make a site reachable to any agent: no vendor documents reading the file, and the specification has been dormant since 2025-08-21. What does matter is that a document already published at a well-known path can be read \u2014 a 200 carrying the site's HTML shell tells a conforming client the resource exists and then gives it nothing to parse, which is worse than a clean 404.",
|
|
@@ -25673,6 +26437,7 @@ var McpDiscoveryAudit = class _McpDiscoveryAudit extends Audit {
|
|
|
25673
26437
|
evidenceGrade: "C",
|
|
25674
26438
|
tier: "informative",
|
|
25675
26439
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-discovery.md",
|
|
26440
|
+
requires: ["origin-reachable"],
|
|
25676
26441
|
defaultPriority: "medium",
|
|
25677
26442
|
guidance: {
|
|
25678
26443
|
impact: "No shipping MCP client is documented as fetching `/.well-known/mcp/servers.json` or `/.well-known/ucp`, so publishing one is not known to make a site reachable to any agent. What does matter is that a document published at a well-known path can be read: a 200 carrying HTML or unparseable JSON tells a conforming client the resource exists and then gives it nothing to parse.",
|
|
@@ -25992,6 +26757,7 @@ var McpEndpointAudit = class _McpEndpointAudit extends Audit {
|
|
|
25992
26757
|
evidenceGrade: "C",
|
|
25993
26758
|
tier: "informative",
|
|
25994
26759
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-endpoint.md",
|
|
26760
|
+
requires: ["origin-reachable"],
|
|
25995
26761
|
defaultPriority: "high",
|
|
25996
26762
|
guidance: {
|
|
25997
26763
|
impact: "If your MCP endpoint does not answer an initialize handshake, AI assistants cannot connect at all. Capabilities and tool annotations come off the same connection: without them an agent cannot tell whether your server offers tools, or which of them are destructive enough to need user confirmation.",
|
|
@@ -26289,6 +27055,7 @@ var SearchEndpointAudit = class _SearchEndpointAudit extends Audit {
|
|
|
26289
27055
|
evidenceGrade: "C",
|
|
26290
27056
|
tier: "informative",
|
|
26291
27057
|
dossier: "docs/evidence/audits/agent-interfaces/search-endpoint.md",
|
|
27058
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
26292
27059
|
defaultPriority: "low",
|
|
26293
27060
|
guidance: {
|
|
26294
27061
|
impact: 'Without a declared search endpoint, an agent asked to "find pricing info on Example.com" has to crawl the site to answer. Note that no vendor documents an agent that reads SearchAction today \u2014 Google retired its only documented consumer in 2024 \u2014 so this check is informative and unscored.',
|
|
@@ -26459,6 +27226,7 @@ var WebmcpRegisteredToolsAudit = class extends Audit {
|
|
|
26459
27226
|
// detector that cannot distinguish "no tools" from "cannot see the tools".
|
|
26460
27227
|
tier: "experimental",
|
|
26461
27228
|
dossier: "docs/evidence/audits/agent-interfaces/webmcp-registered-tools.md",
|
|
27229
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
26462
27230
|
// Was `high` on an admittedly non-standard convention, so it outranked
|
|
26463
27231
|
// genuinely actionable items in the recommendation list.
|
|
26464
27232
|
defaultPriority: "low",
|
|
@@ -26574,6 +27342,7 @@ var WebmcpDeclarativeFormsAudit = class extends Audit {
|
|
|
26574
27342
|
evidenceGrade: "B",
|
|
26575
27343
|
tier: "scored",
|
|
26576
27344
|
dossier: "docs/evidence/audits/agent-interfaces/webmcp-declarative-forms.md",
|
|
27345
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
26577
27346
|
// Softened from 'high': the feature is Baseline "limited" (Chrome 149 /
|
|
26578
27347
|
// Edge 150 origin trials, Brave Leo experimental) and Apple's WebKit
|
|
26579
27348
|
// standards position is "oppose", so this is worth doing, not urgent.
|
|
@@ -26701,6 +27470,7 @@ var OpenApiDescriptionQualityAudit = class _OpenApiDescriptionQualityAudit exten
|
|
|
26701
27470
|
evidenceGrade: "A",
|
|
26702
27471
|
tier: "scored",
|
|
26703
27472
|
dossier: "docs/evidence/audits/agent-interfaces/openapi-description-quality.md",
|
|
27473
|
+
requires: ["origin-reachable"],
|
|
26704
27474
|
defaultPriority: "high",
|
|
26705
27475
|
guidance: {
|
|
26706
27476
|
impact: "LLM tool-calling treats your OpenAPI descriptions as the function-calling prompt. Missing or terse descriptions force the model to guess what each endpoint does and what each parameter accepts, producing wrong tool selection, malformed arguments, and failed API calls that erode user trust in agent-driven workflows on your site.",
|
|
@@ -26882,6 +27652,7 @@ var CorsApiRoutesAudit = class _CorsApiRoutesAudit extends Audit {
|
|
|
26882
27652
|
evidenceGrade: "C",
|
|
26883
27653
|
tier: "informative",
|
|
26884
27654
|
dossier: "docs/evidence/audits/agent-interfaces/cors-api-routes.md",
|
|
27655
|
+
requires: ["origin-reachable"],
|
|
26885
27656
|
// The affected consumer class is small; nothing here should outrank an
|
|
26886
27657
|
// item that changes what a crawler or an MCP client can do.
|
|
26887
27658
|
defaultPriority: "low",
|
|
@@ -27041,6 +27812,7 @@ var McpModernEraReachabilityAudit = class extends Audit {
|
|
|
27041
27812
|
evidenceGrade: "A",
|
|
27042
27813
|
tier: "scored",
|
|
27043
27814
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-modern-era-reachability.md",
|
|
27815
|
+
requires: ["origin-reachable"],
|
|
27044
27816
|
defaultPriority: "high",
|
|
27045
27817
|
guidance: {
|
|
27046
27818
|
impact: "Revision 2026-07-28 abolished the `initialize` handshake and protocol-level sessions: version, client identity and capabilities now travel as per-request `_meta`, and `server/discover` is a MUST-implement RPC. The spec's own compatibility matrix states verbatim that a Modern client against a Legacy server FAILS, with no fall-forward path. Therefore: if a single POST of `server/discover` carrying `_meta` + `MCP-Protocol-Version: 2026-07-28` does not yield either a DiscoverResult or a recognized modern JSON-RPC error, then every client that has moved to the current revision cannot invoke a single tool on this server \u2014 the failure is total, not degraded. Conversely a 404/-32601 on `server/discover` from a server that otherwise answers modern requests is a direct MUST violation that breaks pre-consent capability presentation.",
|
|
@@ -27291,6 +28063,7 @@ var McpOauthDiscoveryChainAudit = class extends Audit {
|
|
|
27291
28063
|
evidenceGrade: "A",
|
|
27292
28064
|
tier: "scored",
|
|
27293
28065
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-oauth-discovery-chain.md",
|
|
28066
|
+
requires: ["origin-reachable"],
|
|
27294
28067
|
defaultPriority: "high",
|
|
27295
28068
|
guidance: {
|
|
27296
28069
|
impact: "The spec makes RFC 9728 mandatory for MCP servers and makes clients apply two hard identity checks: RFC 9728 \xA73.3 requires the PRM's `resource` value to be string-identical to the resource identifier used to construct the request URL, and the MCP AS-discovery rules require the fetched AS metadata's `issuer` to be string-identical to the issuer used to construct the well-known URL \u2014 on either mismatch the client MUST NOT use the metadata. MCP additionally strengthens RFC 9728 by requiring `authorization_servers` to carry at least one entry (it is merely OPTIONAL in the RFC). Each of these is a silent, total blocker: the discovery chain either resolves end to end or the agent never reaches an authorization prompt, so a single character of drift between the deployed endpoint URL and the `resource` claim makes the server unusable to every conforming client while the server's own logs show nothing but 401s.",
|
|
@@ -27547,6 +28320,7 @@ var McpToolContractValidityAudit = class extends Audit {
|
|
|
27547
28320
|
evidenceGrade: "A",
|
|
27548
28321
|
tier: "scored",
|
|
27549
28322
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-tool-contract-validity.md",
|
|
28323
|
+
requires: ["origin-reachable"],
|
|
27550
28324
|
defaultPriority: "critical",
|
|
27551
28325
|
guidance: {
|
|
27552
28326
|
impact: "The spec gives clients an explicit deletion instruction: 'Clients using the Streamable HTTP transport MUST reject tool definitions where any x-mcp-header value violates these constraints. Rejection means the client MUST exclude the invalid tool from the result of tools/list.' This makes malformed tool metadata a silent-invisibility bug rather than an error: the server returns the tool, logs a successful tools/list, and the model never sees it. The constraint set is fully machine-checkable with no network calls beyond the one list fetch \u2014 token syntax, no CR/LF, case-insensitive uniqueness, primitive types only with `number` explicitly excluded, and static reachability through a chain consisting solely of `properties` keys. Alongside it, `inputSchema` MUST be a valid JSON Schema object and not null; a null or scalar inputSchema breaks argument construction in every SDK.",
|
|
@@ -27803,6 +28577,7 @@ var McpToolsListDeterminismAudit = class extends Audit {
|
|
|
27803
28577
|
evidenceGrade: "A",
|
|
27804
28578
|
tier: "scored",
|
|
27805
28579
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-tools-list-determinism.md",
|
|
28580
|
+
requires: ["origin-reachable"],
|
|
27806
28581
|
defaultPriority: "medium",
|
|
27807
28582
|
guidance: {
|
|
27808
28583
|
impact: "The spec states its own causal rationale verbatim: deterministic ordering 'enables clients to reliably cache the tool list and improves LLM prompt cache hit rates when tools are included in model context.' Tool definitions sit near the front of the model's prompt; if their serialized bytes change between turns, the provider-side prefix cache misses and the full tool block is re-billed at uncached rates on every single turn. Separately, servers MUST include caching hints on complete results, and when ttlMs is absent clients SHOULD assume 0 \u2014 immediately stale \u2014 so an omitted hint converts one cheap cached read into a network round-trip on every access. Both defects are invisible in functional testing and both are measurable with three identical requests.",
|
|
@@ -27987,6 +28762,7 @@ var McpVersionDowngradeAudit = class extends Audit {
|
|
|
27987
28762
|
evidenceGrade: "A",
|
|
27988
28763
|
tier: "scored",
|
|
27989
28764
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-version-downgrade.md",
|
|
28765
|
+
requires: ["origin-reachable"],
|
|
27990
28766
|
defaultPriority: "medium",
|
|
27991
28767
|
guidance: {
|
|
27992
28768
|
impact: "With the handshake removed, the ONLY mechanism by which a client discovers a mutually supported version mid-flight is the `UnsupportedProtocolVersionError`: the spec requires code -32022 with `data.supported[]` listing the server's versions, and instructs clients to select from that list and retry. A server that instead returns a 500, a generic -32600/-32602, or a 400 with no `supported` array gives the client nothing to downgrade to \u2014 so a client whose preferred version is one revision ahead of the server's fails permanently even though a mutually supported version exists on both sides. Separately, the spec requires the header and the `_meta` value to agree, with a 400 + -32020 HeaderMismatch on divergence; a server that silently ignores the mismatch is trusting whichever source of truth its proxy layer did not, which is the exact split-brain the header-validation rules exist to prevent.",
|
|
@@ -28150,6 +28926,7 @@ var McpOriginValidationCorsAudit = class extends Audit {
|
|
|
28150
28926
|
weight: weightForGrade("B", "scored"),
|
|
28151
28927
|
defaultPriority: "high",
|
|
28152
28928
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-origin-validation-cors.md",
|
|
28929
|
+
requires: ["origin-reachable"],
|
|
28153
28930
|
guidance: {
|
|
28154
28931
|
impact: "The transport spec is unambiguous: servers MUST validate the Origin header on all incoming connections, and answer 403 when it is present and invalid, because a server that does not is reachable from any web page the user has open. The provable defect is the CORS pairing: an endpoint that reflects the requesting Origin into `Access-Control-Allow-Origin` and returns `Access-Control-Allow-Credentials: true` has authorized any page to enumerate its tool surface and invoke tools with the user\u2019s session.",
|
|
28155
28932
|
fix: "Validate `Origin` on every request and answer 403 when it is present and not one you allow. Never reflect an arbitrary Origin while allowing credentials: return a fixed allow-list, or drop `Access-Control-Allow-Credentials`. `Access-Control-Allow-Origin: *` is only safe on an endpoint that accepts no credentials at all.",
|
|
@@ -28271,61 +29048,6 @@ var McpOriginValidationCorsAudit = class extends Audit {
|
|
|
28271
29048
|
}
|
|
28272
29049
|
};
|
|
28273
29050
|
|
|
28274
|
-
// src/gatherers/domains.ts
|
|
28275
|
-
var MULTI_SUFFIX = /* @__PURE__ */ new Set([
|
|
28276
|
-
"co.uk",
|
|
28277
|
-
"org.uk",
|
|
28278
|
-
"ac.uk",
|
|
28279
|
-
"gov.uk",
|
|
28280
|
-
"me.uk",
|
|
28281
|
-
"net.uk",
|
|
28282
|
-
"com.au",
|
|
28283
|
-
"net.au",
|
|
28284
|
-
"org.au",
|
|
28285
|
-
"edu.au",
|
|
28286
|
-
"gov.au",
|
|
28287
|
-
"co.nz",
|
|
28288
|
-
"co.jp",
|
|
28289
|
-
"or.jp",
|
|
28290
|
-
"ne.jp",
|
|
28291
|
-
"co.za",
|
|
28292
|
-
"co.kr",
|
|
28293
|
-
"co.il",
|
|
28294
|
-
"co.id",
|
|
28295
|
-
"co.th",
|
|
28296
|
-
"com.br",
|
|
28297
|
-
"com.mx",
|
|
28298
|
-
"com.ar",
|
|
28299
|
-
"com.co",
|
|
28300
|
-
"com.pe",
|
|
28301
|
-
"co.in",
|
|
28302
|
-
"com.sg",
|
|
28303
|
-
"com.tr",
|
|
28304
|
-
"com.cn",
|
|
28305
|
-
"com.hk",
|
|
28306
|
-
"com.tw",
|
|
28307
|
-
"com.my",
|
|
28308
|
-
"com.ph",
|
|
28309
|
-
"com.ua",
|
|
28310
|
-
"com.pl",
|
|
28311
|
-
"com.es",
|
|
28312
|
-
"com.pt",
|
|
28313
|
-
"com.gr"
|
|
28314
|
-
]);
|
|
28315
|
-
function registrableDomain(host) {
|
|
28316
|
-
const parts = host.toLowerCase().replace(/\.$/, "").split(".").filter(Boolean);
|
|
28317
|
-
if (parts.length <= 2) return parts.join(".");
|
|
28318
|
-
const lastTwo = parts.slice(-2).join(".");
|
|
28319
|
-
return MULTI_SUFFIX.has(lastTwo) ? parts.slice(-3).join(".") : lastTwo;
|
|
28320
|
-
}
|
|
28321
|
-
function registrableOf(url) {
|
|
28322
|
-
try {
|
|
28323
|
-
return registrableDomain(new URL(url).hostname);
|
|
28324
|
-
} catch {
|
|
28325
|
-
return "";
|
|
28326
|
-
}
|
|
28327
|
-
}
|
|
28328
|
-
|
|
28329
29051
|
// src/audits/agent-interfaces/mcp-registry-listing-ownership.ts
|
|
28330
29052
|
var REGISTRY = "https://registry.modelcontextprotocol.io/v0.1/servers";
|
|
28331
29053
|
var PROOF_PATH = "/.well-known/mcp-registry-auth";
|
|
@@ -28381,6 +29103,7 @@ var McpRegistryListingOwnershipAudit = class extends Audit {
|
|
|
28381
29103
|
weight: weightForGrade("B", "scored"),
|
|
28382
29104
|
defaultPriority: "medium",
|
|
28383
29105
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-registry-listing-ownership.md",
|
|
29106
|
+
requires: ["origin-reachable"],
|
|
28384
29107
|
guidance: {
|
|
28385
29108
|
impact: 'The registry is the index a client resolves "the MCP server for this domain" against. A domain with no first-party entry is absent from it, so the only path to the server is a URL somebody pastes by hand. A listing under an aggregator\u2019s namespace is worse than absent in one way: the brand cannot update or revoke it, and agents routed through it reach a proxy rather than the origin. The reverse-DNS namespace that fixes this is granted on proof of domain control, and that proof has to keep being served.',
|
|
28386
29109
|
fix: "Publish the server under your own reverse-DNS namespace (`com.example/...`), serve the proof at `/.well-known/mcp-registry-auth` in the exact `v=MCPv1; k=ed25519; p=<base64>` form and keep serving it after DNS migrations, keep the listing\u2019s version in step with what the server reports, and offer a `streamable-http` remote rather than only the deprecated `sse`.",
|
|
@@ -28597,6 +29320,7 @@ var McpToolDescriptionCoverageAudit = class extends Audit {
|
|
|
28597
29320
|
weight: weightForGrade("B", "scored"),
|
|
28598
29321
|
defaultPriority: "medium",
|
|
28599
29322
|
dossier: "docs/evidence/audits/agent-interfaces/mcp-tool-description-coverage.md",
|
|
29323
|
+
requires: ["origin-reachable"],
|
|
28600
29324
|
guidance: {
|
|
28601
29325
|
impact: "A tool description and its parameter descriptions are the only prose a model ever sees about a tool \u2014 they are the whole basis on which it decides whether to call it and what to pass. A required parameter with no description, no enum and no pattern gives the model nothing to derive a legal value from, so it guesses. Guessed values come back as validation errors, and the agent spends retry turns per call until it gives up on the tool.",
|
|
28602
29326
|
fix: "Describe every tool and every parameter, in prose long enough to say what a legal value looks like. Constrain string parameters with `enum`, `format` or `pattern` where the legal set is finite. Declare an `outputSchema` so a client can parse the result rather than re-reading it, give each tool a `title` for the consent prompt, and return top-level `instructions` telling a model how the tools fit together.",
|
|
@@ -28834,6 +29558,7 @@ var OfferSchemaAudit = class extends Audit {
|
|
|
28834
29558
|
evidenceGrade: "A",
|
|
28835
29559
|
tier: "scored",
|
|
28836
29560
|
dossier: "docs/evidence/audits/agentic-commerce/offer-schema.md",
|
|
29561
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
28837
29562
|
applicablePageTypes: ["product"],
|
|
28838
29563
|
defaultPriority: "medium",
|
|
28839
29564
|
guidance: {
|
|
@@ -28952,6 +29677,7 @@ var ProductIdentifiersAudit = class extends Audit {
|
|
|
28952
29677
|
evidenceGrade: "A",
|
|
28953
29678
|
tier: "scored",
|
|
28954
29679
|
dossier: "docs/evidence/audits/agentic-commerce/product-identifiers.md",
|
|
29680
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
28955
29681
|
applicablePageTypes: ["product"],
|
|
28956
29682
|
defaultPriority: "high",
|
|
28957
29683
|
guidance: {
|
|
@@ -29069,6 +29795,7 @@ var ProductTransactionCertaintyAudit = class extends Audit {
|
|
|
29069
29795
|
evidenceGrade: "A",
|
|
29070
29796
|
tier: "scored",
|
|
29071
29797
|
dossier: "docs/evidence/audits/agentic-commerce/product-transaction-certainty.md",
|
|
29798
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
29072
29799
|
applicablePageTypes: ["product"],
|
|
29073
29800
|
defaultPriority: "high",
|
|
29074
29801
|
guidance: {
|
|
@@ -29401,6 +30128,7 @@ var BuyableVariantResolutionAudit = class extends Audit {
|
|
|
29401
30128
|
weight: weightForGrade("B", "scored"),
|
|
29402
30129
|
defaultPriority: "high",
|
|
29403
30130
|
dossier: "docs/evidence/audits/agentic-commerce/buyable-variant-resolution.md",
|
|
30131
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
29404
30132
|
applicablePageTypes: ["product"],
|
|
29405
30133
|
guidance: {
|
|
29406
30134
|
impact: "The agentic-commerce feed models a catalogue variant-first: every sellable thing is a variant with its own id, price and availability. A page that shows five sizes and three colours but publishes one Offer \u2014 or an AggregateOffer with only lowPrice and highPrice \u2014 gives an agent no purchasable unit to name and no single price to quote. The row is dropped at feed validation, or the checkout session comes back with `invalid` on the line item.",
|
|
@@ -29614,6 +30342,7 @@ var CartHandoffReachabilityAudit = class extends Audit {
|
|
|
29614
30342
|
weight: weightForGrade("B", "scored"),
|
|
29615
30343
|
defaultPriority: "high",
|
|
29616
30344
|
dossier: "docs/evidence/audits/agentic-commerce/cart-handoff-reachability.md",
|
|
30345
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
29617
30346
|
guidance: {
|
|
29618
30347
|
impact: "Every upstream signal can be perfect and the purchase still dies at the last click. If the cart 302s to a login form because guest checkout is off, or Turnstile is mounted on the checkout document alone, the agent walks the buyer to a wall it cannot pass. ACP reserves a `requires_sign_in` message code for exactly this case, which is a description of the failure, not a fix for it.",
|
|
29619
30348
|
fix: "Allow guest checkout, or at least let an unauthenticated buyer reach the cart and see the totals. Keep bot challenges off the cart and checkout documents \u2014 challenge the payment submission instead, where a human is present. Allow ChatGPT-User in robots.txt and at the edge on cart paths: blocking GPTBot does not block it, and the two are separately tokened.",
|
|
@@ -29819,6 +30548,7 @@ var OfferTruthConsistencyAudit = class extends Audit {
|
|
|
29819
30548
|
weight: weightForGrade("B", "scored"),
|
|
29820
30549
|
defaultPriority: "high",
|
|
29821
30550
|
dossier: "docs/evidence/audits/agentic-commerce/offer-truth-consistency.md",
|
|
30551
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
29822
30552
|
applicablePageTypes: ["product"],
|
|
29823
30553
|
guidance: {
|
|
29824
30554
|
impact: "An agent quotes from the structured data; the seller recomputes the real amount at checkout. When the two disagree the buyer has already committed, and the session comes back with `invalid` or `out_of_stock` \u2014 the most expensive moment at which a purchase can fail. Google says the same thing from the other side: structured data must be a true representation of the page content. Markup that is present and lying passes every syntax validator on the market.",
|
|
@@ -30168,6 +30898,7 @@ var AcpPolicyLinkSurfaceAudit = class _AcpPolicyLinkSurfaceAudit extends Audit {
|
|
|
30168
30898
|
evidenceGrade: "A",
|
|
30169
30899
|
tier: "scored",
|
|
30170
30900
|
dossier: "docs/evidence/audits/agentic-commerce/acp-policy-link-surface.md",
|
|
30901
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
30171
30902
|
defaultPriority: "high",
|
|
30172
30903
|
guidance: {
|
|
30173
30904
|
impact: "Falsifiable claim: ACP spec 2026-04-17 makes `links` one of the 9 REQUIRED fields on every CheckoutSession response, with type enum {terms_of_use, privacy_policy, return_policy, shipping_policy, contact_us, about_us, faq, support}. Independently, the OpenAI product feed spec makes `seller_privacy_policy` and `seller_tos` HARD-REQUIRED whenever `is_eligible_checkout=true`. Therefore a merchant that cannot produce a resolvable HTTPS URL for terms_of_use and privacy_policy CANNOT set is_eligible_checkout=true and its catalogue is excluded from Instant Checkout no matter how good the feed is. Disproof condition: if a merchant with no reachable ToS URL is observed transacting via ACP Instant Checkout, the check is wrong.",
|
|
@@ -30418,6 +31149,7 @@ var LandedCostAndReturnsAudit = class _LandedCostAndReturnsAudit extends Audit {
|
|
|
30418
31149
|
evidenceGrade: "A",
|
|
30419
31150
|
tier: "scored",
|
|
30420
31151
|
dossier: "docs/evidence/audits/agentic-commerce/landed-cost-and-returns.md",
|
|
31152
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
30421
31153
|
applicablePageTypes: ["product"],
|
|
30422
31154
|
defaultPriority: "high",
|
|
30423
31155
|
guidance: {
|
|
@@ -30546,6 +31278,7 @@ var AgentUaCommerceParityAudit = class extends Audit {
|
|
|
30546
31278
|
evidenceGrade: "A",
|
|
30547
31279
|
tier: "scored",
|
|
30548
31280
|
dossier: "docs/evidence/audits/agentic-commerce/agent-ua-commerce-parity.md",
|
|
31281
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
30549
31282
|
defaultPriority: "critical",
|
|
30550
31283
|
guidance: {
|
|
30551
31284
|
impact: "OpenAI operates four separately-tokened agents with separately published IP ranges: OAI-SearchBot (search indexing), ChatGPT-User (user-initiated fetches \u2014 the shopper's agent), GPTBot (training) and OAI-AdsBot (ad landing-page validation). Falsifiable claim: if a product page returns 403, 429, 503 or a challenge interstitial to ChatGPT-User or OAI-SearchBot while returning 200 to a browser, ChatGPT cannot read live price and availability nor follow the buy link, so the product cannot be surfaced or transacted no matter how good the feed is. That block lives at the WAF or CDN edge, which is why an audit that only parses robots.txt is structurally blind to it. Disproof condition: a site 403ing ChatGPT-User on its product pages that still shows live, accurate prices in ChatGPT.",
|
|
@@ -30689,6 +31422,7 @@ var ContactFormAudit = class _ContactFormAudit extends Audit {
|
|
|
30689
31422
|
evidenceGrade: "C",
|
|
30690
31423
|
tier: "informative",
|
|
30691
31424
|
dossier: "docs/evidence/audits/operability-safety/contact-form.md",
|
|
31425
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
30692
31426
|
defaultPriority: "high",
|
|
30693
31427
|
guidance: {
|
|
30694
31428
|
impact: 'When users ask AI agents to "contact this company for a quote" or "send a message to their support team," the agent needs a machine-submittable form or API endpoint. Without one, the agent cannot complete the request and users turn to competitors.',
|
|
@@ -30797,6 +31531,10 @@ var NoBlockingCaptchaAudit = class _NoBlockingCaptchaAudit extends Audit {
|
|
|
30797
31531
|
evidenceGrade: "A",
|
|
30798
31532
|
tier: "scored",
|
|
30799
31533
|
dossier: "docs/evidence/audits/operability-safety/no-blocking-captcha.md",
|
|
31534
|
+
// Gate exemption: a captcha wall is what this audit reports, and a wall denies
|
|
31535
|
+
// `origin-reachable` — gating on it made the finding unreachable for the 403 that
|
|
31536
|
+
// produced it. The wall branch reads `wafProtection`, not any response body.
|
|
31537
|
+
requires: [],
|
|
30800
31538
|
defaultPriority: "high",
|
|
30801
31539
|
guidance: {
|
|
30802
31540
|
impact: 'Blocking CAPTCHAs completely prevent AI agents from submitting forms on behalf of users. When a user asks an agent to "fill out the contact form," the CAPTCHA blocks the action entirely, forcing the user to do it manually or go to a competitor.',
|
|
@@ -30819,6 +31557,30 @@ var NoBlockingCaptchaAudit = class _NoBlockingCaptchaAudit extends Audit {
|
|
|
30819
31557
|
}
|
|
30820
31558
|
};
|
|
30821
31559
|
audit(ctx) {
|
|
31560
|
+
const waf = ctx.wafProtection;
|
|
31561
|
+
if (waf?.isBlocked && !waf.isRateLimit) {
|
|
31562
|
+
return this.fail(
|
|
31563
|
+
`The site answered the scanner with a bot wall (${waf.name}). An AI agent acting for a user meets the same wall.`,
|
|
31564
|
+
"No bot wall or blocking CAPTCHA between an agent and the page",
|
|
31565
|
+
`${waf.name}: ${waf.reason}`,
|
|
31566
|
+
{ priority: "high", description: _NoBlockingCaptchaAudit.meta.description },
|
|
31567
|
+
ctx.baseUrl
|
|
31568
|
+
);
|
|
31569
|
+
}
|
|
31570
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
31571
|
+
return this.notApplicable(
|
|
31572
|
+
"No page here can be attributed to this site, so no form was inspected for a CAPTCHA.",
|
|
31573
|
+
"No recaptcha, hcaptcha, or turnstile script includes detected",
|
|
31574
|
+
unreadSiteReason(ctx.evidence)
|
|
31575
|
+
);
|
|
31576
|
+
}
|
|
31577
|
+
if (ctx.pages.length === 0) {
|
|
31578
|
+
return this.notApplicable(
|
|
31579
|
+
"No page was fetched, so no form could be inspected for a blocking CAPTCHA.",
|
|
31580
|
+
"No recaptcha, hcaptcha, or turnstile script includes detected",
|
|
31581
|
+
"No page fetched"
|
|
31582
|
+
);
|
|
31583
|
+
}
|
|
30822
31584
|
const detectedCaptchas = [];
|
|
30823
31585
|
for (const page of ctx.pages) {
|
|
30824
31586
|
const html = page.fetchResult.body.toLowerCase();
|
|
@@ -30829,6 +31591,13 @@ var NoBlockingCaptchaAudit = class _NoBlockingCaptchaAudit extends Audit {
|
|
|
30829
31591
|
}
|
|
30830
31592
|
}
|
|
30831
31593
|
if (detectedCaptchas.length === 0) {
|
|
31594
|
+
if (!scanReadPageText(ctx.evidence)) {
|
|
31595
|
+
return this.notApplicable(
|
|
31596
|
+
"The scanned page served no readable text, so no form was inspected for a CAPTCHA.",
|
|
31597
|
+
"No recaptcha, hcaptcha, or turnstile script includes detected",
|
|
31598
|
+
unreadPageTextReason(ctx.evidence)
|
|
31599
|
+
);
|
|
31600
|
+
}
|
|
30832
31601
|
return this.pass(
|
|
30833
31602
|
"No blocking CAPTCHA scripts detected on scanned pages.",
|
|
30834
31603
|
"No recaptcha, hcaptcha, or turnstile script includes detected",
|
|
@@ -30875,6 +31644,7 @@ var FormsNoJsAudit = class _FormsNoJsAudit extends Audit {
|
|
|
30875
31644
|
evidenceGrade: "C",
|
|
30876
31645
|
tier: "informative",
|
|
30877
31646
|
dossier: "docs/evidence/audits/operability-safety/forms-no-js.md",
|
|
31647
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
30878
31648
|
defaultPriority: "medium",
|
|
30879
31649
|
guidance: {
|
|
30880
31650
|
impact: "Most AI agents do not execute JavaScript. If your forms rely on JS for submission (e.g., React/Vue event handlers with no HTML action), agents cannot submit them at all. This blocks lead capture, contact requests, and any form-based interaction.",
|
|
@@ -31054,6 +31824,7 @@ var FormActionabilityAudit = class extends Audit {
|
|
|
31054
31824
|
evidenceGrade: "A",
|
|
31055
31825
|
tier: "scored",
|
|
31056
31826
|
dossier: "docs/evidence/audits/operability-safety/form-actionability.md",
|
|
31827
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
31057
31828
|
defaultPriority: "high",
|
|
31058
31829
|
guidance: {
|
|
31059
31830
|
impact: "AI agents do not render your page visually. Unlabeled fields, div-based fake inputs, and missing autocomplete attributes mean agents cannot tell which field is the email address or the name, so submissions fail silently or land in the wrong fields \u2014 lost leads, broken signups, and abandoned checkouts.",
|
|
@@ -31207,6 +31978,7 @@ var AriaLandmarksAudit = class extends Audit {
|
|
|
31207
31978
|
evidenceGrade: "A",
|
|
31208
31979
|
tier: "scored",
|
|
31209
31980
|
dossier: "docs/evidence/audits/operability-safety/aria-landmarks.md",
|
|
31981
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
31210
31982
|
defaultPriority: "high",
|
|
31211
31983
|
guidance: {
|
|
31212
31984
|
impact: "Claude computer use and browser agents rely on ARIA landmarks to identify page regions (navigation, main content, footer). Missing landmarks force agents to guess page structure from raw HTML, leading to misclicked elements and incorrect content extraction.",
|
|
@@ -31346,7 +32118,19 @@ function defineA11yAudit(spec) {
|
|
|
31346
32118
|
};
|
|
31347
32119
|
}
|
|
31348
32120
|
var base = {
|
|
31349
|
-
category: "operability-safety"
|
|
32121
|
+
category: "operability-safety",
|
|
32122
|
+
/**
|
|
32123
|
+
* Every audit built on this base reads the sampled pages through
|
|
32124
|
+
* `A11yBackedAudit`, so they all carry the same requirement set. Declared
|
|
32125
|
+
* once here; `scripts/check-requires.mjs` resolves it for each audit that
|
|
32126
|
+
* spreads `base`.
|
|
32127
|
+
*/
|
|
32128
|
+
requires: [
|
|
32129
|
+
"origin-reachable",
|
|
32130
|
+
"unblocked-fetches",
|
|
32131
|
+
"rendered-body",
|
|
32132
|
+
"sample-adequate"
|
|
32133
|
+
]
|
|
31350
32134
|
};
|
|
31351
32135
|
function graded(grade, slug) {
|
|
31352
32136
|
const tier = grade === "A" || grade === "B" ? "scored" : "informative";
|
|
@@ -31456,6 +32240,7 @@ var FormErrorMessagesAudit = class _FormErrorMessagesAudit extends Audit {
|
|
|
31456
32240
|
evidenceGrade: "A",
|
|
31457
32241
|
tier: "scored",
|
|
31458
32242
|
dossier: "docs/evidence/audits/operability-safety/form-error-messages.md",
|
|
32243
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
31459
32244
|
defaultPriority: "medium",
|
|
31460
32245
|
guidance: {
|
|
31461
32246
|
impact: "A field with no aria-errormessage or aria-describedby reference has no message attached to it in the accessibility tree, so an agent that submits a form and gets it back rejected cannot tell which field was wrong or why. It retries the same values or abandons the form.",
|
|
@@ -31861,6 +32646,7 @@ var SecurityHeaderHygieneAudit = class extends Audit {
|
|
|
31861
32646
|
evidenceGrade: "C",
|
|
31862
32647
|
tier: "informative",
|
|
31863
32648
|
dossier: "docs/evidence/audits/operability-safety/security-header-hygiene.md",
|
|
32649
|
+
requires: ["origin-reachable"],
|
|
31864
32650
|
defaultPriority: "low",
|
|
31865
32651
|
guidance: {
|
|
31866
32652
|
impact: "Vulnerability-disclosure hygiene, reported for completeness. A conformant security.txt tells a security researcher who to contact; it is read by researchers and disclosure scanners, not by AI agents. Publishing one changes nothing about how an agent retrieves, parses or cites the site, which is why nothing here moves your score. If you do publish one, an expired or contactless file is worse than none: it advertises a disclosure route that no longer works.",
|
|
@@ -32119,6 +32905,7 @@ var FormAutofillTokenCoverageAudit = class _FormAutofillTokenCoverageAudit exten
|
|
|
32119
32905
|
evidenceGrade: "A",
|
|
32120
32906
|
tier: "scored",
|
|
32121
32907
|
dossier: "docs/evidence/audits/operability-safety/form-autofill-token-coverage.md",
|
|
32908
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
32122
32909
|
defaultPriority: "high",
|
|
32123
32910
|
guidance: {
|
|
32124
32911
|
impact: 'Falsifiable claim: an agent filling a checkout must map each field to a value from user profile data. When the field declares autocomplete="postal-code", that mapping is a table lookup against a ratified vocabulary; when it declares name="field_7" with a visual-only label, the mapping is an inference that fails on ambiguous cases (address-line2 vs address-level2, cc-exp vs bday, tel-national vs tel). WebSuite measures the consequence directly: complex form filling succeeds 12.5% and 0% for the two agents tested, against 85%/76% for simple operational clicks. Test: add correct autocomplete tokens to a failing form and re-run the same fill task.',
|
|
@@ -32290,6 +33077,7 @@ var NativeControlSubstitutionAudit = class _NativeControlSubstitutionAudit exten
|
|
|
32290
33077
|
evidenceGrade: "A",
|
|
32291
33078
|
tier: "scored",
|
|
32292
33079
|
dossier: "docs/evidence/audits/operability-safety/native-control-substitution.md",
|
|
33080
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
32293
33081
|
defaultPriority: "high",
|
|
32294
33082
|
guidance: {
|
|
32295
33083
|
impact: `Falsifiable claim: native <select>, <input type="date">, and <input type="file"> are single-call primitives in every mainstream agent toolkit (selectOption, fill, setInputFiles) and are keyboard-operable, so they succeed in one action with no actionability risk. A custom equivalent requires open \u2192 wait for popup \u2192 scroll the option list into view \u2192 locate the option \u2192 click, where each step is independently subject to Playwright's visible/stable/receives-events gates, and Anthropic documents dropdowns specifically as 'tricky for Claude to manipulate using mouse movements'. Test: instrument the same form with native vs custom controls and count tool calls and retries to reach an identical value.`,
|
|
@@ -32628,6 +33416,7 @@ var InvisibleInstructionScanAudit = class _InvisibleInstructionScanAudit extends
|
|
|
32628
33416
|
evidenceGrade: "A",
|
|
32629
33417
|
tier: "scored",
|
|
32630
33418
|
dossier: "docs/evidence/audits/operability-safety/invisible-instruction-scan.md",
|
|
33419
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
32631
33420
|
defaultPriority: "critical",
|
|
32632
33421
|
guidance: {
|
|
32633
33422
|
impact: "If a page carries text nodes that a sighted human cannot perceive but that survive DOM-to-text serialization, an LLM browsing agent ingests them with the same weight as body copy and can act on them. Brave demonstrated exactly this against Comet (white-on-white text, HTML comments, invisible elements hidden in a Reddit spoiler tag) and confirmed Opera Neon was exploitable through 'hidden HTML elements and other non-rendered markup'. Falsifier: an agent that ingests only visually perceivable, rendered text would be immune \u2014 the disclosed incidents show current agents are not. Google's spam policy independently enumerates the same hiding techniques and their legitimate exceptions, giving the detector a canonical technique list and a false-positive allowlist.",
|
|
@@ -32646,6 +33435,13 @@ var InvisibleInstructionScanAudit = class _InvisibleInstructionScanAudit extends
|
|
|
32646
33435
|
};
|
|
32647
33436
|
}
|
|
32648
33437
|
async audit(ctx) {
|
|
33438
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
33439
|
+
return this.notApplicable(
|
|
33440
|
+
"No page here can be attributed to this site, so its hidden text was not judged.",
|
|
33441
|
+
EXPECTED50,
|
|
33442
|
+
unreadSiteReason(ctx.evidence)
|
|
33443
|
+
);
|
|
33444
|
+
}
|
|
32649
33445
|
const s = await survey9(ctx);
|
|
32650
33446
|
const partial = s.crossOrigin > 0 ? `; ${s.crossOrigin} cross-origin stylesheet not fetched` : "";
|
|
32651
33447
|
if (s.textNodesSeen === 0) {
|
|
@@ -32904,6 +33700,7 @@ var AriaLayerInjectionScanAudit = class _AriaLayerInjectionScanAudit extends Aud
|
|
|
32904
33700
|
evidenceGrade: "A",
|
|
32905
33701
|
tier: "scored",
|
|
32906
33702
|
dossier: "docs/evidence/audits/operability-safety/aria-layer-injection-scan.md",
|
|
33703
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
32907
33704
|
defaultPriority: "critical",
|
|
32908
33705
|
guidance: {
|
|
32909
33706
|
impact: "Computer-use and browser agents drive pages through the DOM and accessibility tree, not pixels, so a11y attributes enter the model context with the same weight as visible text while remaining invisible to a sighted human. Anthropic names the vector explicitly: 'hidden malicious form fields in a webpage's DOM invisible to humans, and other hard-to-catch injections such as through the URL text and tab title that only an agent might see.' The divergence sub-check is a defect in its own right independent of injection: an agent that clicks by accessible name will actuate an aria-label that contradicts the rendered label. Falsifier: if every a11y attribute is short, descriptive, and token-consistent with its element's visible text, this channel carries no payload.",
|
|
@@ -32922,6 +33719,13 @@ var AriaLayerInjectionScanAudit = class _AriaLayerInjectionScanAudit extends Aud
|
|
|
32922
33719
|
};
|
|
32923
33720
|
}
|
|
32924
33721
|
audit(ctx) {
|
|
33722
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
33723
|
+
return this.notApplicable(
|
|
33724
|
+
"No page here can be attributed to this site, so its non-visual values were not judged.",
|
|
33725
|
+
EXPECTED51,
|
|
33726
|
+
unreadSiteReason(ctx.evidence)
|
|
33727
|
+
);
|
|
33728
|
+
}
|
|
32925
33729
|
const s = survey10(ctx);
|
|
32926
33730
|
if (s.valuesSeen === 0) {
|
|
32927
33731
|
return this.notApplicable(
|
|
@@ -32967,6 +33771,13 @@ var AriaLayerInjectionScanAudit = class _AriaLayerInjectionScanAudit extends Aud
|
|
|
32967
33771
|
warnings[0].pageUrl
|
|
32968
33772
|
);
|
|
32969
33773
|
}
|
|
33774
|
+
if (!scanReadPageText(ctx.evidence)) {
|
|
33775
|
+
return this.notApplicable(
|
|
33776
|
+
"The scanned page served no readable text, so its accessibility layer was not judged.",
|
|
33777
|
+
EXPECTED51,
|
|
33778
|
+
unreadPageTextReason(ctx.evidence)
|
|
33779
|
+
);
|
|
33780
|
+
}
|
|
32970
33781
|
return this.pass(
|
|
32971
33782
|
`All ${s.valuesSeen} non-visual value(s) are descriptions that agree with their element and carry no instruction addressed to an AI.`,
|
|
32972
33783
|
EXPECTED51,
|
|
@@ -33067,6 +33878,7 @@ var GhostClickableElementRatioAudit = class _GhostClickableElementRatioAudit ext
|
|
|
33067
33878
|
evidenceGrade: "B",
|
|
33068
33879
|
tier: "scored",
|
|
33069
33880
|
dossier: "docs/evidence/audits/operability-safety/ghost-clickable-element-ratio.md",
|
|
33881
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
33070
33882
|
defaultPriority: "high",
|
|
33071
33883
|
guidance: {
|
|
33072
33884
|
impact: "An element whose click behaviour comes only from a JS listener on a non-interactive tag, or from cursor:pointer styling, and which carries no role and no accessible name, is omitted from the serialized accessibility snapshot that agent toolkits send to the model. Playwright MCP's default mode is the accessibility tree, not pixel input: every action tool takes an exact element reference from the snapshot, and coordinate clicking exists only behind the optional vision capability. An element absent from the snapshot is therefore unaddressable by the default toolchain \u2014 the agent cannot emit a valid click and must fail or guess a URL. The accessibility linters cannot warn about it either: axe's button-name and link-name rules only fire on elements that already declare button or link semantics, so a bare unroled div is invisible to them by construction.",
|
|
@@ -33085,6 +33897,13 @@ var GhostClickableElementRatioAudit = class _GhostClickableElementRatioAudit ext
|
|
|
33085
33897
|
};
|
|
33086
33898
|
}
|
|
33087
33899
|
async audit(ctx) {
|
|
33900
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
33901
|
+
return this.notApplicable(
|
|
33902
|
+
"No page here can be attributed to this site, so its click targets were not counted.",
|
|
33903
|
+
EXPECTED52,
|
|
33904
|
+
unreadSiteReason(ctx.evidence)
|
|
33905
|
+
);
|
|
33906
|
+
}
|
|
33088
33907
|
const s = await survey11(ctx);
|
|
33089
33908
|
const total = s.semantic + s.ghosts.length;
|
|
33090
33909
|
const partial = s.crossOrigin > 0 ? `; ${s.crossOrigin} cross-origin stylesheet not fetched` : "";
|
|
@@ -33289,6 +34108,7 @@ var StatefulControlIntrospectabilityAudit = class _StatefulControlIntrospectabil
|
|
|
33289
34108
|
evidenceGrade: "B",
|
|
33290
34109
|
tier: "scored",
|
|
33291
34110
|
dossier: "docs/evidence/audits/operability-safety/stateful-control-introspectability.md",
|
|
34111
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
33292
34112
|
defaultPriority: "high",
|
|
33293
34113
|
guidance: {
|
|
33294
34114
|
impact: 'An agent works as observe, act, verify. If a toggle\'s only "on" signal is `class="is-active"` and a colour change, the accessibility snapshot is byte-identical before and after the click, so the agent cannot verify the post-condition: it either clicks again and flips the state back, or reports success with no evidence. The accessibility linters cannot catch this, because `aria-required-attr` fires only once the element already declares `role="switch"` or `role="checkbox"` \u2014 the common class-only toggle declares no role and passes silently. Benchmarks put the cost high: WebSuite measures switch, accordion and dropdown primitives among the worst-performing interactions for web agents, and Operator\'s confirmation design assumes the agent can observe a state transition before acting on it.',
|
|
@@ -33498,6 +34318,7 @@ var HoverOnlyContentAndNavigationAudit = class _HoverOnlyContentAndNavigationAud
|
|
|
33498
34318
|
evidenceGrade: "B",
|
|
33499
34319
|
tier: "scored",
|
|
33500
34320
|
dossier: "docs/evidence/audits/operability-safety/hover-only-content-and-navigation.md",
|
|
34321
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
33501
34322
|
defaultPriority: "high",
|
|
33502
34323
|
guidance: {
|
|
33503
34324
|
impact: "A submenu revealed only by an ancestor `:hover` rule is `display:none` or `visibility:hidden` in the resting DOM, and Playwright's actionability contract defines such an element as not visible \u2014 so every Playwright-derived agent refuses to click it, and the snapshot serializer omits it entirely. The agent never learns those destinations exist: it does not fail loudly, it simply reports that the site has no page for what the user asked. WebSuite measures the information half of the same defect at 0% success for tooltip-based retrieval across both agents it tested. The fix is cheap and it is the same fix keyboard users need, which is why it is worth doing once.",
|
|
@@ -33725,6 +34546,7 @@ var DragAndSliderDependencyAudit = class _DragAndSliderDependencyAudit extends A
|
|
|
33725
34546
|
evidenceGrade: "B",
|
|
33726
34547
|
tier: "scored",
|
|
33727
34548
|
dossier: "docs/evidence/audits/operability-safety/drag-and-slider-dependency.md",
|
|
34549
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
33728
34550
|
defaultPriority: "high",
|
|
33729
34551
|
guidance: {
|
|
33730
34552
|
impact: 'A continuous pointer gesture asks an agent to synthesise a pointerdown, a run of intermediate pointermove events and a pointerup at a computed pixel offset, with no feedback between steps and no way to check the interim value. Every other agent action is discrete and verifiable. WebSuite measures slider interaction at 0% success for both agents it tested \u2014 the worst primitive in its taxonomy \u2014 and Anthropic separately documents scrollbars and dropdowns as unreliable under mouse control, recommending keyboard paths instead. Pair the slider with a numeric input bound to the same value and "set max price to 300" stops being a gesture and becomes a fill.',
|
|
@@ -33973,6 +34795,7 @@ var UrlAddressableStateAndPaginationFallbackAudit = class _UrlAddressableStateAn
|
|
|
33973
34795
|
evidenceGrade: "B",
|
|
33974
34796
|
tier: "scored",
|
|
33975
34797
|
dossier: "docs/evidence/audits/operability-safety/url-addressable-state-and-pagination-fallback.md",
|
|
34798
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
33976
34799
|
defaultPriority: "high",
|
|
33977
34800
|
applicablePageTypes: ["category"],
|
|
33978
34801
|
guidance: {
|
|
@@ -34174,6 +34997,7 @@ var FirstContactConsentGateOperabilityAudit = class _FirstContactConsentGateOper
|
|
|
34174
34997
|
evidenceGrade: "C",
|
|
34175
34998
|
tier: "informative",
|
|
34176
34999
|
dossier: "docs/evidence/audits/operability-safety/first-contact-consent-gate-operability.md",
|
|
35000
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
34177
35001
|
defaultPriority: "low",
|
|
34178
35002
|
guidance: {
|
|
34179
35003
|
impact: "An agent arriving with no cookies spends its first actions on the consent layer, before any step of the actual task. Three properties decide whether it can. A layer rendered inside a cross-origin iframe is invisible to a DOM-text extractor that reads only the top document, so the agent's text and its screenshot disagree and it acts on content it cannot actually see. Accept and reject controls built as unroled, unnamed divs are unaddressable in a snapshot for the same reason a ghost-clickable div is. And main content set `inert` or `aria-hidden=\"true\"` while the layer is open empties every snapshot until the layer is gone \u2014 axe's own guidance is that `aria-hidden` removes the element and all its children from the accessibility API. The evidence here is convention rather than documented consumer behaviour, which is why this audit reports rather than scores.",
|
|
@@ -34425,6 +35249,7 @@ var UnicodeCovertChannelScanAudit = class _UnicodeCovertChannelScanAudit extends
|
|
|
34425
35249
|
evidenceGrade: "B",
|
|
34426
35250
|
tier: "scored",
|
|
34427
35251
|
dossier: "docs/evidence/audits/operability-safety/unicode-covert-channel-scan.md",
|
|
35252
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
34428
35253
|
defaultPriority: "critical",
|
|
34429
35254
|
guidance: {
|
|
34430
35255
|
impact: "Tag-block codepoints mirror ASCII and, per Unicode, render as nothing in tag-unaware implementations \u2014 while modern LLM tokenizers process them normally. A complete instruction can therefore ride inside a product description that no human and no visual QA pass can see. Bidi controls make the rendered order differ from the logical order a text-extracting agent reads, which is the Trojan Source class (CVE-2021-42574). Zero-width characters defeat naive substring matching on both sides at once: the site\u2019s own filters and the agent\u2019s. None of this is visible in a screenshot, a browser, or a review \u2014 only in the bytes.",
|
|
@@ -34443,6 +35268,13 @@ var UnicodeCovertChannelScanAudit = class _UnicodeCovertChannelScanAudit extends
|
|
|
34443
35268
|
};
|
|
34444
35269
|
}
|
|
34445
35270
|
audit(ctx) {
|
|
35271
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
35272
|
+
return this.notApplicable(
|
|
35273
|
+
"No response here can be attributed to this site, so its codepoints were not judged.",
|
|
35274
|
+
EXPECTED58,
|
|
35275
|
+
unreadSiteReason(ctx.evidence)
|
|
35276
|
+
);
|
|
35277
|
+
}
|
|
34446
35278
|
const hits2 = [];
|
|
34447
35279
|
for (const page of ctx.pages) hits2.push(...scanPage(page));
|
|
34448
35280
|
for (const path of ROOT_FILES) {
|
|
@@ -34469,6 +35301,16 @@ var UnicodeCovertChannelScanAudit = class _UnicodeCovertChannelScanAudit extends
|
|
|
34469
35301
|
fillerCount: filler
|
|
34470
35302
|
};
|
|
34471
35303
|
if (hits2.length === 0) {
|
|
35304
|
+
if (!scanReadPageText(ctx.evidence)) {
|
|
35305
|
+
return {
|
|
35306
|
+
...this.notApplicable(
|
|
35307
|
+
"The scanned page served no readable text, so its codepoints were not judged.",
|
|
35308
|
+
EXPECTED58,
|
|
35309
|
+
unreadPageTextReason(ctx.evidence)
|
|
35310
|
+
),
|
|
35311
|
+
details
|
|
35312
|
+
};
|
|
35313
|
+
}
|
|
34472
35314
|
return {
|
|
34473
35315
|
...this.pass(
|
|
34474
35316
|
"No invisible codepoint carries text on the scanned pages or in the root files.",
|
|
@@ -34664,6 +35506,10 @@ var ThirdPartyDomWriteBlastRadiusAudit = class _ThirdPartyDomWriteBlastRadiusAud
|
|
|
34664
35506
|
evidenceGrade: "B",
|
|
34665
35507
|
tier: "scored",
|
|
34666
35508
|
dossier: "docs/evidence/audits/operability-safety/third-party-dom-write-blast-radius.md",
|
|
35509
|
+
// Gate exemption: every origin the served HTML names is counted whether or not the
|
|
35510
|
+
// body renders, so a page that ships a vendor script statically is still reported.
|
|
35511
|
+
// The empty census is the case a shell cannot support, and `audit()` declines it.
|
|
35512
|
+
requires: ["origin-reachable", "unblocked-fetches"],
|
|
34667
35513
|
defaultPriority: "high",
|
|
34668
35514
|
guidance: {
|
|
34669
35515
|
impact: 'An agent reads the DOM as one document with one level of trust. It has no way to tell text the site wrote from text a vendor script injected after load, so every third-party origin that can write to the page can write instructions the agent will read as the site\'s own. The count is the risk: eleven uncontrolled origins is eleven independent companies \u2014 and their own supply chains \u2014 with the same authority over what an agent believes about the site. A Content-Security-Policy with a nonce, a hash or `strict-dynamic` is what turns that list from "whoever" into "these, and only these". A policy whose sources include `https:` or `*` is present in the response and constrains nothing.',
|
|
@@ -34682,6 +35528,13 @@ var ThirdPartyDomWriteBlastRadiusAudit = class _ThirdPartyDomWriteBlastRadiusAud
|
|
|
34682
35528
|
};
|
|
34683
35529
|
}
|
|
34684
35530
|
audit(ctx) {
|
|
35531
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
35532
|
+
return this.notApplicable(
|
|
35533
|
+
"No page here can be attributed to this site, so its third-party surface was not measured.",
|
|
35534
|
+
EXPECTED59,
|
|
35535
|
+
unreadSiteReason(ctx.evidence)
|
|
35536
|
+
);
|
|
35537
|
+
}
|
|
34685
35538
|
if (ctx.pages.length === 0) {
|
|
34686
35539
|
return this.notApplicable(
|
|
34687
35540
|
"No page was fetched, so there is no third-party surface to measure.",
|
|
@@ -34721,6 +35574,17 @@ var ThirdPartyDomWriteBlastRadiusAudit = class _ThirdPartyDomWriteBlastRadiusAud
|
|
|
34721
35574
|
details
|
|
34722
35575
|
};
|
|
34723
35576
|
}
|
|
35577
|
+
if (!scanReadPageText(ctx.evidence)) {
|
|
35578
|
+
return {
|
|
35579
|
+
...this.notApplicable(
|
|
35580
|
+
"The scanned page served no readable text, so the origins writing into it were not counted.",
|
|
35581
|
+
EXPECTED59,
|
|
35582
|
+
unreadPageTextReason(ctx.evidence)
|
|
35583
|
+
),
|
|
35584
|
+
displayValue: found,
|
|
35585
|
+
details
|
|
35586
|
+
};
|
|
35587
|
+
}
|
|
34724
35588
|
return {
|
|
34725
35589
|
...this.pass(
|
|
34726
35590
|
"No third-party origin ships executable code into the page, so nothing but the site itself writes what an agent reads.",
|
|
@@ -34859,6 +35723,7 @@ var UnsafeAgentTriggerableAffordancesAudit = class _UnsafeAgentTriggerableAfford
|
|
|
34859
35723
|
evidenceGrade: "B",
|
|
34860
35724
|
tier: "scored",
|
|
34861
35725
|
dossier: "docs/evidence/audits/operability-safety/unsafe-agent-triggerable-affordances.md",
|
|
35726
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
34862
35727
|
defaultPriority: "critical",
|
|
34863
35728
|
guidance: {
|
|
34864
35729
|
impact: "An agent exploring a site follows links, and a link that changes state changes it on the first fetch \u2014 no click, no intent, no confirmation. The same property makes the site a target for indirect prompt injection: text on a page can name the URL, and an agent that reads it as an instruction performs the action with the user's own session. Disallowing the path in robots.txt is only a partial mitigation, because a user-initiated fetch is documented as not necessarily bound by robots.txt. The underlying rule is older than agents: a GET is a safe method, meaning it must not have side effects, and everything here is a violation of that rule that agents simply make expensive.",
|
|
@@ -34877,6 +35742,13 @@ var UnsafeAgentTriggerableAffordancesAudit = class _UnsafeAgentTriggerableAfford
|
|
|
34877
35742
|
};
|
|
34878
35743
|
}
|
|
34879
35744
|
audit(ctx) {
|
|
35745
|
+
if (!scanReadTheSite(ctx.evidence)) {
|
|
35746
|
+
return this.notApplicable(
|
|
35747
|
+
"No page here can be attributed to this site, so its links were not inspected.",
|
|
35748
|
+
EXPECTED60,
|
|
35749
|
+
unreadSiteReason(ctx.evidence)
|
|
35750
|
+
);
|
|
35751
|
+
}
|
|
34880
35752
|
if (ctx.pages.length === 0) {
|
|
34881
35753
|
return this.notApplicable(
|
|
34882
35754
|
"No page was fetched, so there is no link to inspect.",
|
|
@@ -34895,6 +35767,16 @@ var UnsafeAgentTriggerableAffordancesAudit = class _UnsafeAgentTriggerableAfford
|
|
|
34895
35767
|
urls: findings.slice(0, 10).map((f) => f.href)
|
|
34896
35768
|
};
|
|
34897
35769
|
if (findings.length === 0) {
|
|
35770
|
+
if (!scanReadPageText(ctx.evidence)) {
|
|
35771
|
+
return {
|
|
35772
|
+
...this.notApplicable(
|
|
35773
|
+
"The scanned page served no readable text, so it exposed no links or forms to inspect.",
|
|
35774
|
+
EXPECTED60,
|
|
35775
|
+
unreadPageTextReason(ctx.evidence)
|
|
35776
|
+
),
|
|
35777
|
+
details
|
|
35778
|
+
};
|
|
35779
|
+
}
|
|
34898
35780
|
return {
|
|
34899
35781
|
...this.pass(
|
|
34900
35782
|
"No link or GET form on the scanned pages changes state when it is fetched.",
|
|
@@ -34996,6 +35878,7 @@ var ReflectedParameterInjectionCanaryAudit = class extends Audit {
|
|
|
34996
35878
|
weight: weightForGrade("B", "scored"),
|
|
34997
35879
|
defaultPriority: "critical",
|
|
34998
35880
|
dossier: "docs/evidence/audits/operability-safety/reflected-parameter-injection-canary.md",
|
|
35881
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
34999
35882
|
guidance: {
|
|
35000
35883
|
impact: "Agents and answer engines weight a source by domain authority, and a reflected-input URL passes human inspection because the hostname is genuine. If attacker-controlled query or path input lands in the page's own title, meta description, canonical link, or JSON-LD strings, the domain becomes a self-serve injection host: the attacker does not need to compromise anything, only to share a link. Reflection into rendered text is the same defect one step down, and it is only contained while the page stays out of an index.",
|
|
35001
35884
|
fix: 'Escape URL-derived input before it reaches any template, and keep it out of `<title>`, `<meta name="description">`, `og:description`, `rel="canonical"` and JSON-LD entirely \u2014 those fields should describe the page, not the request. Where a search page must echo the query back to the visitor, render it as escaped text inside the body and mark the page `noindex`.',
|
|
@@ -35264,6 +36147,7 @@ var UgcTrustBoundaryMarkersAudit = class extends Audit {
|
|
|
35264
36147
|
weight: weightForGrade("B", "scored"),
|
|
35265
36148
|
defaultPriority: "high",
|
|
35266
36149
|
dossier: "docs/evidence/audits/operability-safety/ugc-trust-boundary-markers.md",
|
|
36150
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
35267
36151
|
guidance: {
|
|
35268
36152
|
impact: "Attacker-controllable text sits in the same DOM as first-party copy with no boundary, so anything a visitor types reads, to a fetching agent, as a statement the domain made. Google excludes text inside a `data-nosnippet` span, div or section from snippets across web search, Discover and AI Overviews, and includes everything outside it. The sanitizer arm matters most: if a comment body can carry an inline style or an iframe, hiding an instruction inside visitor text becomes self-serve on this site.",
|
|
35269
36153
|
fix: 'Wrap each visitor-written region in a `<div data-nosnippet>` \u2014 the attribute is honoured on span, div and section only \u2014 and add `rel="ugc"` to links inside it. Strip inline `style`, `iframe`, `script` and remote `img` from submitted markup at render time rather than at submit time, so already-stored content is covered too.',
|
|
@@ -35385,6 +36269,7 @@ var AgentUaContentDivergenceDiffAudit = class extends Audit {
|
|
|
35385
36269
|
weight: weightForGrade("B", "scored"),
|
|
35386
36270
|
defaultPriority: "high",
|
|
35387
36271
|
dossier: "docs/evidence/audits/operability-safety/agent-ua-content-divergence-diff.md",
|
|
36272
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
35388
36273
|
guidance: {
|
|
35389
36274
|
impact: "An agent that reads a different page from the one a human sees cannot be checked by the human it answers to. Where the crawler copy is thinner, the answer engine quotes a page the visitor will never find; where it carries text the browser copy does not, the site is speaking to the model privately \u2014 which is the delivery mechanism for every instruction-injection attack that does not need a compromise. A JSON-LD block that differs between variants is the same problem in the field a machine trusts most.",
|
|
35390
36275
|
fix: "Serve one document to every User-Agent. Where a bot-management rule reduces the page for unknown clients, allow the published AI-crawler UAs through it rather than branching on them, and keep the JSON-LD identical across variants. If a crawler should not read the site at all, block it in robots.txt and at the edge rather than serving it a different story.",
|
|
@@ -35755,6 +36640,7 @@ var C2paManifestSurvivesDeliveryAudit = class extends Audit {
|
|
|
35755
36640
|
weight: weightForGrade("B", "scored"),
|
|
35756
36641
|
defaultPriority: "medium",
|
|
35757
36642
|
dossier: "docs/evidence/audits/operability-safety/c2pa-manifest-survives-delivery.md",
|
|
36643
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
35758
36644
|
guidance: {
|
|
35759
36645
|
impact: "Signing an image at creation proves nothing if the bytes a crawler downloads are unsigned. Image transformation layers discard Content Credentials by default \u2014 Cloudflare states outright that with preservation disabled, existing Content Credentials are always discarded \u2014 so the publisher sees signed assets in their library while every consumer sees stripped ones. The provenance work is done and none of it reaches the reader.",
|
|
35760
36646
|
fix: "Turn on Content Credentials preservation in the image pipeline (Cloudflare Images has an explicit setting; Next.js image optimization and most CDN resizers need the manifest copied through or the asset served unoptimized). Verify by fetching the URL the page actually renders, not the asset in the library.",
|
|
@@ -35904,6 +36790,7 @@ var C2paSignerTrustStatusAudit = class extends Audit {
|
|
|
35904
36790
|
weight: weightForGrade("B", "scored"),
|
|
35905
36791
|
defaultPriority: "medium",
|
|
35906
36792
|
dossier: "docs/evidence/audits/operability-safety/c2pa-signer-trust-status.md",
|
|
36793
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
35907
36794
|
guidance: {
|
|
35908
36795
|
impact: "A manifest that exists is not a manifest that verifies. A conforming C2PA validator resolves the signing certificate against the published Trust List and shows the credential as untrusted when it cannot \u2014 which is what a self-signed certificate always produces, and what an expired one produces the day it lapses. The publisher sees Content Credentials on every asset; the consumer sees a warning, or nothing at all.",
|
|
35909
36796
|
fix: "Sign with a certificate from a CA on the C2PA Trust List rather than a self-signed one, renew before it expires, and include an RFC 3161 timestamp so credentials stay valid past the certificate\u2019s own expiry.",
|
|
@@ -36082,6 +36969,7 @@ var OrganizationIdentifierRegistryResolutionAudit = class extends Audit {
|
|
|
36082
36969
|
weight: weightForGrade("B", "scored"),
|
|
36083
36970
|
defaultPriority: "medium",
|
|
36084
36971
|
dossier: "docs/evidence/audits/operability-safety/organization-identifier-registry-resolution.md",
|
|
36972
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
36085
36973
|
guidance: {
|
|
36086
36974
|
impact: "A shopping or payment agent transacting with an unfamiliar merchant needs one thing no amount of markup can self-assert: a legal identity it can check against an authority. The LEI is the only schema.org organization identifier backed by a free, queryable, authoritative registry, which makes it the only one whose truth an outside party can establish. An identifier that resolves to nothing, or to a lapsed registration, or to a different legal name, is worse than none: it looks like verification and is not.",
|
|
36087
36975
|
fix: 'Publish the LEI as `iso6523Code: "0199:<LEI>"` \u2014 Google documents a preference for the prefixed form over bare `leiCode` \u2014 keep the GLEIF registration renewed so its status stays ISSUED, and make sure the `legalName` in your markup is the name GLEIF has on record, not the trading name.',
|
|
@@ -36295,6 +37183,7 @@ var SyntheticMediaDisclosureValidityAudit = class extends Audit {
|
|
|
36295
37183
|
weight: weightForGrade("B", "scored"),
|
|
36296
37184
|
defaultPriority: "medium",
|
|
36297
37185
|
dossier: "docs/evidence/audits/operability-safety/synthetic-media-disclosure-validity.md",
|
|
37186
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
36298
37187
|
guidance: {
|
|
36299
37188
|
impact: "Disclosure only counts if a machine can read it. IPTC types `DigitalSourceType` as a URI from a controlled vocabulary, so a consumer matching against that vocabulary silently ignores `AI-generated`, a bare `trainedAlgorithmicMedia`, or an `https://` spelling of the `http://` vocabulary URI. The publisher believes the image is disclosed; every machine reader sees an undisclosed image. Worse is an asset whose XMP and C2PA manifest disagree about whether a human took the photo \u2014 two provenance channels, one of them wrong.",
|
|
36300
37189
|
fix: "Write the full vocabulary URI, exactly: `http://cv.iptc.org/newscodes/digitalsourcetype/trainedAlgorithmicMedia`. Keep the `http` scheme the vocabulary itself uses, no trailing slash, no free text, and make sure the value agrees with the digital source type asserted in the asset\u2019s C2PA manifest.",
|
|
@@ -36464,6 +37353,7 @@ var TrustTxtReciprocityCoherenceAudit = class extends Audit {
|
|
|
36464
37353
|
weight: 0,
|
|
36465
37354
|
defaultPriority: "low",
|
|
36466
37355
|
dossier: "docs/evidence/audits/operability-safety/trust-txt-reciprocity-coherence.md",
|
|
37356
|
+
requires: ["origin-reachable"],
|
|
36467
37357
|
guidance: {
|
|
36468
37358
|
impact: "trust.txt association attributes are defined as reciprocal: `belongto=<association>` means something only if that association\u2019s own trust.txt carries `member=<this domain>`. That makes the claim checkable rather than self-asserted, which is the whole point of publishing it. Separately, `datatrainingallowed=no` beside a robots.txt that leaves GPTBot and ClaudeBot free to crawl states two opposite policies, and the channel that actually gates crawlers is the one that says yes. Adoption caveat: no AI engine, answer engine or crawler is documented as reading trust.txt.",
|
|
36469
37359
|
fix: "Ask each association you claim to belong to for a reciprocal `member=` line, drop the ones that will not reciprocate, and make `datatrainingallowed=` say the same thing your robots.txt AI-bot groups say.",
|
|
@@ -36656,6 +37546,7 @@ var WikidataRoundTripVerificationAudit = class extends Audit {
|
|
|
36656
37546
|
weight: weightForGrade("B", "scored"),
|
|
36657
37547
|
defaultPriority: "medium",
|
|
36658
37548
|
dossier: "docs/evidence/audits/operability-safety/wikidata-round-trip-verification.md",
|
|
37549
|
+
requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
|
|
36659
37550
|
guidance: {
|
|
36660
37551
|
impact: "A knowledge-graph consumer that grounds a brand to an entity needs corroboration from the authority side, because `sameAs` carries no reciprocity requirement \u2014 Google documents it as a link to a page with more information, nothing more. Wikidata publishes that corroboration for free as P856. A claim whose entity points at an unrelated domain is either the wrong entity or an unbacked identity claim, and an answer engine that resolves it grounds the brand to somebody else.",
|
|
36661
37552
|
fix: "Claim the entity that really is your organization, and make sure the Wikidata item carries your domain as its official website (P856). If the item has no P856 at all, add one: until it does, the claim cannot be corroborated by anyone.",
|
|
@@ -36860,6 +37751,7 @@ function outcomeOf(check) {
|
|
|
36860
37751
|
const tags = check.tags ?? [];
|
|
36861
37752
|
if (tags.includes(TAG_SCAN_ERROR)) return "error";
|
|
36862
37753
|
if (tags.includes(TAG_SKIPPED_PAGE_TYPE)) return "skipped";
|
|
37754
|
+
if (tags.includes(TAG_SKIPPED_NO_EVIDENCE)) return "gated";
|
|
36863
37755
|
return "ran";
|
|
36864
37756
|
}
|
|
36865
37757
|
function traceFromCheck(check, durationMs) {
|
|
@@ -36921,7 +37813,31 @@ function stubCheck(meta2, tag2, explanation) {
|
|
|
36921
37813
|
tier: meta2.tier
|
|
36922
37814
|
};
|
|
36923
37815
|
}
|
|
36924
|
-
function
|
|
37816
|
+
function unmetRequirements(ctx, meta2) {
|
|
37817
|
+
const required = meta2.requires ?? [];
|
|
37818
|
+
if (required.length === 0) return [];
|
|
37819
|
+
const evidence = ctx.evidence;
|
|
37820
|
+
const unmet = [];
|
|
37821
|
+
for (const key2 of required) {
|
|
37822
|
+
if (key2 === "sample-adequate") {
|
|
37823
|
+
const wanted = meta2.applicablePageTypes?.length ? meta2.applicablePageTypes : ["homepage"];
|
|
37824
|
+
if (!wanted.some((type) => evidence.usablePageTypes.has(type))) unmet.push(key2);
|
|
37825
|
+
continue;
|
|
37826
|
+
}
|
|
37827
|
+
if (!evidence.met[key2]) unmet.push(key2);
|
|
37828
|
+
}
|
|
37829
|
+
return unmet;
|
|
37830
|
+
}
|
|
37831
|
+
function gateExplanation(ctx, meta2, unmet) {
|
|
37832
|
+
const reasons = unmet.map((key2) => ctx.evidence.reasons[key2]).filter(Boolean);
|
|
37833
|
+
if (unmet.includes("sample-adequate") && reasons.length === 0) {
|
|
37834
|
+
const wanted = meta2.applicablePageTypes?.length ? meta2.applicablePageTypes.join("/") : "homepage";
|
|
37835
|
+
return `Not assessed: no scanned ${wanted} page served readable text.`;
|
|
37836
|
+
}
|
|
37837
|
+
const why = reasons.length > 0 ? ` ${reasons.join(" ")}` : "";
|
|
37838
|
+
return `Not assessed: this scan has no ${unmet.join(", ")} evidence.${why}`;
|
|
37839
|
+
}
|
|
37840
|
+
function planAudits(ctx, config, options = {}) {
|
|
36925
37841
|
const scannedPageTypes = new Set(ctx.pages.map((p) => p.pageType));
|
|
36926
37842
|
const runnable = [];
|
|
36927
37843
|
const skipped = [];
|
|
@@ -36941,6 +37857,15 @@ function planAudits(ctx, config) {
|
|
|
36941
37857
|
continue;
|
|
36942
37858
|
}
|
|
36943
37859
|
}
|
|
37860
|
+
if (options.enforceEvidence) {
|
|
37861
|
+
const unmet = unmetRequirements(ctx, reg2.meta);
|
|
37862
|
+
if (unmet.length > 0) {
|
|
37863
|
+
skipped.push(
|
|
37864
|
+
stubCheck(reg2.meta, TAG_SKIPPED_NO_EVIDENCE, gateExplanation(ctx, reg2.meta, unmet))
|
|
37865
|
+
);
|
|
37866
|
+
continue;
|
|
37867
|
+
}
|
|
37868
|
+
}
|
|
36944
37869
|
runnable.push({ reg: reg2, categoryId: cat.id });
|
|
36945
37870
|
}
|
|
36946
37871
|
}
|
|
@@ -37263,6 +38188,18 @@ function detectWafProtection(targetUrl, homepageResult, rootFiles, scannedPagesC
|
|
|
37263
38188
|
|
|
37264
38189
|
// src/orchestrator.ts
|
|
37265
38190
|
var A11Y_MAX_PAGES = Math.max(0, Number(process.env.SCANNER_A11Y_MAX_PAGES ?? 3));
|
|
38191
|
+
var RATE_LIMIT_BACKOFF_MS = 5e3;
|
|
38192
|
+
var MAX_RETRY_AFTER_MS = 3e4;
|
|
38193
|
+
async function fetchHomepage(fetcher, url, signal) {
|
|
38194
|
+
const first5 = await fetcher.fetch({ url, signal });
|
|
38195
|
+
if (first5.status !== 429) return first5;
|
|
38196
|
+
const header = Number(first5.headers["retry-after"]);
|
|
38197
|
+
const waitMs = Number.isFinite(header) && header > 0 ? Math.min(header * 1e3, MAX_RETRY_AFTER_MS) : RATE_LIMIT_BACKOFF_MS;
|
|
38198
|
+
logger.debug({ url, waitMs }, `[orchestrator] Homepage answered 429; retrying once in ${waitMs}ms`);
|
|
38199
|
+
await new Promise((resolve4) => setTimeout(resolve4, waitMs));
|
|
38200
|
+
signal?.throwIfAborted();
|
|
38201
|
+
return fetcher.fetch({ url, signal });
|
|
38202
|
+
}
|
|
37266
38203
|
function discoverPages(homepageUrl, domain, rootFiles, homepage$, exclude, maxAdditional) {
|
|
37267
38204
|
const discovered = /* @__PURE__ */ new Set();
|
|
37268
38205
|
const sitemapBody = rootFiles["/sitemap.xml"]?.status === 200 ? rootFiles["/sitemap.xml"].body : rootFiles["/sitemap-index.xml"]?.status === 200 ? rootFiles["/sitemap-index.xml"].body : "";
|
|
@@ -37355,7 +38292,10 @@ async function runScan(url, options) {
|
|
|
37355
38292
|
const signal = options?.signal;
|
|
37356
38293
|
const tracker = new ProgressTracker((event) => onEvent?.(event));
|
|
37357
38294
|
const start = performance.now();
|
|
37358
|
-
const fetcher = createFetcher(
|
|
38295
|
+
const fetcher = createFetcher({
|
|
38296
|
+
dispatcher: options?.dispatcher,
|
|
38297
|
+
maxConcurrent: options?.maxConcurrent
|
|
38298
|
+
});
|
|
37359
38299
|
const baseUrl = new URL(url).origin;
|
|
37360
38300
|
const domain = new URL(url).hostname;
|
|
37361
38301
|
const displayUrl = splitCredentials(url).url;
|
|
@@ -37413,13 +38353,18 @@ async function runScan(url, options) {
|
|
|
37413
38353
|
];
|
|
37414
38354
|
logger.debug({ count: rootFilePaths.length }, "[orchestrator] Phase 1: Fetching root files");
|
|
37415
38355
|
tracker.phaseStart("fetch-root", rootFilePaths.length);
|
|
38356
|
+
const prefetchedRobots = options?.robotsTxt;
|
|
37416
38357
|
const rootResults = await Promise.all(
|
|
37417
|
-
rootFilePaths.map(
|
|
37418
|
-
(path
|
|
38358
|
+
rootFilePaths.map((path) => {
|
|
38359
|
+
if (path === "/robots.txt" && prefetchedRobots) {
|
|
38360
|
+
tracker.unitDone(path);
|
|
38361
|
+
return Promise.resolve(prefetchedRobots);
|
|
38362
|
+
}
|
|
38363
|
+
return fetcher.fetch({ url: `${baseUrl}${path}`, signal }).then((result) => {
|
|
37419
38364
|
tracker.unitDone(path);
|
|
37420
38365
|
return result;
|
|
37421
|
-
})
|
|
37422
|
-
)
|
|
38366
|
+
});
|
|
38367
|
+
})
|
|
37423
38368
|
);
|
|
37424
38369
|
const rootFiles = {};
|
|
37425
38370
|
rootFilePaths.forEach((path, i) => {
|
|
@@ -37430,7 +38375,7 @@ async function runScan(url, options) {
|
|
|
37430
38375
|
signal?.throwIfAborted();
|
|
37431
38376
|
logger.debug("[orchestrator] Phase 2: Fetching pages");
|
|
37432
38377
|
tracker.phaseStart("fetch-pages", 1);
|
|
37433
|
-
const homepageResult = await fetcher
|
|
38378
|
+
const homepageResult = await fetchHomepage(fetcher, url, signal);
|
|
37434
38379
|
tracker.unitDone(displayUrl);
|
|
37435
38380
|
const homepage$ = homepageResult.status === 200 && homepageResult.body ? parseHtml(homepageResult.body) : null;
|
|
37436
38381
|
const discoverLimit = Math.max(0, MAX_PAGES_PER_SCAN - 1 - overrideUrls.length);
|
|
@@ -37489,19 +38434,29 @@ async function runScan(url, options) {
|
|
|
37489
38434
|
signal?.throwIfAborted();
|
|
37490
38435
|
logger.debug("[orchestrator] Phase 3: Running audits");
|
|
37491
38436
|
const wafProtection = detectWafProtection(url, homepageResult, rootFiles, pages.length);
|
|
38437
|
+
const evidence = buildScanEvidence({
|
|
38438
|
+
requestedUrl: url,
|
|
38439
|
+
homepageResult,
|
|
38440
|
+
pages,
|
|
38441
|
+
rootFiles,
|
|
38442
|
+
wafProtection: wafProtection ?? null
|
|
38443
|
+
});
|
|
37492
38444
|
const ctx = {
|
|
37493
38445
|
rootFiles,
|
|
37494
38446
|
pages,
|
|
37495
38447
|
domain,
|
|
37496
38448
|
baseUrl,
|
|
37497
38449
|
fetch: (options2) => fetcher.fetch({ ...options2, signal }),
|
|
37498
|
-
wafProtection: wafProtection ?? void 0
|
|
38450
|
+
wafProtection: wafProtection ?? void 0,
|
|
38451
|
+
evidence
|
|
37499
38452
|
};
|
|
37500
38453
|
const config = filterConfig(defaultConfig, {
|
|
37501
38454
|
categories: options?.categories,
|
|
37502
38455
|
includeExperimental: options?.includeExperimental ?? false
|
|
37503
38456
|
});
|
|
37504
|
-
const auditPlan = planAudits(ctx, config
|
|
38457
|
+
const auditPlan = planAudits(ctx, config, {
|
|
38458
|
+
enforceEvidence: options?.enforceEvidenceGate ?? true
|
|
38459
|
+
});
|
|
37505
38460
|
tracker.phaseStart("audits", auditPlan.runnable.length);
|
|
37506
38461
|
const {
|
|
37507
38462
|
checks: allChecks,
|
|
@@ -37522,7 +38477,7 @@ async function runScan(url, options) {
|
|
|
37522
38477
|
tracker.phaseStart("report", 1);
|
|
37523
38478
|
logger.debug("[orchestrator] Phase 4: Building final report");
|
|
37524
38479
|
const durationMs = Math.round(performance.now() - start);
|
|
37525
|
-
const recommendations = allChecks.filter((c) => c.status
|
|
38480
|
+
const recommendations = allChecks.filter((c) => (c.status === "fail" || c.status === "warn") && !isInformative(c)).slice().sort((a, b) => {
|
|
37526
38481
|
const order = { critical: 0, high: 1, medium: 2, low: 3 };
|
|
37527
38482
|
return (order[a.priority] ?? 3) - (order[b.priority] ?? 3);
|
|
37528
38483
|
});
|
|
@@ -37534,13 +38489,23 @@ async function runScan(url, options) {
|
|
|
37534
38489
|
const readinessScore = Math.round(
|
|
37535
38490
|
readinessVitals.commerce * READINESS_WEIGHTS.commerce + readinessVitals.content * READINESS_WEIGHTS.content + readinessVitals.botAccessibility * READINESS_WEIGHTS.botAccessibility + readinessVitals.technical * READINESS_WEIGHTS.technical
|
|
37536
38491
|
);
|
|
38492
|
+
const gatedShare = gatedMassShare(allChecks);
|
|
38493
|
+
const escalated = gatedShare > GATED_MASS_UNSCORED_THRESHOLD;
|
|
38494
|
+
const unscoredReason = !evidence.judgeable ? Object.values(evidence.reasons).filter(Boolean).join(" ") || "The scan obtained too little evidence to judge this site." : escalated ? `The scan could not feed ${Math.round(gatedShare * 100)}% of the registry's evidence mass, so what remains is not a reading of this site.` : void 0;
|
|
38495
|
+
const scored = unscoredReason === void 0;
|
|
37537
38496
|
const report = {
|
|
37538
38497
|
scanId: "",
|
|
37539
38498
|
// Set by the caller
|
|
37540
38499
|
url: displayUrl,
|
|
37541
38500
|
domain,
|
|
37542
|
-
overallScore,
|
|
37543
|
-
scoreTier: getScoreTier(overallScore),
|
|
38501
|
+
overallScore: scored ? overallScore : null,
|
|
38502
|
+
scoreTier: scored ? getScoreTier(overallScore) : null,
|
|
38503
|
+
scanValidity: {
|
|
38504
|
+
judgeable: evidence.judgeable,
|
|
38505
|
+
evidence: evidence.met,
|
|
38506
|
+
reasons: evidence.reasons,
|
|
38507
|
+
...unscoredReason ? { unscoredReason } : {}
|
|
38508
|
+
},
|
|
37544
38509
|
summary: "",
|
|
37545
38510
|
// Set below
|
|
37546
38511
|
categories,
|
|
@@ -37561,7 +38526,7 @@ async function runScan(url, options) {
|
|
|
37561
38526
|
report.summary = generateScanSummary(report);
|
|
37562
38527
|
tracker.unitDone();
|
|
37563
38528
|
tracker.phaseDone();
|
|
37564
|
-
tracker.scanDone(overallScore);
|
|
38529
|
+
tracker.scanDone(report.overallScore);
|
|
37565
38530
|
logger.debug({ durationMs, score: overallScore }, "[orchestrator] runScan complete");
|
|
37566
38531
|
return report;
|
|
37567
38532
|
}
|
|
@@ -37730,6 +38695,7 @@ export {
|
|
|
37730
38695
|
DEFAULT_SCAN_LIMIT,
|
|
37731
38696
|
DeprecationNoticeSchema,
|
|
37732
38697
|
EvidenceGradeSchema,
|
|
38698
|
+
EvidenceKeySchema,
|
|
37733
38699
|
FixEffortSchema,
|
|
37734
38700
|
MAX_CONCURRENT_REQUESTS,
|
|
37735
38701
|
MAX_PAGES_PER_SCAN,
|
|
@@ -37745,9 +38711,13 @@ export {
|
|
|
37745
38711
|
SCORE_TIER_LABELS,
|
|
37746
38712
|
ScoreDisplayModeSchema,
|
|
37747
38713
|
TAG_SCAN_ERROR,
|
|
38714
|
+
TAG_SKIPPED_NO_EVIDENCE,
|
|
37748
38715
|
TAG_SKIPPED_PAGE_TYPE,
|
|
38716
|
+
allEvidenceMet,
|
|
37749
38717
|
allJsonLdNodes,
|
|
38718
|
+
boundedDispatcher,
|
|
37750
38719
|
buildCategoryResult,
|
|
38720
|
+
buildScanEvidence,
|
|
37751
38721
|
calculateCategoryScore,
|
|
37752
38722
|
calculateOverallScore,
|
|
37753
38723
|
classifyFetch,
|
|
@@ -37778,6 +38748,7 @@ export {
|
|
|
37778
38748
|
formatTrace,
|
|
37779
38749
|
getMainContentText,
|
|
37780
38750
|
getPreset,
|
|
38751
|
+
getRenderedText,
|
|
37781
38752
|
getScoreTier,
|
|
37782
38753
|
getTierColor,
|
|
37783
38754
|
getTierLabel,
|