@reddoorla/maintenance 0.93.0 → 0.94.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -13
- package/dist/airtable-JGCPS7JH.js +9 -0
- package/dist/{announce-6OS4C57D.js → announce-PGT4SZZB.js} +6 -6
- package/dist/{bump-deps-GI7PHMOC.js → bump-deps-6OR7LL4E.js} +2 -2
- package/dist/{chunk-7NYN2P6I.js → chunk-3Z6X3BHU.js} +15 -8
- package/dist/chunk-3Z6X3BHU.js.map +1 -0
- package/dist/{chunk-MVDMKKBN.js → chunk-4AIYWWKV.js} +12 -5
- package/dist/chunk-4AIYWWKV.js.map +1 -0
- package/dist/{chunk-QLIUDTY6.js → chunk-5EEMTJMT.js} +10 -7
- package/dist/chunk-5EEMTJMT.js.map +1 -0
- package/dist/{chunk-4NUKMPOO.js → chunk-5FCBYK6L.js} +3 -3
- package/dist/{chunk-PR343K7F.js → chunk-5GUJ36TL.js} +2 -2
- package/dist/{chunk-P6AORG32.js → chunk-6R56OKJZ.js} +2 -2
- package/dist/{chunk-E22WAIF2.js → chunk-AEUABPAJ.js} +3 -3
- package/dist/{chunk-ZRI5VRPG.js → chunk-BYGPUUCT.js} +2 -2
- package/dist/{chunk-BJQCIJOY.js → chunk-DHVJ7H2F.js} +230 -805
- package/dist/chunk-DHVJ7H2F.js.map +1 -0
- package/dist/{chunk-SKI23ZZH.js → chunk-HTABUCF4.js} +2 -2
- package/dist/{chunk-FCS36FDT.js → chunk-KQB2BX4V.js} +3 -3
- package/dist/{chunk-AX22DH6S.js → chunk-KZRR4NFG.js} +3 -3
- package/dist/{chunk-CVJF2IC5.js → chunk-TGUESYGF.js} +17 -3
- package/dist/{chunk-CVJF2IC5.js.map → chunk-TGUESYGF.js.map} +1 -1
- package/dist/{chunk-JB4Q3C3M.js → chunk-TO7DJYLZ.js} +92 -15
- package/dist/{chunk-JB4Q3C3M.js.map → chunk-TO7DJYLZ.js.map} +1 -1
- package/dist/chunk-VELWHWOW.js +4194 -0
- package/dist/chunk-VELWHWOW.js.map +1 -0
- package/dist/{chunk-NDCOOI3V.js → chunk-WMBLR5VA.js} +3 -3
- package/dist/{chunk-QBAAZY7W.js → chunk-WVNDEYXL.js} +3 -3
- package/dist/{chunk-7BZUOZPU.js → chunk-XN37VTDB.js} +2 -2
- package/dist/{chunk-Q47DU255.js → chunk-Z2K4JPQ3.js} +2 -2
- package/dist/cli/bin.js +26 -26
- package/dist/cli/commands/audit.js +11 -11
- package/dist/{convert-to-pnpm-ONFP3JKF.js → convert-to-pnpm-3RVGUBBR.js} +2 -2
- package/dist/{db-HSZ6D5DU.js → db-FL7ZBF6H.js} +5 -5
- package/dist/{digest-EOA4JLTV.js → digest-TZ3QQOSF.js} +7 -7
- package/dist/{digest-collectors-ZA3BTS3R.js → digest-collectors-EHFY2OQC.js} +6 -6
- package/dist/{ensure-site-SF6KICTN.js → ensure-site-WIDO6LGU.js} +3 -3
- package/dist/{fleet-state-3LQUQHYD.js → fleet-state-UEVZ3X3J.js} +4 -4
- package/dist/forms/index.d.ts +6 -1
- package/dist/forms/index.js +1 -1
- package/dist/forms/index.js.map +1 -1
- package/dist/{forms-notify-target-RMZPKT6S.js → forms-notify-target-2LWF5C5B.js} +3 -3
- package/dist/{github-signals-3WLIJ5RR.js → github-signals-GAHNOAHS.js} +5 -5
- package/dist/{header-image-DOCDFFRO.js → header-image-YN3AQRY3.js} +2 -2
- package/dist/{health-endpoint-DHKJUPRM.js → health-endpoint-XLNBWVHK.js} +2 -2
- package/dist/{health-mirror-55RRU6BI.js → health-mirror-EB76EERV.js} +4 -4
- package/dist/{import-airtable-OHVY344G.js → import-airtable-TAKSN4NV.js} +3 -3
- package/dist/index.d.ts +87 -6
- package/dist/index.js +14 -11
- package/dist/index.js.map +1 -1
- package/dist/{init-ZRKWYCKA.js → init-F5OBBWA5.js} +6 -6
- package/dist/{launch-6X5OXPF2.js → launch-SUTMI4NG.js} +9 -9
- package/dist/{lighthouse-airtable-3GUWHETW.js → lighthouse-airtable-YMH4UA7R.js} +3 -3
- package/dist/{onboard-6NHGCKGB.js → onboard-NKE6GZKL.js} +2 -2
- package/dist/{orchestrate-KMQIUP3H.js → orchestrate-MWXENQOS.js} +4 -4
- package/dist/{parity-623WNFZG.js → parity-CAJ477PS.js} +5 -5
- package/dist/{pipeline-AQBFG5WM.js → pipeline-PRMD4EJT.js} +3 -2
- package/dist/{preflight-VPHQYWTY.js → preflight-4VQP7JJL.js} +5 -5
- package/dist/{prismic-ci-A2HI7NNJ.js → prismic-ci-QIO3I4KB.js} +2 -2
- package/dist/{prismic-models-QTKIMFBC.js → prismic-models-RWDVKZ2W.js} +6 -6
- package/dist/prospect/types.d.ts +584 -1
- package/dist/prospect/types.js +10 -1
- package/dist/{prospect-audit-RKUVSTLN.js → prospect-audit-CIL3NCPM.js} +3 -3
- package/dist/{render-7G43BSK5.js → render-ZUQBJYJR.js} +3 -2
- package/dist/{render-7G43BSK5.js.map → render-ZUQBJYJR.js.map} +1 -1
- package/dist/{render-from-row-P5CWM3MQ.js → render-from-row-IFSBHWDH.js} +3 -3
- package/dist/{renovate-dispatch-6GQWTRKX.js → renovate-dispatch-U5Q52SKT.js} +2 -2
- package/dist/{report-Y4LXDHRV.js → report-6FFHGUUS.js} +14 -14
- package/dist/{report-mirror-6ZDCAQZB.js → report-mirror-IP64R7B3.js} +4 -4
- package/dist/{self-updating-D5XOWS7V.js → self-updating-5DVDIEHB.js} +2 -2
- package/dist/{selftest-IFGSJP4X.js → selftest-LJYXBIO5.js} +6 -6
- package/dist/{site-mirror-3E7JYTJG.js → site-mirror-UKHREAD6.js} +4 -4
- package/dist/{smoke-suite-GXC5JZPH.js → smoke-suite-NUTC7IUK.js} +3 -3
- package/dist/{submissions-UMJS3YST.js → submissions-5GKJPCIH.js} +2 -2
- package/dist/{svelte-codemods-A3EARSRP.js → svelte-codemods-U4NIT6CV.js} +2 -2
- package/dist/{sync-ESP5ZQUU.js → sync-K3PKHVLH.js} +5 -5
- package/dist/{sync-configs-KSESOHFF.js → sync-configs-S2JYY6QU.js} +2 -2
- package/dist/{upgrade-ZLNVEIEA.js → upgrade-YFZCMUPE.js} +2 -2
- package/dist/{websites-D6WYFAES.js → websites-TB4TCAWM.js} +2 -2
- package/dist/{write-audits-to-airtable-K352LZQY.js → write-audits-to-airtable-53EVXKMU.js} +4 -4
- package/package.json +1 -1
- package/dist/airtable-QG5ICI3Y.js +0 -9
- package/dist/chunk-7NYN2P6I.js.map +0 -1
- package/dist/chunk-BJQCIJOY.js.map +0 -1
- package/dist/chunk-MVDMKKBN.js.map +0 -1
- package/dist/chunk-QLIUDTY6.js.map +0 -1
- /package/dist/{airtable-QG5ICI3Y.js.map → airtable-JGCPS7JH.js.map} +0 -0
- /package/dist/{announce-6OS4C57D.js.map → announce-PGT4SZZB.js.map} +0 -0
- /package/dist/{bump-deps-GI7PHMOC.js.map → bump-deps-6OR7LL4E.js.map} +0 -0
- /package/dist/{chunk-4NUKMPOO.js.map → chunk-5FCBYK6L.js.map} +0 -0
- /package/dist/{chunk-PR343K7F.js.map → chunk-5GUJ36TL.js.map} +0 -0
- /package/dist/{chunk-P6AORG32.js.map → chunk-6R56OKJZ.js.map} +0 -0
- /package/dist/{chunk-E22WAIF2.js.map → chunk-AEUABPAJ.js.map} +0 -0
- /package/dist/{chunk-ZRI5VRPG.js.map → chunk-BYGPUUCT.js.map} +0 -0
- /package/dist/{chunk-SKI23ZZH.js.map → chunk-HTABUCF4.js.map} +0 -0
- /package/dist/{chunk-FCS36FDT.js.map → chunk-KQB2BX4V.js.map} +0 -0
- /package/dist/{chunk-AX22DH6S.js.map → chunk-KZRR4NFG.js.map} +0 -0
- /package/dist/{chunk-NDCOOI3V.js.map → chunk-WMBLR5VA.js.map} +0 -0
- /package/dist/{chunk-QBAAZY7W.js.map → chunk-WVNDEYXL.js.map} +0 -0
- /package/dist/{chunk-7BZUOZPU.js.map → chunk-XN37VTDB.js.map} +0 -0
- /package/dist/{chunk-Q47DU255.js.map → chunk-Z2K4JPQ3.js.map} +0 -0
- /package/dist/{convert-to-pnpm-ONFP3JKF.js.map → convert-to-pnpm-3RVGUBBR.js.map} +0 -0
- /package/dist/{db-HSZ6D5DU.js.map → db-FL7ZBF6H.js.map} +0 -0
- /package/dist/{digest-EOA4JLTV.js.map → digest-TZ3QQOSF.js.map} +0 -0
- /package/dist/{digest-collectors-ZA3BTS3R.js.map → digest-collectors-EHFY2OQC.js.map} +0 -0
- /package/dist/{ensure-site-SF6KICTN.js.map → ensure-site-WIDO6LGU.js.map} +0 -0
- /package/dist/{fleet-state-3LQUQHYD.js.map → fleet-state-UEVZ3X3J.js.map} +0 -0
- /package/dist/{forms-notify-target-RMZPKT6S.js.map → forms-notify-target-2LWF5C5B.js.map} +0 -0
- /package/dist/{github-signals-3WLIJ5RR.js.map → github-signals-GAHNOAHS.js.map} +0 -0
- /package/dist/{header-image-DOCDFFRO.js.map → header-image-YN3AQRY3.js.map} +0 -0
- /package/dist/{health-endpoint-DHKJUPRM.js.map → health-endpoint-XLNBWVHK.js.map} +0 -0
- /package/dist/{health-mirror-55RRU6BI.js.map → health-mirror-EB76EERV.js.map} +0 -0
- /package/dist/{import-airtable-OHVY344G.js.map → import-airtable-TAKSN4NV.js.map} +0 -0
- /package/dist/{init-ZRKWYCKA.js.map → init-F5OBBWA5.js.map} +0 -0
- /package/dist/{launch-6X5OXPF2.js.map → launch-SUTMI4NG.js.map} +0 -0
- /package/dist/{lighthouse-airtable-3GUWHETW.js.map → lighthouse-airtable-YMH4UA7R.js.map} +0 -0
- /package/dist/{onboard-6NHGCKGB.js.map → onboard-NKE6GZKL.js.map} +0 -0
- /package/dist/{orchestrate-KMQIUP3H.js.map → orchestrate-MWXENQOS.js.map} +0 -0
- /package/dist/{parity-623WNFZG.js.map → parity-CAJ477PS.js.map} +0 -0
- /package/dist/{pipeline-AQBFG5WM.js.map → pipeline-PRMD4EJT.js.map} +0 -0
- /package/dist/{preflight-VPHQYWTY.js.map → preflight-4VQP7JJL.js.map} +0 -0
- /package/dist/{prismic-ci-A2HI7NNJ.js.map → prismic-ci-QIO3I4KB.js.map} +0 -0
- /package/dist/{prismic-models-QTKIMFBC.js.map → prismic-models-RWDVKZ2W.js.map} +0 -0
- /package/dist/{prospect-audit-RKUVSTLN.js.map → prospect-audit-CIL3NCPM.js.map} +0 -0
- /package/dist/{render-from-row-P5CWM3MQ.js.map → render-from-row-IFSBHWDH.js.map} +0 -0
- /package/dist/{renovate-dispatch-6GQWTRKX.js.map → renovate-dispatch-U5Q52SKT.js.map} +0 -0
- /package/dist/{report-Y4LXDHRV.js.map → report-6FFHGUUS.js.map} +0 -0
- /package/dist/{report-mirror-6ZDCAQZB.js.map → report-mirror-IP64R7B3.js.map} +0 -0
- /package/dist/{self-updating-D5XOWS7V.js.map → self-updating-5DVDIEHB.js.map} +0 -0
- /package/dist/{selftest-IFGSJP4X.js.map → selftest-LJYXBIO5.js.map} +0 -0
- /package/dist/{site-mirror-3E7JYTJG.js.map → site-mirror-UKHREAD6.js.map} +0 -0
- /package/dist/{smoke-suite-GXC5JZPH.js.map → smoke-suite-NUTC7IUK.js.map} +0 -0
- /package/dist/{submissions-UMJS3YST.js.map → submissions-5GKJPCIH.js.map} +0 -0
- /package/dist/{svelte-codemods-A3EARSRP.js.map → svelte-codemods-U4NIT6CV.js.map} +0 -0
- /package/dist/{sync-ESP5ZQUU.js.map → sync-K3PKHVLH.js.map} +0 -0
- /package/dist/{sync-configs-KSESOHFF.js.map → sync-configs-S2JYY6QU.js.map} +0 -0
- /package/dist/{upgrade-ZLNVEIEA.js.map → upgrade-YFZCMUPE.js.map} +0 -0
- /package/dist/{websites-D6WYFAES.js.map → websites-TB4TCAWM.js.map} +0 -0
- /package/dist/{write-audits-to-airtable-K352LZQY.js.map → write-audits-to-airtable-53EVXKMU.js.map} +0 -0
|
@@ -0,0 +1,4194 @@
|
|
|
1
|
+
import {
|
|
2
|
+
isPrivateOrLoopbackHost
|
|
3
|
+
} from "./chunk-RARREDQE.js";
|
|
4
|
+
|
|
5
|
+
// src/prospect/accessibility.ts
|
|
6
|
+
var MAX_REPORTED_RULES = 12;
|
|
7
|
+
var IMPACT_ORDER = { critical: 0, serious: 1, moderate: 2, minor: 3 };
|
|
8
|
+
function summarizeAccessibility(pages) {
|
|
9
|
+
const withResults = pages.filter((p) => p.axe != null);
|
|
10
|
+
if (withResults.length === 0) {
|
|
11
|
+
return {
|
|
12
|
+
measured: false,
|
|
13
|
+
pagesExamined: 0,
|
|
14
|
+
rulesPassed: 0,
|
|
15
|
+
rulesIncomplete: 0,
|
|
16
|
+
incompleteIds: [],
|
|
17
|
+
rulesInapplicable: 0,
|
|
18
|
+
violations: [],
|
|
19
|
+
violationsTotal: 0
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
const byRule = /* @__PURE__ */ new Map();
|
|
23
|
+
let passes = 0;
|
|
24
|
+
let incomplete = 0;
|
|
25
|
+
const undecided = /* @__PURE__ */ new Set();
|
|
26
|
+
let inapplicable = 0;
|
|
27
|
+
for (const page of withResults) {
|
|
28
|
+
const axe = page.axe;
|
|
29
|
+
passes = Math.max(passes, axe.passes);
|
|
30
|
+
incomplete = Math.max(incomplete, axe.incomplete);
|
|
31
|
+
for (const id of axe.incompleteIds ?? []) undecided.add(id);
|
|
32
|
+
inapplicable = Math.max(inapplicable, axe.inapplicable ?? 0);
|
|
33
|
+
for (const v of axe.violations) {
|
|
34
|
+
const existing = byRule.get(v.id);
|
|
35
|
+
if (existing) {
|
|
36
|
+
existing.nodes += v.nodes;
|
|
37
|
+
if (!existing.pages.includes(page.url)) existing.pages.push(page.url);
|
|
38
|
+
existing.sample ??= v.sample;
|
|
39
|
+
} else {
|
|
40
|
+
byRule.set(v.id, { ...v, pages: [page.url] });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
const all = [...byRule.values()].sort(
|
|
45
|
+
(a, b) => (IMPACT_ORDER[a.impact ?? "minor"] ?? 4) - (IMPACT_ORDER[b.impact ?? "minor"] ?? 4) || b.nodes - a.nodes || a.id.localeCompare(b.id)
|
|
46
|
+
);
|
|
47
|
+
return {
|
|
48
|
+
measured: true,
|
|
49
|
+
pagesExamined: withResults.length,
|
|
50
|
+
rulesPassed: passes,
|
|
51
|
+
// The UNION when we have the names, the per-page max otherwise.
|
|
52
|
+
//
|
|
53
|
+
// `Math.max` was only ever a proxy for "how many distinct rules", chosen
|
|
54
|
+
// because summing would have counted the same rule once per page. Now that
|
|
55
|
+
// the ids are collected, the distinct count is known exactly — and the two
|
|
56
|
+
// disagreed on our own site, where the report said "3 rules need a human"
|
|
57
|
+
// and then listed four. A reader who can count is a reader who stops
|
|
58
|
+
// trusting the numbers.
|
|
59
|
+
rulesIncomplete: undecided.size > 0 ? undecided.size : incomplete,
|
|
60
|
+
incompleteIds: [...undecided].sort(),
|
|
61
|
+
rulesInapplicable: inapplicable,
|
|
62
|
+
violations: all.slice(0, MAX_REPORTED_RULES),
|
|
63
|
+
violationsTotal: all.length
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
var MAX_CONSOLE_ERRORS = 8;
|
|
67
|
+
var MAX_FAILED_REQUESTS = 12;
|
|
68
|
+
var MAX_OVERSIZED_IMAGES = 6;
|
|
69
|
+
|
|
70
|
+
// src/prospect/crawl.ts
|
|
71
|
+
import { parse as parse2, NodeType as NodeType2 } from "node-html-parser";
|
|
72
|
+
|
|
73
|
+
// src/prospect/pages.ts
|
|
74
|
+
function fetchedOk(page) {
|
|
75
|
+
return page.status !== null && page.status < 400;
|
|
76
|
+
}
|
|
77
|
+
var INFRA_SEGMENTS = [/^cdn-cgi$/i, /^wp-admin$/i, /^wp-json$/i, /^xmlrpc\.php$/i];
|
|
78
|
+
function isInfraPath(url) {
|
|
79
|
+
let path;
|
|
80
|
+
try {
|
|
81
|
+
path = new URL(url).pathname;
|
|
82
|
+
} catch {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
return path.split("/").filter(Boolean).some((segment) => INFRA_SEGMENTS.some((re) => re.test(segment)));
|
|
86
|
+
}
|
|
87
|
+
function usablePages(pages) {
|
|
88
|
+
const candidates = pages.filter((p) => fetchedOk(p) && !isInfraPath(p.url));
|
|
89
|
+
const withRendered = candidates.filter((p) => p.rendered !== null);
|
|
90
|
+
const withRaw = candidates.filter((p) => p.raw !== null);
|
|
91
|
+
const view = withRendered.length >= withRaw.length ? "rendered" : "raw";
|
|
92
|
+
const kept = view === "rendered" ? withRendered : withRaw;
|
|
93
|
+
const usable = [];
|
|
94
|
+
for (const page of kept) {
|
|
95
|
+
const extract = view === "rendered" ? page.rendered : page.raw;
|
|
96
|
+
if (extract) usable.push({ page, extract });
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
pages: usable,
|
|
100
|
+
view,
|
|
101
|
+
excluded: candidates.length - usable.length,
|
|
102
|
+
anchorsMeasured: usable.length > 0 && usable.every((p) => p.extract.anchors !== void 0)
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// src/prospect/interaction.ts
|
|
107
|
+
var ERROR_WORDS = /\b(required|please\s|cannot be blank|can't be blank|must be|enter a|fill in|invalid|not valid|is not a valid)\b/i;
|
|
108
|
+
var FILLER = {
|
|
109
|
+
text: "Audit Test",
|
|
110
|
+
search: "Audit Test",
|
|
111
|
+
tel: "2085550142",
|
|
112
|
+
url: "https://example.com",
|
|
113
|
+
number: "1",
|
|
114
|
+
date: "2030-01-01",
|
|
115
|
+
textarea: "Checking that this form validates what it is given."
|
|
116
|
+
};
|
|
117
|
+
var INVALID_EMAIL = "not-an-email";
|
|
118
|
+
var SENTINELS = [INVALID_EMAIL, ...Object.values(FILLER)];
|
|
119
|
+
var REMOVE_MARKER = `() => {
|
|
120
|
+
document.querySelector("[data-audit-form]")?.removeAttribute("data-audit-form");
|
|
121
|
+
}`;
|
|
122
|
+
var CHOOSE_FORM = `() => {
|
|
123
|
+
const forms = Array.from(document.querySelectorAll("form"));
|
|
124
|
+
let best = null;
|
|
125
|
+
let bestScore = 0;
|
|
126
|
+
for (const form of forms) {
|
|
127
|
+
const controls = Array.from(form.querySelectorAll("input, textarea, select"));
|
|
128
|
+
const typed = (el) => (el.getAttribute("type") || el.tagName.toLowerCase()).toLowerCase();
|
|
129
|
+
const visible = controls.filter((el) => {
|
|
130
|
+
const t = typed(el);
|
|
131
|
+
if (t === "hidden" || t === "submit" || t === "button" || t === "reset") return false;
|
|
132
|
+
const r = el.getBoundingClientRect();
|
|
133
|
+
return r.width > 0 && r.height > 0;
|
|
134
|
+
});
|
|
135
|
+
if (visible.length === 0) continue;
|
|
136
|
+
|
|
137
|
+
const attrs = (el) =>
|
|
138
|
+
[el.getAttribute("name"), el.getAttribute("id"), el.getAttribute("autocomplete"),
|
|
139
|
+
el.getAttribute("placeholder"), el.getAttribute("aria-label")]
|
|
140
|
+
.filter(Boolean).join(" ").toLowerCase();
|
|
141
|
+
const hasEmail = visible.some(
|
|
142
|
+
(el) => typed(el) === "email" || /\\bemail\\b|e-mail/.test(attrs(el)),
|
|
143
|
+
);
|
|
144
|
+
const hasMessage = visible.some((el) => el.tagName === "TEXTAREA");
|
|
145
|
+
const action = (form.getAttribute("action") || "").toLowerCase();
|
|
146
|
+
const looksLikeSearch =
|
|
147
|
+
visible.length === 1 &&
|
|
148
|
+
(visible.some((el) => typed(el) === "search") || /search|\\/s\\b|\\?q=/.test(action) ||
|
|
149
|
+
/search|query/.test(attrs(visible[0])));
|
|
150
|
+
if (looksLikeSearch) continue;
|
|
151
|
+
// A password field means this is a login, not an enquiry. Interacting with
|
|
152
|
+
// one is both useless and rude.
|
|
153
|
+
if (visible.some((el) => typed(el) === "password")) continue;
|
|
154
|
+
|
|
155
|
+
const submit = form.querySelector(
|
|
156
|
+
'button[type="submit"], input[type="submit"], button:not([type]), button[type="button"]',
|
|
157
|
+
);
|
|
158
|
+
if (!submit) continue;
|
|
159
|
+
|
|
160
|
+
let score = visible.length;
|
|
161
|
+
if (hasEmail) score += 4;
|
|
162
|
+
if (hasMessage) score += 3;
|
|
163
|
+
if (score > bestScore) {
|
|
164
|
+
bestScore = score;
|
|
165
|
+
best = { form, hasEmail, fields: visible.length, action };
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (!best) return null;
|
|
169
|
+
best.form.setAttribute("data-audit-form", "");
|
|
170
|
+
// TARGET REMOVED, and this is a safety measure rather than a tidy-up.
|
|
171
|
+
// A form posting to target="popupwindow" \u2014 which is how Mailchimp's embed
|
|
172
|
+
// ships, and how clearleft.com's only form is written \u2014 submits into a NEW
|
|
173
|
+
// page, and a route registered on this one does not see it. Measured: the
|
|
174
|
+
// request went out and the server received it. Without the target the
|
|
175
|
+
// submission is an ordinary main-frame navigation, which the interceptor
|
|
176
|
+
// already stops. The page is abandoned straight after the probe, so nothing
|
|
177
|
+
// downstream sees the altered form.
|
|
178
|
+
best.form.removeAttribute("target");
|
|
179
|
+
return { fields: best.fields, hasEmail: best.hasEmail, action: best.action };
|
|
180
|
+
}`;
|
|
181
|
+
var FILL_FORM = `(args) => {
|
|
182
|
+
const form = document.querySelector("[data-audit-form]");
|
|
183
|
+
if (!form) return false;
|
|
184
|
+
const controls = Array.from(form.querySelectorAll("input, textarea, select"));
|
|
185
|
+
const fire = (el) => {
|
|
186
|
+
el.dispatchEvent(new Event("input", { bubbles: true }));
|
|
187
|
+
el.dispatchEvent(new Event("change", { bubbles: true }));
|
|
188
|
+
};
|
|
189
|
+
for (const el of controls) {
|
|
190
|
+
const type = (el.getAttribute("type") || el.tagName.toLowerCase()).toLowerCase();
|
|
191
|
+
if (type === "hidden" || type === "submit" || type === "button" || type === "reset") continue;
|
|
192
|
+
const attrs = [el.getAttribute("name"), el.getAttribute("id"), el.getAttribute("autocomplete"),
|
|
193
|
+
el.getAttribute("placeholder")].filter(Boolean).join(" ").toLowerCase();
|
|
194
|
+
const isEmail = type === "email" || /\\bemail\\b|e-mail/.test(attrs);
|
|
195
|
+
if (el.tagName === "SELECT") {
|
|
196
|
+
if (el.options.length > 1) { el.selectedIndex = 1; fire(el); }
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
if (type === "checkbox" || type === "radio") {
|
|
200
|
+
if (!el.checked) { el.checked = true; fire(el); }
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
el.value = isEmail ? args.invalidEmail : (args.filler[type] || args.filler.text);
|
|
204
|
+
fire(el);
|
|
205
|
+
}
|
|
206
|
+
return true;
|
|
207
|
+
}`;
|
|
208
|
+
var READ_STATE = `(args) => {
|
|
209
|
+
const form = document.querySelector("[data-audit-form]");
|
|
210
|
+
if (!form) return null;
|
|
211
|
+
const nativeInvalid = typeof form.checkValidity === "function" ? !form.checkValidity() : false;
|
|
212
|
+
const flagged = form.querySelectorAll('[aria-invalid="true"], .error, .is-invalid, .invalid').length;
|
|
213
|
+
const alerts = Array.from(form.querySelectorAll('[role="alert"], [aria-live]'))
|
|
214
|
+
.map((el) => (el.textContent || "").trim())
|
|
215
|
+
.filter(Boolean)
|
|
216
|
+
.join(" ");
|
|
217
|
+
const text = (form.textContent || "").replace(/\\s+/g, " ").trim();
|
|
218
|
+
return {
|
|
219
|
+
nativeInvalid,
|
|
220
|
+
flagged,
|
|
221
|
+
hasErrorText: new RegExp(args.words, "i").test(alerts + " " + text),
|
|
222
|
+
};
|
|
223
|
+
}`;
|
|
224
|
+
var VALIDATION_SETTLE_MS = 600;
|
|
225
|
+
var PROBE_BUDGET_MS = 45e3;
|
|
226
|
+
function withTimeout(work, ms) {
|
|
227
|
+
let timer;
|
|
228
|
+
return Promise.race([
|
|
229
|
+
work.catch(() => null),
|
|
230
|
+
new Promise((resolve) => {
|
|
231
|
+
timer = setTimeout(() => resolve(null), ms);
|
|
232
|
+
})
|
|
233
|
+
]).finally(() => clearTimeout(timer));
|
|
234
|
+
}
|
|
235
|
+
async function probeForms(deps) {
|
|
236
|
+
const chosen = await deps.evaluate(CHOOSE_FORM);
|
|
237
|
+
if (!chosen) return null;
|
|
238
|
+
const guard = await deps.intercept();
|
|
239
|
+
const probe = {
|
|
240
|
+
url: deps.url,
|
|
241
|
+
emptyRefused: void 0,
|
|
242
|
+
emptyHow: null,
|
|
243
|
+
invalidEmailRefused: chosen.hasEmail ? void 0 : null,
|
|
244
|
+
invalidEmailHow: null,
|
|
245
|
+
blocked: 0
|
|
246
|
+
};
|
|
247
|
+
try {
|
|
248
|
+
const before = await deps.evaluate(READ_STATE, { words: ERROR_WORDS.source });
|
|
249
|
+
if (before?.nativeInvalid) {
|
|
250
|
+
probe.emptyRefused = true;
|
|
251
|
+
probe.emptyHow = "the browser blocks it \u2014 the form marks its fields required";
|
|
252
|
+
} else {
|
|
253
|
+
const beforeBlocked = guard.blocked();
|
|
254
|
+
await deps.clickSubmit();
|
|
255
|
+
await deps.settle(VALIDATION_SETTLE_MS);
|
|
256
|
+
const after = await deps.evaluate(READ_STATE, { words: ERROR_WORDS.source });
|
|
257
|
+
const attempted = guard.blocked() > beforeBlocked;
|
|
258
|
+
const complained = after !== null && (after.hasErrorText || after.flagged > (before?.flagged ?? 0) || after.nativeInvalid);
|
|
259
|
+
if (complained) {
|
|
260
|
+
probe.emptyRefused = true;
|
|
261
|
+
probe.emptyHow = "the form showed an error rather than submitting";
|
|
262
|
+
} else if (attempted) {
|
|
263
|
+
probe.emptyRefused = false;
|
|
264
|
+
probe.emptyHow = "the form submitted with every field empty \u2014 we stopped the request before it left the browser";
|
|
265
|
+
} else if (after === null) {
|
|
266
|
+
probe.emptyRefused = void 0;
|
|
267
|
+
} else {
|
|
268
|
+
probe.emptyRefused = false;
|
|
269
|
+
probe.emptyHow = "the button did nothing at all \u2014 no error, and nothing sent";
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
if (chosen.hasEmail) {
|
|
273
|
+
const back = await guard.reload();
|
|
274
|
+
const filled = back ? await deps.evaluate(FILL_FORM, {
|
|
275
|
+
invalidEmail: INVALID_EMAIL,
|
|
276
|
+
filler: FILLER
|
|
277
|
+
}) : false;
|
|
278
|
+
if (!filled) {
|
|
279
|
+
probe.invalidEmailRefused = void 0;
|
|
280
|
+
} else {
|
|
281
|
+
const state = await deps.evaluate(READ_STATE, { words: ERROR_WORDS.source });
|
|
282
|
+
if (state?.nativeInvalid) {
|
|
283
|
+
probe.invalidEmailRefused = true;
|
|
284
|
+
probe.invalidEmailHow = "the browser catches it \u2014 the field is typed as an email";
|
|
285
|
+
} else {
|
|
286
|
+
const beforeBlocked = guard.blocked();
|
|
287
|
+
await deps.clickSubmit();
|
|
288
|
+
await deps.settle(VALIDATION_SETTLE_MS);
|
|
289
|
+
const after = await deps.evaluate(READ_STATE, { words: ERROR_WORDS.source });
|
|
290
|
+
const attempted = guard.blocked() > beforeBlocked;
|
|
291
|
+
if (after && (after.hasErrorText || after.nativeInvalid || after.flagged > 0)) {
|
|
292
|
+
probe.invalidEmailRefused = true;
|
|
293
|
+
probe.invalidEmailHow = "the form showed an error rather than submitting";
|
|
294
|
+
} else if (attempted) {
|
|
295
|
+
probe.invalidEmailRefused = false;
|
|
296
|
+
probe.invalidEmailHow = `the form accepted \u201C${INVALID_EMAIL}\u201D as an email address \u2014 we stopped the request before it left the browser`;
|
|
297
|
+
} else {
|
|
298
|
+
probe.invalidEmailRefused = void 0;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
} catch {
|
|
304
|
+
} finally {
|
|
305
|
+
probe.blocked = guard.stopped();
|
|
306
|
+
await guard.release();
|
|
307
|
+
}
|
|
308
|
+
return probe;
|
|
309
|
+
}
|
|
310
|
+
function pageInteractionDeps(page, url) {
|
|
311
|
+
return {
|
|
312
|
+
url,
|
|
313
|
+
settle: (ms) => page.waitForTimeout(ms),
|
|
314
|
+
// Playwright evaluates a STRING as an expression and does not call it —
|
|
315
|
+
// `page.evaluate("() => 1 + 1")` yields a function object, which serialises
|
|
316
|
+
// to undefined, so every snippet here silently returned nothing. The call
|
|
317
|
+
// has to be written out, with the argument inlined as JSON (every argument
|
|
318
|
+
// in this file is our own constant, never anything off the page).
|
|
319
|
+
evaluate: (fn, arg) => page.evaluate(`(${fn})(${JSON.stringify(arg ?? null)})`),
|
|
320
|
+
async clickSubmit() {
|
|
321
|
+
const submit = page.locator(
|
|
322
|
+
'[data-audit-form] button[type="submit"], [data-audit-form] input[type="submit"], [data-audit-form] button:not([type]), [data-audit-form] button[type="button"]'
|
|
323
|
+
).first();
|
|
324
|
+
await submit.click({ timeout: 5e3, force: true, noWaitAfter: true });
|
|
325
|
+
},
|
|
326
|
+
async intercept() {
|
|
327
|
+
let stopped = 0;
|
|
328
|
+
let submissions = 0;
|
|
329
|
+
let allowNavigation = false;
|
|
330
|
+
let armed = true;
|
|
331
|
+
let origin = "";
|
|
332
|
+
try {
|
|
333
|
+
origin = new URL(url).origin;
|
|
334
|
+
} catch {
|
|
335
|
+
origin = "";
|
|
336
|
+
}
|
|
337
|
+
const onDialog = (d) => void d.dismiss().catch(() => {
|
|
338
|
+
});
|
|
339
|
+
page.on("dialog", onDialog);
|
|
340
|
+
const router = page.context();
|
|
341
|
+
await router.route("**/*", (route) => {
|
|
342
|
+
if (!armed) {
|
|
343
|
+
void route.continue().catch(() => {
|
|
344
|
+
});
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
let stop;
|
|
348
|
+
let looksLikeSubmission = false;
|
|
349
|
+
try {
|
|
350
|
+
const req = route.request();
|
|
351
|
+
const isNav = req.isNavigationRequest();
|
|
352
|
+
let frame = null;
|
|
353
|
+
try {
|
|
354
|
+
frame = req.frame();
|
|
355
|
+
} catch {
|
|
356
|
+
frame = null;
|
|
357
|
+
}
|
|
358
|
+
const isPopupNav = isNav && frame === null;
|
|
359
|
+
const isNavigation = isNav && frame === page.mainFrame();
|
|
360
|
+
const method = req.method().toUpperCase();
|
|
361
|
+
const sameOrigin = (() => {
|
|
362
|
+
try {
|
|
363
|
+
return new URL(req.url()).origin === origin;
|
|
364
|
+
} catch {
|
|
365
|
+
return false;
|
|
366
|
+
}
|
|
367
|
+
})();
|
|
368
|
+
stop = isPopupNav || isNavigation && !allowNavigation || method !== "GET";
|
|
369
|
+
const contentType = (req.headers()["content-type"] ?? "").toLowerCase();
|
|
370
|
+
const formEncoded = contentType.includes("application/x-www-form-urlencoded") || contentType.includes("multipart/form-data");
|
|
371
|
+
const carriesOurs = (() => {
|
|
372
|
+
try {
|
|
373
|
+
const body = req.postData() ?? "";
|
|
374
|
+
if (body === "") return false;
|
|
375
|
+
return SENTINELS.some(
|
|
376
|
+
(v) => body.includes(v) || body.includes(encodeURIComponent(v))
|
|
377
|
+
);
|
|
378
|
+
} catch {
|
|
379
|
+
return false;
|
|
380
|
+
}
|
|
381
|
+
})();
|
|
382
|
+
looksLikeSubmission = isPopupNav || isNavigation && !allowNavigation || method !== "GET" && (sameOrigin || formEncoded || carriesOurs);
|
|
383
|
+
} catch {
|
|
384
|
+
stop = false;
|
|
385
|
+
}
|
|
386
|
+
if (stop) stopped++;
|
|
387
|
+
if (stop && looksLikeSubmission) submissions++;
|
|
388
|
+
void (stop ? route.abort() : route.continue()).catch(() => {
|
|
389
|
+
});
|
|
390
|
+
});
|
|
391
|
+
return {
|
|
392
|
+
blocked: () => submissions,
|
|
393
|
+
stopped: () => stopped,
|
|
394
|
+
reload: async () => {
|
|
395
|
+
allowNavigation = true;
|
|
396
|
+
try {
|
|
397
|
+
await page.goto(url, { waitUntil: "load", timeout: 2e4 });
|
|
398
|
+
} catch {
|
|
399
|
+
return false;
|
|
400
|
+
} finally {
|
|
401
|
+
allowNavigation = false;
|
|
402
|
+
}
|
|
403
|
+
const again = await page.evaluate(`(${CHOOSE_FORM})(null)`).catch(() => null);
|
|
404
|
+
return again !== null;
|
|
405
|
+
},
|
|
406
|
+
release: async () => {
|
|
407
|
+
armed = false;
|
|
408
|
+
page.off("dialog", onDialog);
|
|
409
|
+
await withTimeout(page.evaluate(`(${REMOVE_MARKER})()`), 5e3);
|
|
410
|
+
await withTimeout(router.unrouteAll({ behavior: "ignoreErrors" }), 3e3);
|
|
411
|
+
}
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
// src/prospect/extract.ts
|
|
418
|
+
import { parse, NodeType } from "node-html-parser";
|
|
419
|
+
var MAX_ANCHORS = 300;
|
|
420
|
+
var MAX_SCRIPTS = 120;
|
|
421
|
+
var MAX_INLINE_URLS = 40;
|
|
422
|
+
var NAMESPACE_URLS = /^https?:\/\/(www\.)?w3\.org\//i;
|
|
423
|
+
function urlsIn(code) {
|
|
424
|
+
const found = /* @__PURE__ */ new Set();
|
|
425
|
+
for (const m of code.matchAll(/https?:\/\/[^\s"'`\\)<>]{4,200}/gi)) {
|
|
426
|
+
if (NAMESPACE_URLS.test(m[0])) continue;
|
|
427
|
+
found.add(m[0]);
|
|
428
|
+
if (found.size >= MAX_INLINE_URLS) break;
|
|
429
|
+
}
|
|
430
|
+
return [...found];
|
|
431
|
+
}
|
|
432
|
+
var NON_FIELD_INPUTS = /* @__PURE__ */ new Set(["hidden", "submit", "button", "image", "reset"]);
|
|
433
|
+
var CONTACT_FIELD = /\b(e-?mail|phone|tel|mobile|contact)\b/i;
|
|
434
|
+
var toWords = (s) => s.replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/[_\-.]+/g, " ");
|
|
435
|
+
var UNRENDERED_TAGS = /* @__PURE__ */ new Set(["STYLE", "NOSCRIPT", "TEMPLATE", "SVG"]);
|
|
436
|
+
var BLOCK = /* @__PURE__ */ new Set([
|
|
437
|
+
"ADDRESS",
|
|
438
|
+
"ARTICLE",
|
|
439
|
+
"ASIDE",
|
|
440
|
+
"BLOCKQUOTE",
|
|
441
|
+
"BR",
|
|
442
|
+
"DD",
|
|
443
|
+
"DIV",
|
|
444
|
+
"DL",
|
|
445
|
+
"DT",
|
|
446
|
+
"FIELDSET",
|
|
447
|
+
"FIGCAPTION",
|
|
448
|
+
"FIGURE",
|
|
449
|
+
"FOOTER",
|
|
450
|
+
"FORM",
|
|
451
|
+
"H1",
|
|
452
|
+
"H2",
|
|
453
|
+
"H3",
|
|
454
|
+
"H4",
|
|
455
|
+
"H5",
|
|
456
|
+
"H6",
|
|
457
|
+
"HEADER",
|
|
458
|
+
"HR",
|
|
459
|
+
"LI",
|
|
460
|
+
"MAIN",
|
|
461
|
+
"NAV",
|
|
462
|
+
"OL",
|
|
463
|
+
"P",
|
|
464
|
+
"PRE",
|
|
465
|
+
"SECTION",
|
|
466
|
+
"TABLE",
|
|
467
|
+
"TD",
|
|
468
|
+
"TH",
|
|
469
|
+
"TR",
|
|
470
|
+
"UL"
|
|
471
|
+
]);
|
|
472
|
+
var collapse = (s) => s.replace(/\s+/g, " ").trim();
|
|
473
|
+
var MAX_WALK_DEPTH = 100;
|
|
474
|
+
function headingTextOf(el) {
|
|
475
|
+
const label = (el.getAttribute("aria-label") ?? "").trim();
|
|
476
|
+
if (label) return collapse(label);
|
|
477
|
+
const text = textOf(el);
|
|
478
|
+
if (text) return text;
|
|
479
|
+
const alts = el.querySelectorAll("img").map((img) => (img.getAttribute("alt") ?? "").trim()).filter((a) => a.length > 0);
|
|
480
|
+
return collapse(alts.join(" "));
|
|
481
|
+
}
|
|
482
|
+
function textOf(el) {
|
|
483
|
+
const parts = [];
|
|
484
|
+
const walk = (node, depth) => {
|
|
485
|
+
if (depth > MAX_WALK_DEPTH) return;
|
|
486
|
+
for (const child of node.childNodes) {
|
|
487
|
+
if (child.nodeType === NodeType.TEXT_NODE) {
|
|
488
|
+
parts.push(child.text);
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
if (child.nodeType !== NodeType.ELEMENT_NODE) continue;
|
|
492
|
+
const e = child;
|
|
493
|
+
const tag = e.tagName;
|
|
494
|
+
if (UNRENDERED_TAGS.has(tag) || tag === "SCRIPT" || tag === "TITLE") continue;
|
|
495
|
+
const block = BLOCK.has(tag);
|
|
496
|
+
if (block) parts.push("\n");
|
|
497
|
+
walk(e, depth + 1);
|
|
498
|
+
if (block) parts.push("\n");
|
|
499
|
+
}
|
|
500
|
+
};
|
|
501
|
+
walk(el, 0);
|
|
502
|
+
return collapse(parts.join(""));
|
|
503
|
+
}
|
|
504
|
+
function collect(el, out, depth = 0) {
|
|
505
|
+
if (depth > MAX_WALK_DEPTH) return;
|
|
506
|
+
for (const child of el.childNodes) {
|
|
507
|
+
if (child.nodeType !== NodeType.ELEMENT_NODE) continue;
|
|
508
|
+
const e = child;
|
|
509
|
+
const tag = e.tagName;
|
|
510
|
+
if (UNRENDERED_TAGS.has(tag)) continue;
|
|
511
|
+
switch (tag) {
|
|
512
|
+
case "META":
|
|
513
|
+
out.metas.push(e);
|
|
514
|
+
break;
|
|
515
|
+
case "LINK":
|
|
516
|
+
out.links.push(e);
|
|
517
|
+
break;
|
|
518
|
+
case "IMG":
|
|
519
|
+
out.images.push(e);
|
|
520
|
+
break;
|
|
521
|
+
case "A":
|
|
522
|
+
if ((e.getAttribute("href") ?? "").trim()) out.anchors.push(e);
|
|
523
|
+
break;
|
|
524
|
+
case "FORM":
|
|
525
|
+
out.forms.push(e);
|
|
526
|
+
break;
|
|
527
|
+
case "TITLE":
|
|
528
|
+
if (out.title === null) out.title = collapse(e.text) || null;
|
|
529
|
+
break;
|
|
530
|
+
case "SCRIPT": {
|
|
531
|
+
const isJsonLd = (e.getAttribute("type") ?? "").toLowerCase().trim() === "application/ld+json";
|
|
532
|
+
if (isJsonLd) out.jsonLd.push(e.text);
|
|
533
|
+
const src = (e.getAttribute("src") ?? "").trim();
|
|
534
|
+
if (src) {
|
|
535
|
+
out.scriptSrcs.push(src);
|
|
536
|
+
} else if (isJsonLd) {
|
|
537
|
+
} else {
|
|
538
|
+
for (const url of urlsIn(e.text)) out.inlineScriptUrls.push(url);
|
|
539
|
+
}
|
|
540
|
+
continue;
|
|
541
|
+
}
|
|
542
|
+
case "H1":
|
|
543
|
+
case "H2":
|
|
544
|
+
case "H3":
|
|
545
|
+
case "H4":
|
|
546
|
+
case "H5":
|
|
547
|
+
case "H6": {
|
|
548
|
+
const text = headingTextOf(e);
|
|
549
|
+
if (text) out.headings.push({ level: Number(tag.slice(1)), text });
|
|
550
|
+
break;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
collect(e, out, depth + 1);
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
var NON_ENQUIRY_SEGMENTS = /* @__PURE__ */ new Set([
|
|
557
|
+
// Account doors
|
|
558
|
+
"login",
|
|
559
|
+
"log-in",
|
|
560
|
+
"log_in",
|
|
561
|
+
"logon",
|
|
562
|
+
"signin",
|
|
563
|
+
"sign-in",
|
|
564
|
+
"sign_in",
|
|
565
|
+
"signup",
|
|
566
|
+
"sign-up",
|
|
567
|
+
"sign_up",
|
|
568
|
+
"register",
|
|
569
|
+
"registration",
|
|
570
|
+
// Commerce
|
|
571
|
+
"checkout",
|
|
572
|
+
"cart",
|
|
573
|
+
"basket",
|
|
574
|
+
// Retrieval, not contact
|
|
575
|
+
"search",
|
|
576
|
+
// WordPress, whose comment endpoint is a fixed path on a great many sites
|
|
577
|
+
"wp-login",
|
|
578
|
+
"wp-signup",
|
|
579
|
+
"wp-register",
|
|
580
|
+
"wp-comments-post"
|
|
581
|
+
]);
|
|
582
|
+
function isNonEnquiryAction(action) {
|
|
583
|
+
if (!action) return false;
|
|
584
|
+
let path;
|
|
585
|
+
try {
|
|
586
|
+
path = new URL(action, "https://form.invalid/").pathname;
|
|
587
|
+
} catch {
|
|
588
|
+
path = action;
|
|
589
|
+
}
|
|
590
|
+
return path.toLowerCase().split("/").some((segment) => NON_ENQUIRY_SEGMENTS.has(segment.replace(/\.[a-z0-9]+$/, "")));
|
|
591
|
+
}
|
|
592
|
+
function formShape(form) {
|
|
593
|
+
const controls = form.querySelectorAll("input, textarea, select");
|
|
594
|
+
let fieldCount = 0;
|
|
595
|
+
let contactFieldCount = 0;
|
|
596
|
+
let hasContactField = false;
|
|
597
|
+
let hasPassword = false;
|
|
598
|
+
let hasTextarea = false;
|
|
599
|
+
let hasSubmit = form.querySelectorAll("button").length > 0;
|
|
600
|
+
const fields = [];
|
|
601
|
+
for (const control of controls) {
|
|
602
|
+
const type = (control.getAttribute("type") ?? "").toLowerCase().trim();
|
|
603
|
+
if (control.tagName === "INPUT" && NON_FIELD_INPUTS.has(type)) {
|
|
604
|
+
if (type === "submit" || type === "image") hasSubmit = true;
|
|
605
|
+
continue;
|
|
606
|
+
}
|
|
607
|
+
if (control.tagName === "TEXTAREA") hasTextarea = true;
|
|
608
|
+
if (control.tagName === "INPUT" && type === "password") hasPassword = true;
|
|
609
|
+
fieldCount += 1;
|
|
610
|
+
fields.push({
|
|
611
|
+
// A bare <input> defaults to text, exactly as a browser treats it, so an
|
|
612
|
+
// author who wrote nothing reads the same as one who wrote type="text".
|
|
613
|
+
type: control.tagName === "INPUT" ? type || "text" : control.tagName.toLowerCase(),
|
|
614
|
+
name: (control.getAttribute("name") ?? "").toLowerCase().trim() || null,
|
|
615
|
+
autocomplete: (control.getAttribute("autocomplete") ?? "").toLowerCase().trim() || null,
|
|
616
|
+
required: control.hasAttribute("required")
|
|
617
|
+
});
|
|
618
|
+
const signature = toWords(
|
|
619
|
+
[
|
|
620
|
+
type,
|
|
621
|
+
control.getAttribute("name") ?? "",
|
|
622
|
+
control.getAttribute("id") ?? "",
|
|
623
|
+
control.getAttribute("placeholder") ?? "",
|
|
624
|
+
control.getAttribute("autocomplete") ?? "",
|
|
625
|
+
control.getAttribute("aria-label") ?? ""
|
|
626
|
+
].join(" ")
|
|
627
|
+
);
|
|
628
|
+
if (type === "email" || type === "tel" || CONTACT_FIELD.test(signature)) {
|
|
629
|
+
hasContactField = true;
|
|
630
|
+
contactFieldCount += 1;
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
const action = form.getAttribute("action")?.trim() || null;
|
|
634
|
+
const isAccountForm = hasPassword || isNonEnquiryAction(action);
|
|
635
|
+
const asksSomethingBack = hasTextarea || fieldCount - contactFieldCount >= 2;
|
|
636
|
+
return {
|
|
637
|
+
// A lone contact field is a newsletter box, not an enquiry form. See
|
|
638
|
+
// `FormKind`: on one audited site a footer email box put every page at zero
|
|
639
|
+
// clicks from "reaching them", when the only form that reaches a person
|
|
640
|
+
// was the nine-field one on its contact page.
|
|
641
|
+
kind: isAccountForm || !hasContactField ? "other" : fieldCount >= 2 && asksSomethingBack ? "enquiry" : "subscribe",
|
|
642
|
+
action,
|
|
643
|
+
method: (form.getAttribute("method") ?? "get").toLowerCase().trim() || "get",
|
|
644
|
+
fieldCount,
|
|
645
|
+
hasContactField,
|
|
646
|
+
hasSubmit,
|
|
647
|
+
fields
|
|
648
|
+
};
|
|
649
|
+
}
|
|
650
|
+
function extractPage(html) {
|
|
651
|
+
const root = parse(html);
|
|
652
|
+
const documentEl = root.querySelector("html") ?? root;
|
|
653
|
+
const out = {
|
|
654
|
+
metas: [],
|
|
655
|
+
links: [],
|
|
656
|
+
jsonLd: [],
|
|
657
|
+
images: [],
|
|
658
|
+
headings: [],
|
|
659
|
+
title: null,
|
|
660
|
+
anchors: [],
|
|
661
|
+
forms: [],
|
|
662
|
+
scriptSrcs: [],
|
|
663
|
+
inlineScriptUrls: []
|
|
664
|
+
};
|
|
665
|
+
collect(documentEl, out);
|
|
666
|
+
const social = {};
|
|
667
|
+
const metas = {};
|
|
668
|
+
let metaDescription = null;
|
|
669
|
+
let hasViewportMeta = false;
|
|
670
|
+
for (const m of out.metas) {
|
|
671
|
+
const charset = (m.getAttribute("charset") ?? "").trim();
|
|
672
|
+
if (charset) {
|
|
673
|
+
metas["charset"] = charset;
|
|
674
|
+
continue;
|
|
675
|
+
}
|
|
676
|
+
const key = (m.getAttribute("property") ?? m.getAttribute("name") ?? "").toLowerCase().trim();
|
|
677
|
+
const content = (m.getAttribute("content") ?? "").trim();
|
|
678
|
+
if (!key) {
|
|
679
|
+
const equiv = (m.getAttribute("http-equiv") ?? "").toLowerCase().trim();
|
|
680
|
+
if (equiv === "content-type" && /charset=/i.test(content)) {
|
|
681
|
+
metas["charset"] = content.replace(/^.*charset=/i, "").trim();
|
|
682
|
+
}
|
|
683
|
+
continue;
|
|
684
|
+
}
|
|
685
|
+
if (key === "description") metaDescription = content || null;
|
|
686
|
+
else if (key === "viewport") hasViewportMeta = content.length > 0;
|
|
687
|
+
if (key.startsWith("og:") || key.startsWith("twitter:")) social[key] = content;
|
|
688
|
+
else metas[key] = content;
|
|
689
|
+
}
|
|
690
|
+
const canonicalEl = out.links.find(
|
|
691
|
+
(l) => (l.getAttribute("rel") ?? "").toLowerCase().trim() === "canonical"
|
|
692
|
+
);
|
|
693
|
+
return {
|
|
694
|
+
title: out.title,
|
|
695
|
+
metaDescription,
|
|
696
|
+
canonical: canonicalEl?.getAttribute("href")?.trim() || null,
|
|
697
|
+
social,
|
|
698
|
+
headings: out.headings,
|
|
699
|
+
jsonLd: out.jsonLd,
|
|
700
|
+
images: {
|
|
701
|
+
total: out.images.length,
|
|
702
|
+
withAlt: out.images.filter((i) => (i.getAttribute("alt") ?? "").trim().length > 0).length
|
|
703
|
+
},
|
|
704
|
+
hasViewportMeta,
|
|
705
|
+
// Body-scoped: <head> has no visible text, and scoping here rather than
|
|
706
|
+
// filtering keeps the rule obvious.
|
|
707
|
+
text: textOf(root.querySelector("body") ?? documentEl),
|
|
708
|
+
anchors: out.anchors.slice(0, MAX_ANCHORS).map((a) => ({
|
|
709
|
+
href: (a.getAttribute("href") ?? "").trim(),
|
|
710
|
+
// The visible label, not the raw text: an anchor wrapping an icon and a
|
|
711
|
+
// span should read as its span. `textOf` already drops the unrendered
|
|
712
|
+
// subtrees an icon sprite lives in.
|
|
713
|
+
text: textOf(a).slice(0, 120),
|
|
714
|
+
rel: (a.getAttribute("rel") ?? "").toLowerCase().trim(),
|
|
715
|
+
target: (a.getAttribute("target") ?? "").toLowerCase().trim(),
|
|
716
|
+
// NOT lower-cased and NOT trimmed to a keyword: this is prose, and the
|
|
717
|
+
// check that reads it needs to know whether a human wrote a destination
|
|
718
|
+
// into it. apple.com's “Learn more” links all carry one — `aria-label="Learn
|
|
719
|
+
// more about accessibility"` — so judging their link text without it
|
|
720
|
+
// called 60 well-labelled links bare.
|
|
721
|
+
ariaLabel: (a.getAttribute("aria-label") ?? "").trim().slice(0, 120)
|
|
722
|
+
})),
|
|
723
|
+
// The TRUE total, so a capped list is never mistaken for a complete one.
|
|
724
|
+
anchorCount: out.anchors.length,
|
|
725
|
+
imageSrcs: out.images.map((i) => (i.getAttribute("src") ?? "").trim()).filter((src) => src.length > 0),
|
|
726
|
+
forms: out.forms.map(formShape),
|
|
727
|
+
metas,
|
|
728
|
+
// A projection of elements `collect` was already gathering — until now only
|
|
729
|
+
// the canonical one was ever read off them.
|
|
730
|
+
links: out.links.map((l) => ({
|
|
731
|
+
rel: (l.getAttribute("rel") ?? "").toLowerCase().trim(),
|
|
732
|
+
href: (l.getAttribute("href") ?? "").trim(),
|
|
733
|
+
hreflang: (l.getAttribute("hreflang") ?? "").trim(),
|
|
734
|
+
type: (l.getAttribute("type") ?? "").toLowerCase().trim()
|
|
735
|
+
})),
|
|
736
|
+
scriptSrcs: out.scriptSrcs.slice(0, MAX_SCRIPTS),
|
|
737
|
+
inlineScriptUrls: [...new Set(out.inlineScriptUrls)].slice(0, MAX_INLINE_URLS),
|
|
738
|
+
// The TRUE total, for the same reason `anchorCount` exists: a capped list
|
|
739
|
+
// must never be mistaken for a complete one.
|
|
740
|
+
scriptCount: out.scriptSrcs.length
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
// src/prospect/crawl.ts
|
|
745
|
+
var AI_AGENTS = [
|
|
746
|
+
"GPTBot",
|
|
747
|
+
"OAI-SearchBot",
|
|
748
|
+
"ClaudeBot",
|
|
749
|
+
"PerplexityBot",
|
|
750
|
+
"Google-Extended",
|
|
751
|
+
"CCBot"
|
|
752
|
+
];
|
|
753
|
+
var CLASSICAL_AGENTS = ["Googlebot", "Bingbot"];
|
|
754
|
+
var ALL_AGENTS = [...AI_AGENTS, ...CLASSICAL_AGENTS];
|
|
755
|
+
function parseRobots(txt) {
|
|
756
|
+
const groups = [];
|
|
757
|
+
let current = null;
|
|
758
|
+
let lastWasAgent = false;
|
|
759
|
+
for (const rawLine of txt.split(/\r?\n/)) {
|
|
760
|
+
const line = (rawLine.split("#")[0] ?? "").trim();
|
|
761
|
+
if (!line) continue;
|
|
762
|
+
const idx = line.indexOf(":");
|
|
763
|
+
if (idx < 0) continue;
|
|
764
|
+
const field = line.slice(0, idx).trim().toLowerCase();
|
|
765
|
+
const value = line.slice(idx + 1).trim();
|
|
766
|
+
if (field === "user-agent") {
|
|
767
|
+
if (!current || !lastWasAgent) {
|
|
768
|
+
current = { agents: [], rules: [] };
|
|
769
|
+
groups.push(current);
|
|
770
|
+
}
|
|
771
|
+
current.agents.push(value.toLowerCase());
|
|
772
|
+
lastWasAgent = true;
|
|
773
|
+
} else if (field === "allow" || field === "disallow") {
|
|
774
|
+
if (!current) continue;
|
|
775
|
+
current.rules.push({ type: field === "allow" ? "allow" : "disallow", path: value, line });
|
|
776
|
+
lastWasAgent = false;
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
return groups;
|
|
780
|
+
}
|
|
781
|
+
function pathCoversRoot(pattern) {
|
|
782
|
+
if (!pattern) return false;
|
|
783
|
+
const anchored = pattern.endsWith("$");
|
|
784
|
+
const body = anchored ? pattern.slice(0, -1) : pattern;
|
|
785
|
+
const source = body.split("*").map((part) => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join(".*");
|
|
786
|
+
return new RegExp(`^${source}${anchored ? "$" : ""}`).test("/");
|
|
787
|
+
}
|
|
788
|
+
function robotsPathMatches(pattern, path) {
|
|
789
|
+
if (!pattern) return false;
|
|
790
|
+
const anchored = pattern.endsWith("$");
|
|
791
|
+
const body = anchored ? pattern.slice(0, -1) : pattern;
|
|
792
|
+
const source = body.split("*").map((part) => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join(".*");
|
|
793
|
+
return new RegExp(`^${source}${anchored ? "$" : ""}`).test(path);
|
|
794
|
+
}
|
|
795
|
+
function robotsAllowsUs(robotsTxt, url, agent) {
|
|
796
|
+
if (robotsTxt === null) return true;
|
|
797
|
+
let path;
|
|
798
|
+
try {
|
|
799
|
+
const u = new URL(url);
|
|
800
|
+
path = `${u.pathname}${u.search}`;
|
|
801
|
+
} catch {
|
|
802
|
+
return true;
|
|
803
|
+
}
|
|
804
|
+
const groups = parseRobots(robotsTxt);
|
|
805
|
+
const lower = agent.toLowerCase();
|
|
806
|
+
const named = groups.filter(
|
|
807
|
+
(g) => g.agents.some((a) => a !== "*" && (lower.includes(a) || a.includes(lower.split("/")[0] ?? "")))
|
|
808
|
+
);
|
|
809
|
+
const matched = named.length > 0 ? named : groups.filter((g) => g.agents.includes("*"));
|
|
810
|
+
let best = null;
|
|
811
|
+
for (const rule of matched.flatMap((g) => g.rules)) {
|
|
812
|
+
if (!robotsPathMatches(rule.path, path)) continue;
|
|
813
|
+
if (best === null || rule.path.length > best.length) {
|
|
814
|
+
best = { type: rule.type, length: rule.path.length };
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
return best === null || best.type === "allow";
|
|
818
|
+
}
|
|
819
|
+
function evaluateAgentAccess(robotsTxt) {
|
|
820
|
+
if (robotsTxt === null) {
|
|
821
|
+
return ALL_AGENTS.map((agent) => ({ agent, allowed: true, matchedRule: null }));
|
|
822
|
+
}
|
|
823
|
+
const groups = parseRobots(robotsTxt);
|
|
824
|
+
return ALL_AGENTS.map((agent) => {
|
|
825
|
+
const lower = agent.toLowerCase();
|
|
826
|
+
const named = groups.filter((g) => g.agents.includes(lower));
|
|
827
|
+
const matched = named.length > 0 ? named : groups.filter((g) => g.agents.includes("*"));
|
|
828
|
+
if (matched.length === 0) return { agent, allowed: true, matchedRule: null };
|
|
829
|
+
const header = `User-agent: ${named.length > 0 ? agent : "*"}`;
|
|
830
|
+
const rootRules = matched.flatMap((g) => g.rules).filter((r) => pathCoversRoot(r.path));
|
|
831
|
+
const block = rootRules.find((r) => r.type === "disallow");
|
|
832
|
+
const allow = rootRules.find((r) => r.type === "allow");
|
|
833
|
+
if (block && !allow) return { agent, allowed: false, matchedRule: `${header} \u2192 ${block.line}` };
|
|
834
|
+
return { agent, allowed: true, matchedRule: allow ? `${header} \u2192 ${allow.line}` : null };
|
|
835
|
+
});
|
|
836
|
+
}
|
|
837
|
+
var MAX_WALK_DEPTH2 = 100;
|
|
838
|
+
function sameOriginLinks(html, baseUrl) {
|
|
839
|
+
const site = new URL(baseUrl);
|
|
840
|
+
const doc = parse2(html);
|
|
841
|
+
const baseHref = doc.querySelector("base")?.getAttribute("href");
|
|
842
|
+
let resolveBase = site;
|
|
843
|
+
if (baseHref) {
|
|
844
|
+
try {
|
|
845
|
+
resolveBase = new URL(baseHref, site);
|
|
846
|
+
} catch {
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
const out = [];
|
|
850
|
+
const seen = /* @__PURE__ */ new Set();
|
|
851
|
+
const walk = (el, depth) => {
|
|
852
|
+
if (depth > MAX_WALK_DEPTH2) return;
|
|
853
|
+
for (const child of el.childNodes) {
|
|
854
|
+
if (child.nodeType !== NodeType2.ELEMENT_NODE) continue;
|
|
855
|
+
const e = child;
|
|
856
|
+
if (UNRENDERED_TAGS.has(e.tagName)) continue;
|
|
857
|
+
if (e.tagName === "A") {
|
|
858
|
+
const href = e.getAttribute("href");
|
|
859
|
+
if (href) {
|
|
860
|
+
let u;
|
|
861
|
+
try {
|
|
862
|
+
u = new URL(href, resolveBase);
|
|
863
|
+
} catch {
|
|
864
|
+
u = null;
|
|
865
|
+
}
|
|
866
|
+
if (u && u.origin === site.origin && (u.protocol === "http:" || u.protocol === "https:")) {
|
|
867
|
+
u.hash = "";
|
|
868
|
+
const norm = u.toString();
|
|
869
|
+
if (!seen.has(norm)) {
|
|
870
|
+
seen.add(norm);
|
|
871
|
+
out.push(norm);
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
walk(e, depth + 1);
|
|
877
|
+
}
|
|
878
|
+
};
|
|
879
|
+
walk(doc, 0);
|
|
880
|
+
return out;
|
|
881
|
+
}
|
|
882
|
+
function decodeXmlText(s) {
|
|
883
|
+
return s.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1").replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(parseInt(hex, 16))).replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number(dec))).replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&");
|
|
884
|
+
}
|
|
885
|
+
function parseSitemapLocs(xml) {
|
|
886
|
+
return [...xml.matchAll(/<(?:[\w-]+:)?loc\b[^>]*>([\s\S]*?)<\/(?:[\w-]+:)?loc>/gi)].map((m) => decodeXmlText(m[1] ?? "").trim()).filter(Boolean);
|
|
887
|
+
}
|
|
888
|
+
function isSafeNestedSitemap(child, origin) {
|
|
889
|
+
let url;
|
|
890
|
+
try {
|
|
891
|
+
url = new URL(child, origin);
|
|
892
|
+
} catch {
|
|
893
|
+
return false;
|
|
894
|
+
}
|
|
895
|
+
if (url.protocol !== "https:" && url.protocol !== "http:") return false;
|
|
896
|
+
if (url.origin !== origin) return false;
|
|
897
|
+
return !isPrivateOrLoopbackHost(url.hostname);
|
|
898
|
+
}
|
|
899
|
+
function isSitemapIndex(xml) {
|
|
900
|
+
return /<(?:[\w-]+:)?sitemapindex[\s>]/i.test(xml);
|
|
901
|
+
}
|
|
902
|
+
function asRendered(v) {
|
|
903
|
+
if (v === void 0) return null;
|
|
904
|
+
return typeof v === "string" ? { html: v, axe: null, vitals: null } : v;
|
|
905
|
+
}
|
|
906
|
+
var USER_AGENT = "ReddoorAudit/1.0 (+https://reddoorla.com/; operator-run site audit)";
|
|
907
|
+
var ASSET_EXT = /\.(pdf|jpe?g|png|gif|webp|avif|svg|zip|mp4|mov|css|js|xml|json)$/i;
|
|
908
|
+
var MAX_RESPONSE_BYTES = 5e6;
|
|
909
|
+
var RENDER_SETTLE_MS = 1500;
|
|
910
|
+
var ResponseTooLargeError = class extends Error {
|
|
911
|
+
constructor(url) {
|
|
912
|
+
super(`response exceeds the ${MAX_RESPONSE_BYTES}-byte limit: ${url}`);
|
|
913
|
+
this.name = "ResponseTooLargeError";
|
|
914
|
+
}
|
|
915
|
+
};
|
|
916
|
+
var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
917
|
+
var MAX_SITEMAP_SAMPLE = 120;
|
|
918
|
+
var MAX_SITEMAP_CHILDREN = 25;
|
|
919
|
+
var CONTENT_BUDGET_MS = 2e4;
|
|
920
|
+
var AXE_BUDGET_MS = 45e3;
|
|
921
|
+
var VITALS_BUDGET_MS = 3e4;
|
|
922
|
+
async function pacedEach(items, delayMs, fn, sleepFn = sleep) {
|
|
923
|
+
for (let i = 0; i < items.length; i++) {
|
|
924
|
+
if (i > 0 && delayMs > 0) await sleepFn(delayMs);
|
|
925
|
+
await fn(items[i]);
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
async function optional(deps, url) {
|
|
929
|
+
try {
|
|
930
|
+
const res = await deps.fetchUrl(url);
|
|
931
|
+
return res.status >= 400 ? { res: null, error: null } : { res, error: null };
|
|
932
|
+
} catch (err) {
|
|
933
|
+
if (err instanceof ResponseTooLargeError) return { res: null, error: null };
|
|
934
|
+
return { res: null, error: err instanceof Error ? err.message : String(err) };
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
async function optionalRobots(deps, url) {
|
|
938
|
+
const first = await optional(deps, url);
|
|
939
|
+
return first.error === null ? first : optional(deps, url);
|
|
940
|
+
}
|
|
941
|
+
function textSidecar(res) {
|
|
942
|
+
if (!res) return null;
|
|
943
|
+
const body = res.body.trim();
|
|
944
|
+
if (!body || body.startsWith("<")) return null;
|
|
945
|
+
return res.body;
|
|
946
|
+
}
|
|
947
|
+
function headerValue(headers, name) {
|
|
948
|
+
for (const [k, v] of Object.entries(headers)) {
|
|
949
|
+
if (k.toLowerCase() === name) return v;
|
|
950
|
+
}
|
|
951
|
+
return null;
|
|
952
|
+
}
|
|
953
|
+
function samePageKey(u) {
|
|
954
|
+
const path = u.pathname.replace(/\/+$/, "") || "/";
|
|
955
|
+
return `${u.origin.toLowerCase()}${path}${u.search}`;
|
|
956
|
+
}
|
|
957
|
+
function normalizeCandidates(urls, origin, max, robotsTxt) {
|
|
958
|
+
const firstOfPath = [];
|
|
959
|
+
const variants = [];
|
|
960
|
+
const seen = /* @__PURE__ */ new Set();
|
|
961
|
+
const paths = /* @__PURE__ */ new Set();
|
|
962
|
+
for (const raw of urls) {
|
|
963
|
+
let u;
|
|
964
|
+
try {
|
|
965
|
+
u = new URL(raw);
|
|
966
|
+
} catch {
|
|
967
|
+
continue;
|
|
968
|
+
}
|
|
969
|
+
if (u.origin !== origin) continue;
|
|
970
|
+
if (ASSET_EXT.test(u.pathname)) continue;
|
|
971
|
+
if (isInfraPath(u.toString())) continue;
|
|
972
|
+
if (!robotsAllowsUs(robotsTxt, u.toString(), USER_AGENT)) continue;
|
|
973
|
+
u.hash = "";
|
|
974
|
+
const key = samePageKey(u);
|
|
975
|
+
if (seen.has(key)) continue;
|
|
976
|
+
seen.add(key);
|
|
977
|
+
const path = `${u.origin.toLowerCase()}${u.pathname.replace(/\/+$/, "") || "/"}`;
|
|
978
|
+
if (paths.has(path)) variants.push(u.toString());
|
|
979
|
+
else {
|
|
980
|
+
paths.add(path);
|
|
981
|
+
firstOfPath.push(u.toString());
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
return [...firstOfPath, ...variants].slice(0, max);
|
|
985
|
+
}
|
|
986
|
+
function sitemapTruncatedOf(s) {
|
|
987
|
+
return s.sitemapTruncated;
|
|
988
|
+
}
|
|
989
|
+
async function fetchSidecars(origin, deps) {
|
|
990
|
+
const robots = await optionalRobots(deps, `${origin}/robots.txt`);
|
|
991
|
+
const robotsTxt = textSidecar(robots.res);
|
|
992
|
+
const agentAccess = evaluateAgentAccess(
|
|
993
|
+
robotsTxt && /user-agent/i.test(robotsTxt) ? robotsTxt : null
|
|
994
|
+
);
|
|
995
|
+
const llms = await optional(deps, `${origin}/llms.txt`);
|
|
996
|
+
const llmsRaw = textSidecar(llms.res);
|
|
997
|
+
const llmsTxt = llmsRaw ? {
|
|
998
|
+
present: true,
|
|
999
|
+
firstLine: llmsRaw.split(/\r?\n/).find((l) => l.trim())?.trim() ?? null
|
|
1000
|
+
} : { present: false, firstLine: null };
|
|
1001
|
+
const sitemap = await optional(deps, `${origin}/sitemap.xml`);
|
|
1002
|
+
let sitemapUrls = [];
|
|
1003
|
+
let sitemapPresent = false;
|
|
1004
|
+
let sitemapTruncated = false;
|
|
1005
|
+
if (sitemap.res && /<(urlset|sitemapindex)[\s>]/i.test(sitemap.res.body)) {
|
|
1006
|
+
sitemapPresent = true;
|
|
1007
|
+
if (isSitemapIndex(sitemap.res.body)) {
|
|
1008
|
+
const safe2 = parseSitemapLocs(sitemap.res.body).filter(
|
|
1009
|
+
(child) => isSafeNestedSitemap(child, origin)
|
|
1010
|
+
);
|
|
1011
|
+
const children = safe2.slice(0, MAX_SITEMAP_CHILDREN);
|
|
1012
|
+
sitemapTruncated = safe2.length > children.length;
|
|
1013
|
+
for (const child of children) {
|
|
1014
|
+
const nested = await optional(deps, child);
|
|
1015
|
+
if (nested.res) sitemapUrls.push(...parseSitemapLocs(nested.res.body));
|
|
1016
|
+
else sitemapTruncated = true;
|
|
1017
|
+
}
|
|
1018
|
+
} else {
|
|
1019
|
+
sitemapUrls = parseSitemapLocs(sitemap.res.body);
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
return {
|
|
1023
|
+
robotsTxt,
|
|
1024
|
+
agentAccess,
|
|
1025
|
+
llmsTxt,
|
|
1026
|
+
sitemapUrls,
|
|
1027
|
+
sitemapPresent,
|
|
1028
|
+
sitemapTruncated,
|
|
1029
|
+
sidecarErrors: { robots: robots.error, llms: llms.error, sitemap: sitemap.error }
|
|
1030
|
+
};
|
|
1031
|
+
}
|
|
1032
|
+
async function crawlSite(rawUrl, deps) {
|
|
1033
|
+
const start = new URL(rawUrl);
|
|
1034
|
+
start.hash = "";
|
|
1035
|
+
start.username = "";
|
|
1036
|
+
start.password = "";
|
|
1037
|
+
if (isPrivateOrLoopbackHost(start.hostname)) {
|
|
1038
|
+
throw Object.assign(
|
|
1039
|
+
new Error(
|
|
1040
|
+
`${start.toString()} is a private address (${start.hostname}) \u2014 refusing to crawl it.`
|
|
1041
|
+
),
|
|
1042
|
+
{ exitCode: 1 }
|
|
1043
|
+
);
|
|
1044
|
+
}
|
|
1045
|
+
let home;
|
|
1046
|
+
try {
|
|
1047
|
+
home = await deps.fetchUrl(start.toString());
|
|
1048
|
+
} catch (err) {
|
|
1049
|
+
throw Object.assign(
|
|
1050
|
+
new Error(
|
|
1051
|
+
`Could not reach ${start.toString()}: ${err instanceof Error ? err.message : String(err)}`
|
|
1052
|
+
),
|
|
1053
|
+
{ exitCode: 1 }
|
|
1054
|
+
);
|
|
1055
|
+
}
|
|
1056
|
+
if (home.status >= 400) {
|
|
1057
|
+
throw Object.assign(
|
|
1058
|
+
new Error(`${start.toString()} returned HTTP ${home.status} \u2014 nothing to audit.`),
|
|
1059
|
+
{ exitCode: 1 }
|
|
1060
|
+
);
|
|
1061
|
+
}
|
|
1062
|
+
let resolved;
|
|
1063
|
+
try {
|
|
1064
|
+
resolved = new URL(home.url ?? start.toString());
|
|
1065
|
+
} catch {
|
|
1066
|
+
resolved = start;
|
|
1067
|
+
}
|
|
1068
|
+
if (isPrivateOrLoopbackHost(resolved.hostname)) {
|
|
1069
|
+
throw Object.assign(
|
|
1070
|
+
new Error(
|
|
1071
|
+
`${start.toString()} redirected to a private address (${resolved.hostname}) \u2014 refusing to crawl it.`
|
|
1072
|
+
),
|
|
1073
|
+
{ exitCode: 1 }
|
|
1074
|
+
);
|
|
1075
|
+
}
|
|
1076
|
+
const resolvedUrl = resolved.toString();
|
|
1077
|
+
const origin = resolved.origin;
|
|
1078
|
+
const sidecars = await fetchSidecars(origin, deps);
|
|
1079
|
+
const pageUrls = normalizeCandidates(
|
|
1080
|
+
[resolvedUrl, ...sidecars.sitemapUrls, ...sameOriginLinks(home.body, resolvedUrl)],
|
|
1081
|
+
origin,
|
|
1082
|
+
deps.maxPages,
|
|
1083
|
+
sidecars.robotsTxt
|
|
1084
|
+
);
|
|
1085
|
+
const rendered = await deps.renderPages(pageUrls).catch(() => /* @__PURE__ */ new Map());
|
|
1086
|
+
const pages = [];
|
|
1087
|
+
for (const url of pageUrls) {
|
|
1088
|
+
let res = null;
|
|
1089
|
+
let error = null;
|
|
1090
|
+
if (url === resolvedUrl) {
|
|
1091
|
+
res = home;
|
|
1092
|
+
} else {
|
|
1093
|
+
if (deps.delayMs > 0) await sleep(deps.delayMs);
|
|
1094
|
+
try {
|
|
1095
|
+
res = await deps.fetchUrl(url);
|
|
1096
|
+
} catch (err) {
|
|
1097
|
+
error = err instanceof Error ? err.message : String(err);
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
const renderedPage = asRendered(rendered.get(url));
|
|
1101
|
+
const renderedHtml = renderedPage?.html ?? null;
|
|
1102
|
+
const contentType = res ? headerValue(res.headers, "content-type") : null;
|
|
1103
|
+
const notHtmlReason = contentType !== null && !contentType.toLowerCase().includes("html") ? `not HTML (${contentType})` : null;
|
|
1104
|
+
const usable = res !== null && res.status < 400 && notHtmlReason === null;
|
|
1105
|
+
pages.push({
|
|
1106
|
+
url,
|
|
1107
|
+
status: res?.status ?? null,
|
|
1108
|
+
raw: usable ? extractPage(res.body) : null,
|
|
1109
|
+
// Gated on `usable` for the same reason `raw` is. A browser paints
|
|
1110
|
+
// something for a 404 — Cloudflare's email-protection interstitial is the
|
|
1111
|
+
// case that bit us — and a captured paint is not evidence that the URL is
|
|
1112
|
+
// a page of this website. Leaving it non-null let every cross-page check
|
|
1113
|
+
// that asked "is there an extract?" admit a page the server refused, and
|
|
1114
|
+
// that one URL became the only dead end and the only off-template page
|
|
1115
|
+
// found in the entire stored corpus.
|
|
1116
|
+
rendered: usable && renderedHtml ? extractPage(renderedHtml) : null,
|
|
1117
|
+
error: error ?? (res && res.status >= 400 ? `HTTP ${res.status}` : notHtmlReason),
|
|
1118
|
+
// Gated on `usable` for the same reason as `rendered`: a browser paints
|
|
1119
|
+
// something for a 404, and running the rule set over an error page would
|
|
1120
|
+
// report that page's failings as the website's.
|
|
1121
|
+
axe: usable ? renderedPage?.axe ?? null : null,
|
|
1122
|
+
vitals: usable ? renderedPage?.vitals ?? null : null,
|
|
1123
|
+
// Spread, so an absent probe stays absent: `formProbe: undefined` is
|
|
1124
|
+
// rejected under exactOptionalPropertyTypes, and `?? null` would turn
|
|
1125
|
+
// "we never tried" into "we tried and there was nothing".
|
|
1126
|
+
...usable && renderedPage?.formProbe !== void 0 ? { formProbe: renderedPage.formProbe } : {}
|
|
1127
|
+
});
|
|
1128
|
+
}
|
|
1129
|
+
const homeHeaders = {};
|
|
1130
|
+
for (const [k, v] of Object.entries(home.headers)) homeHeaders[k.toLowerCase()] = v;
|
|
1131
|
+
return {
|
|
1132
|
+
origin,
|
|
1133
|
+
robotsTxt: sidecars.robotsTxt,
|
|
1134
|
+
agentAccess: sidecars.agentAccess,
|
|
1135
|
+
sitemap: {
|
|
1136
|
+
present: sidecars.sitemapPresent,
|
|
1137
|
+
urlCount: sidecars.sitemapUrls.length,
|
|
1138
|
+
// A capped sample, kept so Tier 2 can ask whether the URLs a sitemap
|
|
1139
|
+
// advertises actually answer — `urlCount` alone cannot be probed. Capped
|
|
1140
|
+
// because this lands in `result_json`, and a 4,000-URL sitemap has no
|
|
1141
|
+
// business being stored whole for the sake of sampling twelve of them.
|
|
1142
|
+
sample: sidecars.sitemapUrls.slice(0, MAX_SITEMAP_SAMPLE),
|
|
1143
|
+
truncated: sitemapTruncatedOf(sidecars)
|
|
1144
|
+
},
|
|
1145
|
+
llmsTxt: sidecars.llmsTxt,
|
|
1146
|
+
sidecarErrors: sidecars.sidecarErrors,
|
|
1147
|
+
homeHeaders,
|
|
1148
|
+
pages
|
|
1149
|
+
};
|
|
1150
|
+
}
|
|
1151
|
+
async function readCapped(res, url) {
|
|
1152
|
+
const declared = res.headers.get("content-length");
|
|
1153
|
+
if (declared !== null && Number(declared) > MAX_RESPONSE_BYTES) {
|
|
1154
|
+
throw new ResponseTooLargeError(url);
|
|
1155
|
+
}
|
|
1156
|
+
if (!res.body) return "";
|
|
1157
|
+
const reader = res.body.getReader();
|
|
1158
|
+
const chunks = [];
|
|
1159
|
+
let total = 0;
|
|
1160
|
+
for (; ; ) {
|
|
1161
|
+
const { done, value } = await reader.read();
|
|
1162
|
+
if (done) break;
|
|
1163
|
+
if (!value) continue;
|
|
1164
|
+
total += value.byteLength;
|
|
1165
|
+
if (total > MAX_RESPONSE_BYTES) {
|
|
1166
|
+
await reader.cancel();
|
|
1167
|
+
throw new ResponseTooLargeError(url);
|
|
1168
|
+
}
|
|
1169
|
+
chunks.push(value);
|
|
1170
|
+
}
|
|
1171
|
+
const merged = new Uint8Array(total);
|
|
1172
|
+
let offset = 0;
|
|
1173
|
+
for (const chunk of chunks) {
|
|
1174
|
+
merged.set(chunk, offset);
|
|
1175
|
+
offset += chunk.byteLength;
|
|
1176
|
+
}
|
|
1177
|
+
return new TextDecoder("utf-8").decode(merged);
|
|
1178
|
+
}
|
|
1179
|
+
var MOBILE_WIDTH = 375;
|
|
1180
|
+
var MOBILE_HEIGHT = 812;
|
|
1181
|
+
var DESKTOP_WIDTH = 1280;
|
|
1182
|
+
var DESKTOP_HEIGHT = 720;
|
|
1183
|
+
var OVERFLOW_SETTLE_BUDGET_MS = 5e3;
|
|
1184
|
+
var OVERFLOW_SETTLE_STEP_MS = 250;
|
|
1185
|
+
async function measureVitals(page, collected) {
|
|
1186
|
+
try {
|
|
1187
|
+
const desktop = await page.evaluate(
|
|
1188
|
+
({ maxImages }) => {
|
|
1189
|
+
const tiny = { count: 0, sample: null };
|
|
1190
|
+
for (const el of Array.from(document.body.querySelectorAll("*"))) {
|
|
1191
|
+
const ownText = Array.from(el.childNodes).filter((n) => n.nodeType === 3).map((n) => n.textContent ?? "").join("").trim();
|
|
1192
|
+
if (ownText.length < 12) continue;
|
|
1193
|
+
const size = parseFloat(getComputedStyle(el).fontSize);
|
|
1194
|
+
if (Number.isFinite(size) && size > 0 && size < 12) {
|
|
1195
|
+
tiny.count += 1;
|
|
1196
|
+
tiny.sample ??= ownText.slice(0, 80);
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
const oversized = [];
|
|
1200
|
+
for (const img of Array.from(document.images)) {
|
|
1201
|
+
const rendered = img.clientWidth;
|
|
1202
|
+
if (rendered > 0 && img.naturalWidth > rendered * 2.5) {
|
|
1203
|
+
oversized.push({
|
|
1204
|
+
src: img.currentSrc || img.src,
|
|
1205
|
+
naturalWidth: img.naturalWidth,
|
|
1206
|
+
renderedWidth: rendered
|
|
1207
|
+
});
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
oversized.sort((a, b) => b.naturalWidth - a.naturalWidth);
|
|
1211
|
+
return { tiny, oversized: oversized.slice(0, maxImages) };
|
|
1212
|
+
},
|
|
1213
|
+
{ maxImages: MAX_OVERSIZED_IMAGES }
|
|
1214
|
+
);
|
|
1215
|
+
await page.setViewportSize({ width: MOBILE_WIDTH, height: MOBILE_HEIGHT });
|
|
1216
|
+
const overflowAt375 = await settledOverflow(
|
|
1217
|
+
() => page.evaluate(() => {
|
|
1218
|
+
const doc = document.documentElement;
|
|
1219
|
+
return Math.max(0, Math.round(doc.scrollWidth - doc.clientWidth));
|
|
1220
|
+
}),
|
|
1221
|
+
(ms) => page.waitForTimeout(ms)
|
|
1222
|
+
);
|
|
1223
|
+
await page.setViewportSize({ width: DESKTOP_WIDTH, height: DESKTOP_HEIGHT });
|
|
1224
|
+
return {
|
|
1225
|
+
consoleErrors: collected.consoleErrors.slice(0, MAX_CONSOLE_ERRORS),
|
|
1226
|
+
failedRequests: collected.failed.slice(0, MAX_FAILED_REQUESTS),
|
|
1227
|
+
overflowAt375,
|
|
1228
|
+
tinyText: desktop.tiny,
|
|
1229
|
+
oversizedImages: desktop.oversized
|
|
1230
|
+
};
|
|
1231
|
+
} catch {
|
|
1232
|
+
return null;
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
async function settledOverflow(read, wait, budgetMs = OVERFLOW_SETTLE_BUDGET_MS, stepMs = OVERFLOW_SETTLE_STEP_MS) {
|
|
1236
|
+
let last = -1;
|
|
1237
|
+
const deadline = Date.now() + budgetMs;
|
|
1238
|
+
for (; ; ) {
|
|
1239
|
+
await wait(stepMs);
|
|
1240
|
+
const now = await read();
|
|
1241
|
+
if (now === last) return now;
|
|
1242
|
+
if (Date.now() > deadline) return 0;
|
|
1243
|
+
last = now;
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
async function runAxe(page) {
|
|
1247
|
+
try {
|
|
1248
|
+
const { AxeBuilder } = await import("@axe-core/playwright");
|
|
1249
|
+
const results = await new AxeBuilder({ page }).analyze();
|
|
1250
|
+
return {
|
|
1251
|
+
violations: results.violations.map((v) => ({
|
|
1252
|
+
id: v.id,
|
|
1253
|
+
impact: v.impact ?? null,
|
|
1254
|
+
help: v.help,
|
|
1255
|
+
helpUrl: v.helpUrl,
|
|
1256
|
+
nodes: v.nodes.length,
|
|
1257
|
+
// One element, truncated: enough for a reader to find it on the page,
|
|
1258
|
+
// not enough to bloat a stored report with a page of markup.
|
|
1259
|
+
sample: v.nodes[0]?.html?.slice(0, 200) ?? null
|
|
1260
|
+
})),
|
|
1261
|
+
passes: results.passes.length,
|
|
1262
|
+
incomplete: results.incomplete.length,
|
|
1263
|
+
// The ids as well as the count: "three rules need a human" is a worry,
|
|
1264
|
+
// "colour contrast over an image needs a human" is something to look at.
|
|
1265
|
+
incompleteIds: results.incomplete.map((r) => r.id),
|
|
1266
|
+
// Carried so the report can say how many rules had something to check
|
|
1267
|
+
// rather than implying we checked all ninety.
|
|
1268
|
+
inapplicable: results.inapplicable.length
|
|
1269
|
+
};
|
|
1270
|
+
} catch {
|
|
1271
|
+
return null;
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
function defaultCrawlDeps(over = {}) {
|
|
1275
|
+
const { probeForms: pressForms = true, ...deps } = over;
|
|
1276
|
+
const maxPages = over.maxPages ?? 20;
|
|
1277
|
+
const delayMs = over.delayMs ?? 500;
|
|
1278
|
+
return {
|
|
1279
|
+
async fetchUrl(url) {
|
|
1280
|
+
const res = await fetch(url, {
|
|
1281
|
+
headers: {
|
|
1282
|
+
"user-agent": USER_AGENT,
|
|
1283
|
+
accept: "text/html,application/xhtml+xml,text/plain,*/*"
|
|
1284
|
+
},
|
|
1285
|
+
redirect: "follow",
|
|
1286
|
+
signal: AbortSignal.timeout(2e4)
|
|
1287
|
+
});
|
|
1288
|
+
const headers = {};
|
|
1289
|
+
res.headers.forEach((v, k) => {
|
|
1290
|
+
headers[k] = v;
|
|
1291
|
+
});
|
|
1292
|
+
return { status: res.status, body: await readCapped(res, url), headers, url: res.url };
|
|
1293
|
+
},
|
|
1294
|
+
async renderPages(urls) {
|
|
1295
|
+
const { chromium } = await import("@playwright/test");
|
|
1296
|
+
const out = /* @__PURE__ */ new Map();
|
|
1297
|
+
const browser = await chromium.launch();
|
|
1298
|
+
try {
|
|
1299
|
+
const ctx = await browser.newContext({
|
|
1300
|
+
userAgent: USER_AGENT,
|
|
1301
|
+
viewport: { width: DESKTOP_WIDTH, height: DESKTOP_HEIGHT }
|
|
1302
|
+
});
|
|
1303
|
+
const page = await ctx.newPage();
|
|
1304
|
+
const collected = {
|
|
1305
|
+
consoleErrors: [],
|
|
1306
|
+
failed: []
|
|
1307
|
+
};
|
|
1308
|
+
let currentOrigin = "";
|
|
1309
|
+
const noteError = (message) => {
|
|
1310
|
+
const trimmed = message.trim().slice(0, 300);
|
|
1311
|
+
if (trimmed && !collected.consoleErrors.includes(trimmed)) {
|
|
1312
|
+
collected.consoleErrors.push(trimmed);
|
|
1313
|
+
}
|
|
1314
|
+
};
|
|
1315
|
+
page.on("pageerror", (err) => noteError(err.message));
|
|
1316
|
+
page.on("console", (msg) => {
|
|
1317
|
+
if (msg.type() === "error") noteError(msg.text());
|
|
1318
|
+
});
|
|
1319
|
+
const noteFailed = (url, status) => {
|
|
1320
|
+
const firstParty = (() => {
|
|
1321
|
+
try {
|
|
1322
|
+
return new URL(url).origin === currentOrigin;
|
|
1323
|
+
} catch {
|
|
1324
|
+
return false;
|
|
1325
|
+
}
|
|
1326
|
+
})();
|
|
1327
|
+
if (collected.failed.length < MAX_FAILED_REQUESTS * 4) {
|
|
1328
|
+
collected.failed.push({ url: url.slice(0, 200), status, firstParty });
|
|
1329
|
+
}
|
|
1330
|
+
};
|
|
1331
|
+
page.on("requestfailed", (req) => {
|
|
1332
|
+
const why = req.failure()?.errorText ?? "";
|
|
1333
|
+
if (/ERR_ABORTED/i.test(why)) return;
|
|
1334
|
+
noteFailed(req.url(), null);
|
|
1335
|
+
});
|
|
1336
|
+
page.on("response", (res) => {
|
|
1337
|
+
if (res.status() >= 400) noteFailed(res.url(), res.status());
|
|
1338
|
+
});
|
|
1339
|
+
let formProbed = false;
|
|
1340
|
+
await pacedEach(urls, delayMs, async (url) => {
|
|
1341
|
+
try {
|
|
1342
|
+
collected.consoleErrors.length = 0;
|
|
1343
|
+
collected.failed.length = 0;
|
|
1344
|
+
try {
|
|
1345
|
+
currentOrigin = new URL(url).origin;
|
|
1346
|
+
} catch {
|
|
1347
|
+
currentOrigin = "";
|
|
1348
|
+
}
|
|
1349
|
+
await page.goto(url, { waitUntil: "load", timeout: 2e4 });
|
|
1350
|
+
await page.waitForTimeout(RENDER_SETTLE_MS);
|
|
1351
|
+
const html = await withTimeout(page.content(), CONTENT_BUDGET_MS);
|
|
1352
|
+
if (html === null) return;
|
|
1353
|
+
const axe = await withTimeout(runAxe(page), AXE_BUDGET_MS);
|
|
1354
|
+
const vitals = await withTimeout(measureVitals(page, collected), VITALS_BUDGET_MS);
|
|
1355
|
+
if (!pressForms) {
|
|
1356
|
+
out.set(url, { html, axe, vitals });
|
|
1357
|
+
} else {
|
|
1358
|
+
let formProbe = null;
|
|
1359
|
+
if (!formProbed) {
|
|
1360
|
+
formProbe = await withTimeout(
|
|
1361
|
+
probeForms(pageInteractionDeps(page, url)),
|
|
1362
|
+
PROBE_BUDGET_MS
|
|
1363
|
+
);
|
|
1364
|
+
if (formProbe) formProbed = true;
|
|
1365
|
+
}
|
|
1366
|
+
out.set(url, { html, axe, vitals, formProbe });
|
|
1367
|
+
}
|
|
1368
|
+
} catch {
|
|
1369
|
+
}
|
|
1370
|
+
});
|
|
1371
|
+
} finally {
|
|
1372
|
+
await browser.close();
|
|
1373
|
+
}
|
|
1374
|
+
return out;
|
|
1375
|
+
},
|
|
1376
|
+
maxPages,
|
|
1377
|
+
delayMs,
|
|
1378
|
+
...deps
|
|
1379
|
+
};
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
// src/prospect/stack.ts
|
|
1383
|
+
var LAYER_ORDER = [
|
|
1384
|
+
"cms",
|
|
1385
|
+
"theme",
|
|
1386
|
+
"page-builder",
|
|
1387
|
+
"framework",
|
|
1388
|
+
"ecommerce",
|
|
1389
|
+
"plugin",
|
|
1390
|
+
"forms",
|
|
1391
|
+
"analytics",
|
|
1392
|
+
"fonts",
|
|
1393
|
+
"hosting"
|
|
1394
|
+
];
|
|
1395
|
+
var LAYER_LABELS = {
|
|
1396
|
+
cms: "Platform",
|
|
1397
|
+
theme: "Theme",
|
|
1398
|
+
"page-builder": "Page builder",
|
|
1399
|
+
framework: "Framework",
|
|
1400
|
+
ecommerce: "Store",
|
|
1401
|
+
plugin: "Plugins",
|
|
1402
|
+
forms: "Forms",
|
|
1403
|
+
analytics: "Analytics and tracking",
|
|
1404
|
+
fonts: "Fonts",
|
|
1405
|
+
hosting: "Hosting and CDN"
|
|
1406
|
+
};
|
|
1407
|
+
var ASSET_SIGNATURES = [
|
|
1408
|
+
// Platforms, by the paths only they serve.
|
|
1409
|
+
{ layer: "cms", name: "WordPress", match: /\/wp-(?:content|includes)\// },
|
|
1410
|
+
{ layer: "cms", name: "Squarespace", match: /(?:^|\/\/|\.)squarespace\.com\// },
|
|
1411
|
+
{ layer: "cms", name: "Wix", match: /(?:^|\/\/|\.)wixstatic\.com\// },
|
|
1412
|
+
{ layer: "cms", name: "Webflow", match: /(?:^|\/\/|\.)website-files\.com\// },
|
|
1413
|
+
{ layer: "cms", name: "HubSpot CMS", match: /(?:^|\/\/|\.)(?:hs-scripts|hs-banner)\.com\// },
|
|
1414
|
+
{ layer: "cms", name: "Duda", match: /(?:^|\/\/|\.)cdn-website\.com\// },
|
|
1415
|
+
{ layer: "cms", name: "GoDaddy Website Builder", match: /(?:^|\/\/|\.)wsimg\.com\// },
|
|
1416
|
+
{ layer: "cms", name: "Drupal", match: /\/(?:core|modules)\/misc\/drupal\.js/ },
|
|
1417
|
+
{ layer: "cms", name: "Ghost", match: /\/assets\/built\/[a-z-]+\.js/ },
|
|
1418
|
+
// Frameworks, by their build output paths.
|
|
1419
|
+
{ layer: "framework", name: "Next.js", match: /\/_next\/static\// },
|
|
1420
|
+
{ layer: "framework", name: "Nuxt", match: /\/_nuxt\// },
|
|
1421
|
+
{ layer: "framework", name: "SvelteKit", match: /\/_app\/immutable\// },
|
|
1422
|
+
{ layer: "framework", name: "Gatsby", match: /\/page-data\/|webpack-runtime-[a-f0-9]+\.js/ },
|
|
1423
|
+
{ layer: "framework", name: "Angular", match: /\/(?:polyfills|runtime)[.-][a-z0-9]+\.js/ },
|
|
1424
|
+
// Page builders. All WordPress plugins, but a reader thinks of them as the
|
|
1425
|
+
// thing they edit in, so they get their own layer rather than being buried
|
|
1426
|
+
// among thirty plugin names.
|
|
1427
|
+
{ layer: "page-builder", name: "Elementor", match: /\/wp-content\/plugins\/elementor/ },
|
|
1428
|
+
{ layer: "page-builder", name: "Divi", match: /\/wp-content\/themes\/[Dd]ivi\// },
|
|
1429
|
+
{ layer: "page-builder", name: "WPBakery", match: /\/wp-content\/plugins\/js_composer\// },
|
|
1430
|
+
{ layer: "page-builder", name: "Beaver Builder", match: /\/wp-content\/plugins\/beaver-builder/ },
|
|
1431
|
+
{ layer: "page-builder", name: "Oxygen", match: /\/wp-content\/plugins\/oxygen\// },
|
|
1432
|
+
// Stores.
|
|
1433
|
+
{ layer: "ecommerce", name: "WooCommerce", match: /\/wp-content\/plugins\/woocommerce\// },
|
|
1434
|
+
{ layer: "ecommerce", name: "Shopify", match: /(?:^|\/\/|\.)shopify\.com\// },
|
|
1435
|
+
{ layer: "ecommerce", name: "BigCommerce", match: /(?:^|\/\/|\.)bigcommerce\.com\// },
|
|
1436
|
+
// Forms. Which one matters commercially: it is the thing an enquiry actually
|
|
1437
|
+
// travels through, and half of these have a free tier that silently drops mail.
|
|
1438
|
+
{ layer: "forms", name: "Gravity Forms", match: /\/plugins\/gravityforms\// },
|
|
1439
|
+
{ layer: "forms", name: "Contact Form 7", match: /\/plugins\/contact-form-7\// },
|
|
1440
|
+
{ layer: "forms", name: "WPForms", match: /\/plugins\/wpforms/ },
|
|
1441
|
+
{ layer: "forms", name: "Ninja Forms", match: /\/plugins\/ninja-forms\// },
|
|
1442
|
+
{ layer: "forms", name: "Typeform", match: /(?:^|\/\/|\.)typeform\.com\// },
|
|
1443
|
+
{ layer: "forms", name: "Jotform", match: /(?:^|\/\/|\.)jotform\.com\// },
|
|
1444
|
+
{ layer: "forms", name: "HubSpot Forms", match: /(?:^|\/\/|\.)hsforms\.net\// },
|
|
1445
|
+
{ layer: "forms", name: "Formspree", match: /(?:^|\/\/|\.)formspree\.io\// },
|
|
1446
|
+
// Analytics and tracking. Absence of ALL of these is its own check (T0-46);
|
|
1447
|
+
// here we are only naming what is there.
|
|
1448
|
+
{ layer: "analytics", name: "Google Tag Manager", match: /googletagmanager\.com\/gtm\.js/ },
|
|
1449
|
+
{ layer: "analytics", name: "Google Analytics 4", match: /googletagmanager\.com\/gtag\/js/ },
|
|
1450
|
+
{
|
|
1451
|
+
layer: "analytics",
|
|
1452
|
+
name: "Google Analytics (legacy)",
|
|
1453
|
+
match: /google-analytics\.com\/(?:analytics|ga)\.js/
|
|
1454
|
+
},
|
|
1455
|
+
{ layer: "analytics", name: "Meta Pixel", match: /connect\.facebook\.net\// },
|
|
1456
|
+
{ layer: "analytics", name: "LinkedIn Insight", match: /snap\.licdn\.com\// },
|
|
1457
|
+
{ layer: "analytics", name: "Hotjar", match: /(?:static|script)\.hotjar\.com\// },
|
|
1458
|
+
{ layer: "analytics", name: "Microsoft Clarity", match: /(?:^|\/\/|\.)clarity\.ms\// },
|
|
1459
|
+
{ layer: "analytics", name: "Intercom", match: /(?:widget|js)\.intercom(?:cdn)?\.(?:io|com)\// },
|
|
1460
|
+
{ layer: "analytics", name: "Drift", match: /js\.driftt\.com\// },
|
|
1461
|
+
{ layer: "analytics", name: "Plausible", match: /plausible\.io\/js\// },
|
|
1462
|
+
{ layer: "analytics", name: "Fathom", match: /cdn\.usefathom\.com\// },
|
|
1463
|
+
{ layer: "analytics", name: "Segment", match: /cdn\.segment\.(?:com|io)\// },
|
|
1464
|
+
// Headless CMSes, by the asset and API hosts they serve from. A modern
|
|
1465
|
+
// agency-built site is far likelier to be one of these than to be WordPress,
|
|
1466
|
+
// and naming only "SvelteKit and Netlify" back to such a client says we did
|
|
1467
|
+
// not look very hard.
|
|
1468
|
+
{ layer: "cms", name: "Prismic", match: /(?:^|\/\/|\.)prismic\.io\// },
|
|
1469
|
+
{ layer: "cms", name: "Contentful", match: /(?:^|\/\/|\.)(?:ctfassets\.net|contentful\.com)\// },
|
|
1470
|
+
{ layer: "cms", name: "Sanity", match: /(?:^|\/\/|\.)sanity\.io\// },
|
|
1471
|
+
{ layer: "cms", name: "Storyblok", match: /(?:^|\/\/|\.)storyblok\.com\// },
|
|
1472
|
+
{ layer: "cms", name: "DatoCMS", match: /(?:^|\/\/|\.)datocms-assets\.com\// },
|
|
1473
|
+
{ layer: "cms", name: "Strapi", match: /(?:^|\/\/|\.)strapi\.io\// },
|
|
1474
|
+
{ layer: "cms", name: "Craft CMS", match: /\/cpresources\// },
|
|
1475
|
+
// Fonts. Typekit is a script; Google Fonts arrives as a stylesheet link, so
|
|
1476
|
+
// it needs the `<link>` set — which is fed in above.
|
|
1477
|
+
{ layer: "fonts", name: "Adobe Fonts (Typekit)", match: /use\.typekit\.net\// },
|
|
1478
|
+
{ layer: "fonts", name: "Google Fonts", match: /fonts\.(?:googleapis|gstatic)\.com\// }
|
|
1479
|
+
];
|
|
1480
|
+
var GENERATOR_SIGNATURES = [
|
|
1481
|
+
{ layer: "cms", name: "WordPress", match: /wordpress/i },
|
|
1482
|
+
{ layer: "cms", name: "Drupal", match: /drupal/i },
|
|
1483
|
+
{ layer: "cms", name: "Joomla", match: /joomla/i },
|
|
1484
|
+
{ layer: "cms", name: "Webflow", match: /webflow/i },
|
|
1485
|
+
{ layer: "cms", name: "Squarespace", match: /squarespace/i },
|
|
1486
|
+
{ layer: "cms", name: "Wix", match: /wix\.com/i },
|
|
1487
|
+
{ layer: "cms", name: "Ghost", match: /ghost/i },
|
|
1488
|
+
{ layer: "cms", name: "Shopify", match: /shopify/i },
|
|
1489
|
+
{ layer: "page-builder", name: "Elementor", match: /elementor/i },
|
|
1490
|
+
{ layer: "framework", name: "Hugo", match: /hugo/i },
|
|
1491
|
+
{ layer: "framework", name: "Jekyll", match: /jekyll/i },
|
|
1492
|
+
{ layer: "framework", name: "Gatsby", match: /gatsby/i },
|
|
1493
|
+
{ layer: "framework", name: "Next.js", match: /next\.js/i }
|
|
1494
|
+
];
|
|
1495
|
+
var HEADER_SIGNATURES = [
|
|
1496
|
+
{ name: "Cloudflare", header: "cf-ray" },
|
|
1497
|
+
{ name: "Netlify", header: "x-nf-request-id" },
|
|
1498
|
+
{ name: "Vercel", header: "x-vercel-id" },
|
|
1499
|
+
{ name: "AWS CloudFront", header: "x-amz-cf-id" },
|
|
1500
|
+
{ name: "Fastly", header: "x-served-by" },
|
|
1501
|
+
{ name: "GitHub Pages", header: "x-github-request-id" },
|
|
1502
|
+
{ name: "Pantheon", header: "x-pantheon-styx-hostname" },
|
|
1503
|
+
{ name: "WP Engine", header: "x-wpe-backend" },
|
|
1504
|
+
{ name: "Kinsta", header: "x-kinsta-cache" },
|
|
1505
|
+
{ name: "LiteSpeed", header: "x-litespeed-cache" },
|
|
1506
|
+
{ name: "Wix", header: "x-wix-request-id" },
|
|
1507
|
+
{ name: "Shopify", header: "x-shopid" },
|
|
1508
|
+
{ name: "Squarespace", header: "x-contextid" },
|
|
1509
|
+
{ name: "Nginx", header: "server", value: /nginx/i },
|
|
1510
|
+
{ name: "Apache", header: "server", value: /apache/i },
|
|
1511
|
+
{ name: "Microsoft IIS", header: "server", value: /microsoft-iis/i }
|
|
1512
|
+
];
|
|
1513
|
+
var MAX_EVIDENCE = 120;
|
|
1514
|
+
function receipt(url) {
|
|
1515
|
+
return url.length <= MAX_EVIDENCE ? url : `${url.slice(0, MAX_EVIDENCE - 1)}\u2026`;
|
|
1516
|
+
}
|
|
1517
|
+
function slugsFrom(urls, kind) {
|
|
1518
|
+
const re = new RegExp(`/wp-content/${kind}/([^/?#]+)/`);
|
|
1519
|
+
const found = /* @__PURE__ */ new Map();
|
|
1520
|
+
for (const url of urls) {
|
|
1521
|
+
const m = re.exec(url);
|
|
1522
|
+
const slug = m?.[1];
|
|
1523
|
+
if (slug && !found.has(slug)) found.set(slug, receipt(url));
|
|
1524
|
+
}
|
|
1525
|
+
return found;
|
|
1526
|
+
}
|
|
1527
|
+
function readStack(crawl) {
|
|
1528
|
+
const set = usablePages(crawl.pages);
|
|
1529
|
+
const headers = crawl.homeHeaders ?? {};
|
|
1530
|
+
const headersExamined = Object.keys(headers).length > 0;
|
|
1531
|
+
const readable = set.pages.filter((p) => p.extract.scriptSrcs !== void 0);
|
|
1532
|
+
const measured = readable.length > 0;
|
|
1533
|
+
const items = [];
|
|
1534
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1535
|
+
const add = (layer, name, evidence) => {
|
|
1536
|
+
const key = `${layer}::${name}`;
|
|
1537
|
+
if (seen.has(key)) return;
|
|
1538
|
+
seen.add(key);
|
|
1539
|
+
items.push({ layer, name, evidence });
|
|
1540
|
+
};
|
|
1541
|
+
const assetUrls = [];
|
|
1542
|
+
for (const { extract } of readable) {
|
|
1543
|
+
assetUrls.push(
|
|
1544
|
+
...extract.scriptSrcs ?? [],
|
|
1545
|
+
...extract.imageSrcs ?? [],
|
|
1546
|
+
...extract.inlineScriptUrls ?? [],
|
|
1547
|
+
...(extract.links ?? []).map((l) => l.href)
|
|
1548
|
+
);
|
|
1549
|
+
}
|
|
1550
|
+
for (const { extract } of readable) {
|
|
1551
|
+
const generator = extract.metas?.["generator"];
|
|
1552
|
+
if (!generator) continue;
|
|
1553
|
+
for (const sig of GENERATOR_SIGNATURES) {
|
|
1554
|
+
if (sig.match.test(generator)) {
|
|
1555
|
+
add(sig.layer, sig.name, `<meta name="generator"> ${receipt(generator)}`);
|
|
1556
|
+
}
|
|
1557
|
+
}
|
|
1558
|
+
}
|
|
1559
|
+
for (const sig of ASSET_SIGNATURES) {
|
|
1560
|
+
const hit = assetUrls.find((url) => sig.match.test(url));
|
|
1561
|
+
if (hit) add(sig.layer, sig.name, receipt(hit));
|
|
1562
|
+
}
|
|
1563
|
+
for (const [slug, evidence] of slugsFrom(assetUrls, "themes")) add("theme", slug, evidence);
|
|
1564
|
+
for (const [slug, evidence] of slugsFrom(assetUrls, "plugins")) add("plugin", slug, evidence);
|
|
1565
|
+
for (const sig of HEADER_SIGNATURES) {
|
|
1566
|
+
const value = headers[sig.header];
|
|
1567
|
+
if (value === void 0) continue;
|
|
1568
|
+
if (sig.value && !sig.value.test(value)) continue;
|
|
1569
|
+
add(
|
|
1570
|
+
"hosting",
|
|
1571
|
+
sig.name,
|
|
1572
|
+
sig.value ? `${sig.header}: ${receipt(value)}` : `${sig.header} header`
|
|
1573
|
+
);
|
|
1574
|
+
}
|
|
1575
|
+
return {
|
|
1576
|
+
measured,
|
|
1577
|
+
items: items.sort(
|
|
1578
|
+
(a, b) => LAYER_ORDER.indexOf(a.layer) - LAYER_ORDER.indexOf(b.layer) || a.name.localeCompare(b.name)
|
|
1579
|
+
),
|
|
1580
|
+
pagesExamined: readable.length,
|
|
1581
|
+
headersExamined
|
|
1582
|
+
};
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
// src/prospect/journey.ts
|
|
1586
|
+
function canonicalizeUrl(raw) {
|
|
1587
|
+
try {
|
|
1588
|
+
const u = new URL(raw);
|
|
1589
|
+
if (u.protocol !== "http:" && u.protocol !== "https:") return null;
|
|
1590
|
+
const path = u.pathname.replace(/\/+$/, "") || "/";
|
|
1591
|
+
return `${u.hostname.replace(/^www\./i, "").toLowerCase()}${path}`;
|
|
1592
|
+
} catch {
|
|
1593
|
+
return null;
|
|
1594
|
+
}
|
|
1595
|
+
}
|
|
1596
|
+
function resolveNavigable(href, pageUrl) {
|
|
1597
|
+
const trimmed = href.trim();
|
|
1598
|
+
if (!trimmed || trimmed.startsWith("#")) return null;
|
|
1599
|
+
if (/^(tel:|mailto:|javascript:|sms:|data:)/i.test(trimmed)) return null;
|
|
1600
|
+
try {
|
|
1601
|
+
return new URL(trimmed, pageUrl).toString();
|
|
1602
|
+
} catch {
|
|
1603
|
+
return null;
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
var TEL_HREF = /^tel:(.+)$/i;
|
|
1607
|
+
var MAILTO_HREF = /^mailto:([^?]+)/i;
|
|
1608
|
+
function affordancesOn(page, view) {
|
|
1609
|
+
const extract = view ?? page.rendered ?? page.raw;
|
|
1610
|
+
if (!extract) return [];
|
|
1611
|
+
const found = [];
|
|
1612
|
+
for (const anchor of extract.anchors ?? []) {
|
|
1613
|
+
const tel = TEL_HREF.exec(anchor.href);
|
|
1614
|
+
if (tel?.[1]) {
|
|
1615
|
+
found.push({ kind: "tel", page: page.url, detail: tel[1].trim() });
|
|
1616
|
+
continue;
|
|
1617
|
+
}
|
|
1618
|
+
const mail = MAILTO_HREF.exec(anchor.href);
|
|
1619
|
+
if (mail?.[1]) {
|
|
1620
|
+
found.push({ kind: "mailto", page: page.url, detail: mail[1].trim() });
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
for (const form of extract.forms ?? []) {
|
|
1624
|
+
if (form.kind !== "enquiry") continue;
|
|
1625
|
+
found.push({ kind: "form", page: page.url, detail: form.action ?? page.url });
|
|
1626
|
+
}
|
|
1627
|
+
return found;
|
|
1628
|
+
}
|
|
1629
|
+
function buildJourney(pages) {
|
|
1630
|
+
const usable = usablePages(pages);
|
|
1631
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
1632
|
+
for (const entry of usable.pages) {
|
|
1633
|
+
const key = canonicalizeUrl(entry.page.url);
|
|
1634
|
+
if (key) nodes.set(key, entry);
|
|
1635
|
+
}
|
|
1636
|
+
const affordances = [];
|
|
1637
|
+
const hasContact = /* @__PURE__ */ new Set();
|
|
1638
|
+
for (const [key, entry] of nodes) {
|
|
1639
|
+
const found = affordancesOn(entry.page, entry.extract);
|
|
1640
|
+
affordances.push(...found);
|
|
1641
|
+
if (found.length > 0) hasContact.add(key);
|
|
1642
|
+
}
|
|
1643
|
+
const outgoing = /* @__PURE__ */ new Map();
|
|
1644
|
+
const incoming = /* @__PURE__ */ new Map();
|
|
1645
|
+
for (const [key, entry] of nodes) {
|
|
1646
|
+
const out = /* @__PURE__ */ new Set();
|
|
1647
|
+
for (const anchor of entry.extract.anchors ?? []) {
|
|
1648
|
+
const abs = resolveNavigable(anchor.href, entry.page.url);
|
|
1649
|
+
if (!abs) continue;
|
|
1650
|
+
const target = canonicalizeUrl(abs);
|
|
1651
|
+
if (!target || target === key || !nodes.has(target)) continue;
|
|
1652
|
+
out.add(target);
|
|
1653
|
+
if (!incoming.has(target)) incoming.set(target, /* @__PURE__ */ new Set());
|
|
1654
|
+
incoming.get(target)?.add(key);
|
|
1655
|
+
}
|
|
1656
|
+
outgoing.set(key, out);
|
|
1657
|
+
}
|
|
1658
|
+
const distance = /* @__PURE__ */ new Map();
|
|
1659
|
+
let frontier = [...hasContact];
|
|
1660
|
+
for (const key of frontier) distance.set(key, 0);
|
|
1661
|
+
let depth = 0;
|
|
1662
|
+
while (frontier.length > 0) {
|
|
1663
|
+
depth += 1;
|
|
1664
|
+
const next = [];
|
|
1665
|
+
for (const key of frontier) {
|
|
1666
|
+
for (const source of incoming.get(key) ?? []) {
|
|
1667
|
+
if (distance.has(source)) continue;
|
|
1668
|
+
distance.set(source, depth);
|
|
1669
|
+
next.push(source);
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
frontier = next;
|
|
1673
|
+
}
|
|
1674
|
+
const journeys = [...nodes.entries()].map(([key, entry]) => ({
|
|
1675
|
+
url: entry.page.url,
|
|
1676
|
+
clicksToContact: usable.anchorsMeasured ? distance.get(key) ?? null : null,
|
|
1677
|
+
internalLinks: outgoing.get(key)?.size ?? 0
|
|
1678
|
+
}));
|
|
1679
|
+
const reachable = journeys.map((j) => j.clicksToContact).filter((d) => d !== null);
|
|
1680
|
+
return {
|
|
1681
|
+
affordances,
|
|
1682
|
+
pages: journeys,
|
|
1683
|
+
// Without recorded anchors there is no evidence either way, so there is no
|
|
1684
|
+
// finding — not "every page is a dead end", which is what reading the
|
|
1685
|
+
// absent array as an empty one used to produce.
|
|
1686
|
+
deadEnds: usable.anchorsMeasured ? journeys.filter((j) => j.clicksToContact === null).map((j) => j.url) : [],
|
|
1687
|
+
worstClicksToContact: reachable.length > 0 ? Math.max(...reachable) : null,
|
|
1688
|
+
pagesExamined: journeys.length,
|
|
1689
|
+
anchorsMeasured: usable.anchorsMeasured
|
|
1690
|
+
};
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
// src/prospect/http-probes.ts
|
|
1694
|
+
var MAX_SITEMAP_PROBES = 12;
|
|
1695
|
+
var MAX_EXTERNAL_PROBES = 15;
|
|
1696
|
+
var MAX_REDIRECT_PROBES = 10;
|
|
1697
|
+
var MAX_ORDINARY_HOPS = 2;
|
|
1698
|
+
var IMAGE_HEADER_BYTES = 32768;
|
|
1699
|
+
var MIN_OG_IMAGE_EDGE = 200;
|
|
1700
|
+
function classify(status) {
|
|
1701
|
+
if (status === null) return "unverified";
|
|
1702
|
+
if (status >= 200 && status < 300) return "ok";
|
|
1703
|
+
if (status >= 300 && status < 400) return "broken";
|
|
1704
|
+
if (status === 404 || status === 410) return "broken";
|
|
1705
|
+
return "unverified";
|
|
1706
|
+
}
|
|
1707
|
+
function safe(url) {
|
|
1708
|
+
try {
|
|
1709
|
+
const u = new URL(url);
|
|
1710
|
+
if (u.protocol !== "http:" && u.protocol !== "https:") return null;
|
|
1711
|
+
if (isPrivateOrLoopbackHost(u.hostname)) return null;
|
|
1712
|
+
return u.toString();
|
|
1713
|
+
} catch {
|
|
1714
|
+
return null;
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
function sameAddress(a, b) {
|
|
1718
|
+
if (!a || !b) return false;
|
|
1719
|
+
const key = (u) => canonicalizeUrl(u)?.replace(/^www\./, "").replace(/\/$/, "") ?? u;
|
|
1720
|
+
return key(a) === key(b);
|
|
1721
|
+
}
|
|
1722
|
+
function extractOf(page) {
|
|
1723
|
+
return page.rendered ?? page.raw;
|
|
1724
|
+
}
|
|
1725
|
+
function canonicalOf(body) {
|
|
1726
|
+
if (!body) return null;
|
|
1727
|
+
const head = new TextDecoder("utf-8", { fatal: false }).decode(body.slice(0, 65536));
|
|
1728
|
+
const tag = head.match(/<link\b[^>]*\brel\s*=\s*["']?canonical["']?[^>]*>/i)?.[0];
|
|
1729
|
+
const href = tag?.match(/\bhref\s*=\s*["']([^"']+)["']/i)?.[1];
|
|
1730
|
+
return href?.trim() || null;
|
|
1731
|
+
}
|
|
1732
|
+
function imageSize(bytes) {
|
|
1733
|
+
if (!bytes || bytes.length < 24) return null;
|
|
1734
|
+
const be32 = (o) => (bytes[o] << 24 | bytes[o + 1] << 16 | bytes[o + 2] << 8 | bytes[o + 3]) >>> 0;
|
|
1735
|
+
const be16 = (o) => bytes[o] << 8 | bytes[o + 1];
|
|
1736
|
+
const le16 = (o) => bytes[o] | bytes[o + 1] << 8;
|
|
1737
|
+
if (be32(0) === 2303741511 && be32(4) === 218765834) {
|
|
1738
|
+
return { width: be32(16), height: be32(20) };
|
|
1739
|
+
}
|
|
1740
|
+
if (bytes[0] === 71 && bytes[1] === 73 && bytes[2] === 70) {
|
|
1741
|
+
return { width: le16(6), height: le16(8) };
|
|
1742
|
+
}
|
|
1743
|
+
if (be32(0) === 1380533830 && be32(8) === 1464156752 && bytes.length >= 30) {
|
|
1744
|
+
const fourcc = String.fromCharCode(bytes[12], bytes[13], bytes[14], bytes[15]);
|
|
1745
|
+
if (fourcc === "VP8 ") return { width: le16(26) & 16383, height: le16(28) & 16383 };
|
|
1746
|
+
if (fourcc === "VP8L") {
|
|
1747
|
+
const b = bytes[21] | bytes[22] << 8 | bytes[23] << 16 | bytes[24] << 24;
|
|
1748
|
+
return { width: (b & 16383) + 1, height: (b >> 14 & 16383) + 1 };
|
|
1749
|
+
}
|
|
1750
|
+
if (fourcc === "VP8X") {
|
|
1751
|
+
const dim = (o) => 1 + (bytes[o] | bytes[o + 1] << 8 | bytes[o + 2] << 16);
|
|
1752
|
+
return { width: dim(24), height: dim(27) };
|
|
1753
|
+
}
|
|
1754
|
+
return null;
|
|
1755
|
+
}
|
|
1756
|
+
if (bytes[0] === 255 && bytes[1] === 216) {
|
|
1757
|
+
let i = 2;
|
|
1758
|
+
while (i + 9 < bytes.length) {
|
|
1759
|
+
if (bytes[i] !== 255) {
|
|
1760
|
+
i++;
|
|
1761
|
+
continue;
|
|
1762
|
+
}
|
|
1763
|
+
const marker = bytes[i + 1];
|
|
1764
|
+
if (marker === 216 || marker >= 208 && marker <= 217) {
|
|
1765
|
+
i += 2;
|
|
1766
|
+
continue;
|
|
1767
|
+
}
|
|
1768
|
+
if (marker === 218) return null;
|
|
1769
|
+
const len = be16(i + 2);
|
|
1770
|
+
if (marker >= 192 && marker <= 207 && marker !== 196 && marker !== 200 && marker !== 204) {
|
|
1771
|
+
return { width: be16(i + 7), height: be16(i + 5) };
|
|
1772
|
+
}
|
|
1773
|
+
if (len < 2) return null;
|
|
1774
|
+
i += 2 + len;
|
|
1775
|
+
}
|
|
1776
|
+
}
|
|
1777
|
+
return null;
|
|
1778
|
+
}
|
|
1779
|
+
function empty(measured) {
|
|
1780
|
+
return {
|
|
1781
|
+
measured,
|
|
1782
|
+
requests: 0,
|
|
1783
|
+
favicon: void 0,
|
|
1784
|
+
httpUpgrade: void 0,
|
|
1785
|
+
homeSamples: void 0,
|
|
1786
|
+
trailingSlash: void 0,
|
|
1787
|
+
indexAlias: void 0,
|
|
1788
|
+
caseAlias: void 0,
|
|
1789
|
+
sitemapUrls: void 0,
|
|
1790
|
+
externalLinks: void 0,
|
|
1791
|
+
ogImage: void 0,
|
|
1792
|
+
logo: void 0,
|
|
1793
|
+
redirectChains: void 0
|
|
1794
|
+
};
|
|
1795
|
+
}
|
|
1796
|
+
async function probeHttp(crawl, deps) {
|
|
1797
|
+
const origin = (() => {
|
|
1798
|
+
const ok = safe(crawl.origin);
|
|
1799
|
+
return ok ? new URL(ok).origin : null;
|
|
1800
|
+
})();
|
|
1801
|
+
if (!origin) return empty(false);
|
|
1802
|
+
const out = empty(true);
|
|
1803
|
+
const sleep2 = deps.sleep ?? sleep;
|
|
1804
|
+
let requests = 0;
|
|
1805
|
+
const ask = async (url, opts = { method: "HEAD" }) => {
|
|
1806
|
+
const target = safe(url);
|
|
1807
|
+
if (!target) return null;
|
|
1808
|
+
if (requests > 0 && deps.delayMs > 0) await sleep2(deps.delayMs);
|
|
1809
|
+
requests++;
|
|
1810
|
+
try {
|
|
1811
|
+
return await deps.request(target, opts);
|
|
1812
|
+
} catch (err) {
|
|
1813
|
+
return {
|
|
1814
|
+
status: null,
|
|
1815
|
+
headers: {},
|
|
1816
|
+
finalUrl: null,
|
|
1817
|
+
hops: null,
|
|
1818
|
+
body: null,
|
|
1819
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1820
|
+
};
|
|
1821
|
+
}
|
|
1822
|
+
};
|
|
1823
|
+
const home = crawl.pages[0];
|
|
1824
|
+
const homeExtract = home ? extractOf(home) : null;
|
|
1825
|
+
{
|
|
1826
|
+
const declared = homeExtract?.links?.find(
|
|
1827
|
+
(l) => /(^|\s)(icon|shortcut icon|apple-touch-icon)(\s|$)/i.test(l.rel ?? "")
|
|
1828
|
+
);
|
|
1829
|
+
const href = declared?.href ? resolveNavigable(declared.href, home?.url ?? origin) : null;
|
|
1830
|
+
const url = href ?? `${origin}/favicon.ico`;
|
|
1831
|
+
const res = await ask(url, { method: "GET", rangeBytes: 2048 });
|
|
1832
|
+
if (res) {
|
|
1833
|
+
const type = res.headers["content-type"] ?? "";
|
|
1834
|
+
const verdict = classify(res.status);
|
|
1835
|
+
out.favicon = {
|
|
1836
|
+
url,
|
|
1837
|
+
declared: Boolean(href),
|
|
1838
|
+
// A 200 that hands back the HTML of a soft-404 page is not an icon.
|
|
1839
|
+
// The content type is what separates "served" from "answered".
|
|
1840
|
+
verdict: verdict === "ok" && /^(image|text\/xml|application\/xml)/i.test(type) ? "ok" : verdict === "ok" ? "broken" : verdict
|
|
1841
|
+
};
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
{
|
|
1845
|
+
const insecure = origin.replace(/^https:/, "http:");
|
|
1846
|
+
if (insecure !== origin) {
|
|
1847
|
+
const res = await ask(insecure, { method: "HEAD" });
|
|
1848
|
+
if (res && res.status !== null) {
|
|
1849
|
+
out.httpUpgrade = { hops: res.hops, https: (res.finalUrl ?? "").startsWith("https:") };
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
{
|
|
1854
|
+
const a = await ask(origin, { method: "HEAD" });
|
|
1855
|
+
const b = await ask(origin, { method: "HEAD" });
|
|
1856
|
+
if (a && b) out.homeSamples = [a.status, b.status];
|
|
1857
|
+
}
|
|
1858
|
+
{
|
|
1859
|
+
const sample = crawl.pages.map((p) => p.url).find((u) => {
|
|
1860
|
+
try {
|
|
1861
|
+
const path = new URL(u).pathname;
|
|
1862
|
+
return path !== "/" && !/\.[a-z0-9]+$/i.test(path);
|
|
1863
|
+
} catch {
|
|
1864
|
+
return false;
|
|
1865
|
+
}
|
|
1866
|
+
});
|
|
1867
|
+
if (sample) {
|
|
1868
|
+
const bare = sample.replace(/\/+$/, "");
|
|
1869
|
+
const slashed = `${bare}/`;
|
|
1870
|
+
const [a, b] = [
|
|
1871
|
+
await ask(bare, { method: "GET", rangeBytes: 65536 }),
|
|
1872
|
+
await ask(slashed, { method: "GET", rangeBytes: 65536 })
|
|
1873
|
+
];
|
|
1874
|
+
if (a && b && a.status !== null && b.status !== null) {
|
|
1875
|
+
const bothServed = classify(a.status) === "ok" && classify(b.status) === "ok";
|
|
1876
|
+
if (!bothServed) {
|
|
1877
|
+
out.trailingSlash = {
|
|
1878
|
+
path: new URL(bare).pathname,
|
|
1879
|
+
settled: true,
|
|
1880
|
+
detail: "one form redirects to the other"
|
|
1881
|
+
};
|
|
1882
|
+
} else if (a.finalUrl !== null && a.finalUrl === b.finalUrl) {
|
|
1883
|
+
out.trailingSlash = {
|
|
1884
|
+
path: new URL(bare).pathname,
|
|
1885
|
+
settled: true,
|
|
1886
|
+
detail: "both forms resolve to the same address"
|
|
1887
|
+
};
|
|
1888
|
+
} else {
|
|
1889
|
+
const ca = canonicalOf(a.body);
|
|
1890
|
+
const cb = canonicalOf(b.body);
|
|
1891
|
+
const agree = sameAddress(ca, cb);
|
|
1892
|
+
out.trailingSlash = {
|
|
1893
|
+
path: new URL(bare).pathname,
|
|
1894
|
+
settled: agree,
|
|
1895
|
+
detail: agree ? "both forms answer, and both declare the same canonical" : "both forms answer with no canonical to settle which is the real one"
|
|
1896
|
+
};
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
{
|
|
1902
|
+
const alias = `${origin}/index.html`;
|
|
1903
|
+
const res = await ask(alias, { method: "GET", rangeBytes: 65536 });
|
|
1904
|
+
if (res && res.status !== null) {
|
|
1905
|
+
if (classify(res.status) !== "ok") {
|
|
1906
|
+
out.indexAlias = null;
|
|
1907
|
+
} else if (sameAddress(res.finalUrl, origin)) {
|
|
1908
|
+
out.indexAlias = { duplicate: false, detail: "it redirects to the homepage" };
|
|
1909
|
+
} else {
|
|
1910
|
+
const canonical = canonicalOf(res.body);
|
|
1911
|
+
const points = sameAddress(canonical, origin);
|
|
1912
|
+
out.indexAlias = {
|
|
1913
|
+
duplicate: !points,
|
|
1914
|
+
detail: points ? "it answers, and declares the homepage canonical" : "it answers with the homepage's content and does not point back at it"
|
|
1915
|
+
};
|
|
1916
|
+
}
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
{
|
|
1920
|
+
const sample = crawl.pages.map((p) => p.url).find((u) => {
|
|
1921
|
+
try {
|
|
1922
|
+
return /\/[a-z][a-z0-9-]*\/?$/.test(new URL(u).pathname);
|
|
1923
|
+
} catch {
|
|
1924
|
+
return false;
|
|
1925
|
+
}
|
|
1926
|
+
});
|
|
1927
|
+
if (sample) {
|
|
1928
|
+
const url = new URL(sample);
|
|
1929
|
+
const flipped = url.pathname.replace(/\/([a-z])/g, (_m, c) => `/${c.toUpperCase()}`);
|
|
1930
|
+
url.pathname = flipped;
|
|
1931
|
+
const res = await ask(url.toString(), { method: "GET", rangeBytes: 65536 });
|
|
1932
|
+
if (res && res.status !== null) {
|
|
1933
|
+
if (classify(res.status) !== "ok") {
|
|
1934
|
+
out.caseAlias = null;
|
|
1935
|
+
} else {
|
|
1936
|
+
const canonical = canonicalOf(res.body);
|
|
1937
|
+
const points = sameAddress(canonical, sample) || sameAddress(res.finalUrl, sample);
|
|
1938
|
+
out.caseAlias = {
|
|
1939
|
+
path: flipped,
|
|
1940
|
+
duplicate: !points,
|
|
1941
|
+
detail: points ? "it resolves back to the lower-case address" : "it answers as its own page, so the same content has two addresses"
|
|
1942
|
+
};
|
|
1943
|
+
}
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
}
|
|
1947
|
+
const sitemapSample = crawl.sitemap.sample ?? [];
|
|
1948
|
+
if (!crawl.sitemap.present || sitemapSample.length === 0) {
|
|
1949
|
+
out.sitemapUrls = crawl.sitemap.present ? void 0 : null;
|
|
1950
|
+
} else {
|
|
1951
|
+
const step = Math.max(1, Math.floor(sitemapSample.length / MAX_SITEMAP_PROBES));
|
|
1952
|
+
const picked = sitemapSample.filter((_, i) => i % step === 0).slice(0, MAX_SITEMAP_PROBES);
|
|
1953
|
+
const broken = [];
|
|
1954
|
+
await pacedEach(picked, 0, async (url) => {
|
|
1955
|
+
const res = await ask(url, { method: "HEAD" });
|
|
1956
|
+
if (!res) return;
|
|
1957
|
+
const verdict = classify(res.status);
|
|
1958
|
+
if (verdict === "broken") broken.push({ url, status: res.status, verdict });
|
|
1959
|
+
});
|
|
1960
|
+
out.sitemapUrls = { checked: picked.length, total: crawl.sitemap.urlCount, broken };
|
|
1961
|
+
}
|
|
1962
|
+
{
|
|
1963
|
+
const originKey = canonicalizeUrl(origin)?.split("/")[0] ?? "";
|
|
1964
|
+
const external = /* @__PURE__ */ new Set();
|
|
1965
|
+
for (const page of crawl.pages) {
|
|
1966
|
+
for (const a of extractOf(page)?.anchors ?? []) {
|
|
1967
|
+
const abs = resolveNavigable(a.href, page.url);
|
|
1968
|
+
if (!abs || !safe(abs)) continue;
|
|
1969
|
+
const host = canonicalizeUrl(abs)?.split("/")[0] ?? "";
|
|
1970
|
+
if (host && host !== originKey) external.add(abs);
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1973
|
+
const all = [...external];
|
|
1974
|
+
if (all.length === 0) {
|
|
1975
|
+
out.externalLinks = null;
|
|
1976
|
+
} else {
|
|
1977
|
+
const picked = all.slice(0, MAX_EXTERNAL_PROBES);
|
|
1978
|
+
const broken = [];
|
|
1979
|
+
await pacedEach(picked, 0, async (url) => {
|
|
1980
|
+
const res = await ask(url, { method: "HEAD" });
|
|
1981
|
+
if (!res) return;
|
|
1982
|
+
const verdict = classify(res.status);
|
|
1983
|
+
if (verdict === "broken") broken.push({ url, status: res.status, verdict });
|
|
1984
|
+
});
|
|
1985
|
+
out.externalLinks = { checked: picked.length, total: all.length, broken };
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
{
|
|
1989
|
+
const declared = homeExtract?.social?.["og:image"] ?? null;
|
|
1990
|
+
const url = declared ? resolveNavigable(declared, home?.url ?? origin) : null;
|
|
1991
|
+
if (!url) {
|
|
1992
|
+
out.ogImage = null;
|
|
1993
|
+
} else {
|
|
1994
|
+
const res = await ask(url, { method: "GET", rangeBytes: IMAGE_HEADER_BYTES });
|
|
1995
|
+
if (res) {
|
|
1996
|
+
const size = imageSize(res.body);
|
|
1997
|
+
out.ogImage = {
|
|
1998
|
+
url,
|
|
1999
|
+
verdict: classify(res.status),
|
|
2000
|
+
width: size?.width ?? null,
|
|
2001
|
+
height: size?.height ?? null
|
|
2002
|
+
};
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
{
|
|
2007
|
+
const images = homeExtract?.imageSrcs;
|
|
2008
|
+
const byName = images?.find((src) => /logo/i.test(src));
|
|
2009
|
+
const chosen = byName ?? images?.[0] ?? null;
|
|
2010
|
+
const url = chosen ? resolveNavigable(chosen, home?.url ?? origin) : null;
|
|
2011
|
+
if (!images) {
|
|
2012
|
+
out.logo = void 0;
|
|
2013
|
+
} else if (!url) {
|
|
2014
|
+
out.logo = null;
|
|
2015
|
+
} else {
|
|
2016
|
+
const res = await ask(url, { method: "HEAD" });
|
|
2017
|
+
if (res) {
|
|
2018
|
+
out.logo = {
|
|
2019
|
+
url,
|
|
2020
|
+
// Named, because "the logo" is a guess and the reader deserves to
|
|
2021
|
+
// know which guess we made before they act on the verdict.
|
|
2022
|
+
how: byName ? "the first image with 'logo' in its file name" : "the first image on the homepage",
|
|
2023
|
+
verdict: classify(res.status)
|
|
2024
|
+
};
|
|
2025
|
+
}
|
|
2026
|
+
}
|
|
2027
|
+
}
|
|
2028
|
+
{
|
|
2029
|
+
const originKey = canonicalizeUrl(origin)?.split("/")[0] ?? "";
|
|
2030
|
+
const internal = /* @__PURE__ */ new Set();
|
|
2031
|
+
for (const page of crawl.pages) {
|
|
2032
|
+
for (const a of extractOf(page)?.anchors ?? []) {
|
|
2033
|
+
const abs = resolveNavigable(a.href, page.url);
|
|
2034
|
+
if (!abs || !safe(abs)) continue;
|
|
2035
|
+
const host = canonicalizeUrl(abs)?.split("/")[0] ?? "";
|
|
2036
|
+
if (host === originKey && !sameAddress(abs, page.url)) internal.add(abs);
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
2039
|
+
const all = [...internal];
|
|
2040
|
+
const step = Math.max(1, Math.floor(all.length / MAX_REDIRECT_PROBES));
|
|
2041
|
+
const picked = all.filter((_, i) => i % step === 0).slice(0, MAX_REDIRECT_PROBES);
|
|
2042
|
+
if (picked.length > 0) {
|
|
2043
|
+
const chained = [];
|
|
2044
|
+
await pacedEach(picked, 0, async (url) => {
|
|
2045
|
+
const res = await ask(url, { method: "HEAD" });
|
|
2046
|
+
if (res?.hops != null && res.hops > MAX_ORDINARY_HOPS) {
|
|
2047
|
+
chained.push({ url, hops: res.hops });
|
|
2048
|
+
}
|
|
2049
|
+
});
|
|
2050
|
+
out.redirectChains = { checked: picked.length, chained };
|
|
2051
|
+
}
|
|
2052
|
+
}
|
|
2053
|
+
out.requests = requests;
|
|
2054
|
+
return out;
|
|
2055
|
+
}
|
|
2056
|
+
var MAX_HOPS = 6;
|
|
2057
|
+
function defaultHttpProbeDeps(userAgent, fetchImpl = fetch) {
|
|
2058
|
+
return {
|
|
2059
|
+
delayMs: 150,
|
|
2060
|
+
async request(url, opts) {
|
|
2061
|
+
const headers = {
|
|
2062
|
+
"user-agent": userAgent,
|
|
2063
|
+
accept: "text/html,image/*,*/*"
|
|
2064
|
+
};
|
|
2065
|
+
if (opts.rangeBytes) headers.range = `bytes=0-${opts.rangeBytes - 1}`;
|
|
2066
|
+
let current = url;
|
|
2067
|
+
let hops = 0;
|
|
2068
|
+
for (; ; ) {
|
|
2069
|
+
if (!safe(current)) {
|
|
2070
|
+
return { status: null, headers: {}, finalUrl: null, hops, body: null, error: "refused" };
|
|
2071
|
+
}
|
|
2072
|
+
const res = await fetchImpl(current, {
|
|
2073
|
+
method: opts.method,
|
|
2074
|
+
headers,
|
|
2075
|
+
redirect: "manual",
|
|
2076
|
+
signal: AbortSignal.timeout(15e3)
|
|
2077
|
+
});
|
|
2078
|
+
const flat = Object.fromEntries(
|
|
2079
|
+
[...res.headers].map(([k, v]) => [k.toLowerCase(), v])
|
|
2080
|
+
);
|
|
2081
|
+
const location = flat["location"];
|
|
2082
|
+
if (res.status >= 300 && res.status < 400 && location && hops < MAX_HOPS) {
|
|
2083
|
+
await res.body?.cancel();
|
|
2084
|
+
const next = resolveNavigable(location, current);
|
|
2085
|
+
if (!next) break;
|
|
2086
|
+
current = next;
|
|
2087
|
+
hops++;
|
|
2088
|
+
continue;
|
|
2089
|
+
}
|
|
2090
|
+
let body = null;
|
|
2091
|
+
if (opts.rangeBytes) {
|
|
2092
|
+
const buf = await res.arrayBuffer();
|
|
2093
|
+
body = new Uint8Array(buf.slice(0, opts.rangeBytes));
|
|
2094
|
+
} else {
|
|
2095
|
+
await res.body?.cancel();
|
|
2096
|
+
}
|
|
2097
|
+
return { status: res.status, headers: flat, finalUrl: current, hops, body, error: null };
|
|
2098
|
+
}
|
|
2099
|
+
return {
|
|
2100
|
+
status: null,
|
|
2101
|
+
headers: {},
|
|
2102
|
+
finalUrl: current,
|
|
2103
|
+
hops,
|
|
2104
|
+
body: null,
|
|
2105
|
+
error: "redirect loop"
|
|
2106
|
+
};
|
|
2107
|
+
}
|
|
2108
|
+
};
|
|
2109
|
+
}
|
|
2110
|
+
|
|
2111
|
+
// src/prospect/site-checks.ts
|
|
2112
|
+
function countOf(n, singular, plural) {
|
|
2113
|
+
return n === 1 ? `the one ${singular}` : `all ${n} ${plural}`;
|
|
2114
|
+
}
|
|
2115
|
+
function tally(checks) {
|
|
2116
|
+
const passed = checks.filter((c) => c.status === "pass").length;
|
|
2117
|
+
const failed = checks.filter((c) => c.status === "fail").length;
|
|
2118
|
+
return { passed, failed, total: passed + failed };
|
|
2119
|
+
}
|
|
2120
|
+
function check(key, label, why, scope, ok, evidence) {
|
|
2121
|
+
return { key, label, status: ok ? "pass" : "fail", evidence, why, scope };
|
|
2122
|
+
}
|
|
2123
|
+
function skip(key, label, why, scope, note) {
|
|
2124
|
+
return { key, label, status: "not-applicable", evidence: note, why, scope };
|
|
2125
|
+
}
|
|
2126
|
+
function unknown(key, label, why, scope, note = null) {
|
|
2127
|
+
return { key, label, status: "unmeasured", evidence: note, why, scope };
|
|
2128
|
+
}
|
|
2129
|
+
var VAGUE_LINK_TEXT = /* @__PURE__ */ new Set([
|
|
2130
|
+
"click here",
|
|
2131
|
+
"here",
|
|
2132
|
+
"read more",
|
|
2133
|
+
"learn more",
|
|
2134
|
+
"more",
|
|
2135
|
+
"more info",
|
|
2136
|
+
"more information",
|
|
2137
|
+
"find out more",
|
|
2138
|
+
"this",
|
|
2139
|
+
"link",
|
|
2140
|
+
"this link",
|
|
2141
|
+
"continue",
|
|
2142
|
+
"go"
|
|
2143
|
+
]);
|
|
2144
|
+
var LOCAL_HOST = /^(localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1\]|.*\.local|.*\.test|.*\.localhost)$/i;
|
|
2145
|
+
var PLACEHOLDER_SOCIAL = /^(?:https?:\/\/)?(?:www\.)?(?:facebook|twitter|x|instagram|linkedin|youtube|tiktok|pinterest)\.com\/?(?:(?:in|company|user|channel|@)\/)?(?:yourhandle|username|yourusername|yourpage|yourcompany|your-business|profile\.php|home)?\/?$/i;
|
|
2146
|
+
var DIALABLE_TEL = /^tel:\+?[0-9().\-\s]{6,}$/i;
|
|
2147
|
+
function anchorChecks(pages, origin) {
|
|
2148
|
+
const out = [];
|
|
2149
|
+
const measured = pages.length > 0 && pages.every((p) => p.extract.anchors !== void 0);
|
|
2150
|
+
const WHY_DEAD = "A link that goes nowhere reads as a broken site to the person who clicks it.";
|
|
2151
|
+
if (!measured) {
|
|
2152
|
+
return [
|
|
2153
|
+
unknown("dead-links", "Links that go nowhere", WHY_DEAD, "quick"),
|
|
2154
|
+
unknown(
|
|
2155
|
+
"staging-links",
|
|
2156
|
+
"Links pointing at a development address",
|
|
2157
|
+
"A link to localhost or a staging host takes the visitor to a browser error page with your name on it.",
|
|
2158
|
+
"quick"
|
|
2159
|
+
),
|
|
2160
|
+
unknown(
|
|
2161
|
+
"insecure-links",
|
|
2162
|
+
"Links back to your own http address",
|
|
2163
|
+
"A link to the http:// spelling of your own address makes the visitor ask for the page in the clear before your redirect sends them back, and costs them a round trip to do it.",
|
|
2164
|
+
"quick"
|
|
2165
|
+
),
|
|
2166
|
+
unknown(
|
|
2167
|
+
"link-text",
|
|
2168
|
+
"Links that say where they go",
|
|
2169
|
+
"\u201CRead more\u201D tells a visitor nothing, and it is the whole label a screen reader announces.",
|
|
2170
|
+
"content"
|
|
2171
|
+
),
|
|
2172
|
+
unknown(
|
|
2173
|
+
"tel-dialable",
|
|
2174
|
+
"Phone links a phone can dial",
|
|
2175
|
+
"A tel: link that is not a number does nothing at all when tapped.",
|
|
2176
|
+
"quick"
|
|
2177
|
+
),
|
|
2178
|
+
unknown(
|
|
2179
|
+
"social-placeholders",
|
|
2180
|
+
"Social links that go to your accounts",
|
|
2181
|
+
"An unedited theme placeholder sends visitors to a login page or a stranger.",
|
|
2182
|
+
"quick"
|
|
2183
|
+
),
|
|
2184
|
+
unknown(
|
|
2185
|
+
"nav-consistency",
|
|
2186
|
+
"Navigation that agrees with itself",
|
|
2187
|
+
"The same menu item leading to two different pages is confusing in a way visitors blame on you.",
|
|
2188
|
+
"quick"
|
|
2189
|
+
)
|
|
2190
|
+
];
|
|
2191
|
+
}
|
|
2192
|
+
const all = [];
|
|
2193
|
+
for (const p of pages) for (const a of p.extract.anchors ?? []) all.push({ a, page: p.url });
|
|
2194
|
+
const trueTotal = pages.reduce((n, p) => n + (p.extract.anchorCount ?? 0), 0);
|
|
2195
|
+
const ofTrue = trueTotal > all.length ? ` \u2014 of ${trueTotal} on these pages` : "";
|
|
2196
|
+
const dead = all.filter(({ a }) => /^javascript:\s*(void\s*\(\s*0\s*\)|;)?\s*$/i.test(a.href));
|
|
2197
|
+
out.push(
|
|
2198
|
+
check(
|
|
2199
|
+
"dead-links",
|
|
2200
|
+
"Links that go nowhere",
|
|
2201
|
+
WHY_DEAD,
|
|
2202
|
+
"quick",
|
|
2203
|
+
dead.length === 0,
|
|
2204
|
+
dead.length === 0 ? `none among the ${all.length} links we read` : `${dead.length} of ${all.length}: ${dead.slice(0, 3).map((d) => `\u201C${d.a.text || d.a.href}\u201D on ${d.page}`).join(", ")}`
|
|
2205
|
+
)
|
|
2206
|
+
);
|
|
2207
|
+
let ownHost = null;
|
|
2208
|
+
try {
|
|
2209
|
+
ownHost = new URL(origin).hostname.toLowerCase();
|
|
2210
|
+
} catch {
|
|
2211
|
+
ownHost = null;
|
|
2212
|
+
}
|
|
2213
|
+
const staging = all.filter(({ a, page }) => {
|
|
2214
|
+
const abs = resolveNavigable(a.href, page);
|
|
2215
|
+
if (!abs) return false;
|
|
2216
|
+
try {
|
|
2217
|
+
const host = new URL(abs).hostname.toLowerCase();
|
|
2218
|
+
return host !== ownHost && LOCAL_HOST.test(host);
|
|
2219
|
+
} catch {
|
|
2220
|
+
return false;
|
|
2221
|
+
}
|
|
2222
|
+
});
|
|
2223
|
+
out.push(
|
|
2224
|
+
check(
|
|
2225
|
+
"staging-links",
|
|
2226
|
+
"Links pointing at a development address",
|
|
2227
|
+
"A link to localhost or a staging host takes the visitor to a browser error page with your name on it.",
|
|
2228
|
+
"quick",
|
|
2229
|
+
staging.length === 0,
|
|
2230
|
+
staging.length === 0 ? "none" : `${staging.length} ${staging.length === 1 ? "link" : "links"}, including ${staging.slice(0, 3).map((s) => s.a.href).join(", ")}`
|
|
2231
|
+
)
|
|
2232
|
+
);
|
|
2233
|
+
if (origin.startsWith("https://")) {
|
|
2234
|
+
const insecure = all.filter(({ a }) => {
|
|
2235
|
+
const href = a.href.trim();
|
|
2236
|
+
if (!/^http:\/\//i.test(href)) return false;
|
|
2237
|
+
try {
|
|
2238
|
+
return new URL(href).hostname.replace(/^www\./i, "") === ownHost?.replace(/^www\./i, "");
|
|
2239
|
+
} catch {
|
|
2240
|
+
return false;
|
|
2241
|
+
}
|
|
2242
|
+
});
|
|
2243
|
+
out.push(
|
|
2244
|
+
check(
|
|
2245
|
+
"insecure-links",
|
|
2246
|
+
"Links back to your own http address",
|
|
2247
|
+
"A link to the http:// spelling of your own address makes the visitor ask for the page in the clear before your redirect sends them back, and costs them a round trip to do it.",
|
|
2248
|
+
"quick",
|
|
2249
|
+
insecure.length === 0,
|
|
2250
|
+
insecure.length === 0 ? "every link to your own pages stays on https" : `${insecure.length} ${insecure.length === 1 ? "link" : "links"} to your own http:// address, including ${insecure.slice(0, 3).map((s) => s.a.href).join(", ")}`
|
|
2251
|
+
)
|
|
2252
|
+
);
|
|
2253
|
+
} else {
|
|
2254
|
+
out.push(
|
|
2255
|
+
skip(
|
|
2256
|
+
"insecure-links",
|
|
2257
|
+
"Links back to your own http address",
|
|
2258
|
+
"A link to the http:// spelling of your own address makes the visitor ask for the page in the clear before your redirect sends them back, and costs them a round trip to do it.",
|
|
2259
|
+
"quick",
|
|
2260
|
+
"the site is not served over https, which is the finding above instead"
|
|
2261
|
+
)
|
|
2262
|
+
);
|
|
2263
|
+
}
|
|
2264
|
+
const labelled = all.every(({ a }) => a.ariaLabel !== void 0);
|
|
2265
|
+
const WHY_TEXT = "\u201CRead more\u201D tells a visitor nothing, and with no aria-label behind it, it is the whole label a screen reader announces when it lists the links on your page.";
|
|
2266
|
+
if (!labelled) {
|
|
2267
|
+
out.push(unknown("link-text", "Links that say where they go", WHY_TEXT, "content"));
|
|
2268
|
+
} else {
|
|
2269
|
+
const vague = all.filter(
|
|
2270
|
+
({ a }) => VAGUE_LINK_TEXT.has((a.ariaLabel || a.text).trim().toLowerCase())
|
|
2271
|
+
);
|
|
2272
|
+
const VAGUE_LIMIT = 5;
|
|
2273
|
+
out.push(
|
|
2274
|
+
check(
|
|
2275
|
+
"link-text",
|
|
2276
|
+
"Links that say where they go",
|
|
2277
|
+
WHY_TEXT,
|
|
2278
|
+
"content",
|
|
2279
|
+
vague.length <= VAGUE_LIMIT,
|
|
2280
|
+
vague.length === 0 ? `all ${all.length} links we read${ofTrue} name their destination` : `${vague.length} of the ${all.length} we read${ofTrue} say only \u201C${[...new Set(vague.map((v) => (v.a.ariaLabel || v.a.text).trim()))].slice(0, 3).join("\u201D, \u201C")}\u201D`
|
|
2281
|
+
)
|
|
2282
|
+
);
|
|
2283
|
+
}
|
|
2284
|
+
const tels = all.filter(({ a }) => /^tel:/i.test(a.href.trim()));
|
|
2285
|
+
const badTels = tels.filter(({ a }) => !DIALABLE_TEL.test(a.href.trim()));
|
|
2286
|
+
const WHY_TEL = "A tel: link that is not a number does nothing at all when it is tapped.";
|
|
2287
|
+
out.push(
|
|
2288
|
+
tels.length === 0 ? skip(
|
|
2289
|
+
"tel-dialable",
|
|
2290
|
+
"Phone links a phone can dial",
|
|
2291
|
+
WHY_TEL,
|
|
2292
|
+
"quick",
|
|
2293
|
+
"no tel: links on the pages we read"
|
|
2294
|
+
) : check(
|
|
2295
|
+
"tel-dialable",
|
|
2296
|
+
"Phone links a phone can dial",
|
|
2297
|
+
WHY_TEL,
|
|
2298
|
+
"quick",
|
|
2299
|
+
badTels.length === 0,
|
|
2300
|
+
badTels.length === 0 ? `all ${tels.length} dial a number` : badTels.map((b) => b.a.href).slice(0, 3).join(", ")
|
|
2301
|
+
)
|
|
2302
|
+
);
|
|
2303
|
+
const placeholders = all.filter(({ a }) => PLACEHOLDER_SOCIAL.test(a.href.trim()));
|
|
2304
|
+
out.push(
|
|
2305
|
+
check(
|
|
2306
|
+
"social-placeholders",
|
|
2307
|
+
"Social links that go to your accounts",
|
|
2308
|
+
"An unedited theme placeholder sends a visitor to a login page or to somebody else entirely.",
|
|
2309
|
+
"quick",
|
|
2310
|
+
placeholders.length === 0,
|
|
2311
|
+
placeholders.length === 0 ? "none left unedited" : placeholders.map((p) => p.a.href).slice(0, 3).join(", ")
|
|
2312
|
+
)
|
|
2313
|
+
);
|
|
2314
|
+
out.push(navConsistency(pages));
|
|
2315
|
+
return out;
|
|
2316
|
+
}
|
|
2317
|
+
function navConsistency(pages) {
|
|
2318
|
+
const WHY = "The same menu item leading to two different pages is the kind of confusion visitors blame on you rather than on the menu.";
|
|
2319
|
+
if (pages.length < 2) {
|
|
2320
|
+
return skip(
|
|
2321
|
+
"nav-consistency",
|
|
2322
|
+
"Navigation that agrees with itself",
|
|
2323
|
+
WHY,
|
|
2324
|
+
"quick",
|
|
2325
|
+
"only one page was read, so there is no shared navigation to compare"
|
|
2326
|
+
);
|
|
2327
|
+
}
|
|
2328
|
+
const perPage = pages.map(({ url, extract }) => {
|
|
2329
|
+
const m = /* @__PURE__ */ new Map();
|
|
2330
|
+
for (const a of extract.anchors ?? []) {
|
|
2331
|
+
const text = a.text.trim().toLowerCase();
|
|
2332
|
+
const abs = resolveNavigable(a.href, url);
|
|
2333
|
+
if (!text || !abs) continue;
|
|
2334
|
+
if (!m.has(text)) m.set(text, /* @__PURE__ */ new Set());
|
|
2335
|
+
m.get(text).add(abs);
|
|
2336
|
+
}
|
|
2337
|
+
return m;
|
|
2338
|
+
});
|
|
2339
|
+
const shared = [...perPage[0]?.keys() ?? []].filter(
|
|
2340
|
+
(text) => perPage.every((m) => m.has(text))
|
|
2341
|
+
);
|
|
2342
|
+
if (shared.length === 0) {
|
|
2343
|
+
return skip(
|
|
2344
|
+
"nav-consistency",
|
|
2345
|
+
"Navigation that agrees with itself",
|
|
2346
|
+
WHY,
|
|
2347
|
+
"quick",
|
|
2348
|
+
"no link text appears on every page, so we could not identify a shared navigation"
|
|
2349
|
+
);
|
|
2350
|
+
}
|
|
2351
|
+
const drifting = shared.filter((text) => {
|
|
2352
|
+
let common = null;
|
|
2353
|
+
for (const m of perPage) {
|
|
2354
|
+
const here = m.get(text) ?? /* @__PURE__ */ new Set();
|
|
2355
|
+
const kept = common === null ? [...here] : [...common].filter((t) => here.has(t));
|
|
2356
|
+
common = new Set(kept);
|
|
2357
|
+
if (common.size === 0) return true;
|
|
2358
|
+
}
|
|
2359
|
+
return common === null || common.size === 0;
|
|
2360
|
+
});
|
|
2361
|
+
return check(
|
|
2362
|
+
"nav-consistency",
|
|
2363
|
+
"Navigation that agrees with itself",
|
|
2364
|
+
WHY,
|
|
2365
|
+
"quick",
|
|
2366
|
+
drifting.length === 0,
|
|
2367
|
+
drifting.length === 0 ? `${shared.length} shared menu ${shared.length === 1 ? "item goes" : "items go"} to the same place on every page` : `${drifting.map((d) => `\u201C${d}\u201D`).slice(0, 3).join(
|
|
2368
|
+
", "
|
|
2369
|
+
)} ${drifting.length === 1 ? "leads" : "lead"} somewhere different depending on the page`
|
|
2370
|
+
);
|
|
2371
|
+
}
|
|
2372
|
+
var LOREM = /\b(lorem ipsum|dolor sit amet|consectetur adipiscing)\b/i;
|
|
2373
|
+
var PLACEHOLDER_COPY = /\b(your (?:text|content|title|headline|company name|business name) here|insert (?:text|content|your) |add your (?:text|content) here|edit this text|this is a sample|sample text|placeholder text|coming soon\.{3})/i;
|
|
2374
|
+
var TEMPLATE_LEAKAGE = /(\{\{[^}]{0,80}\}\}|\[object Object\]|<%=?[^%]{0,80}%>|\$\{[^}]{0,60}\})/;
|
|
2375
|
+
var MOJIBAKE = /(â€[™œ“”˜]|Ã[©¨¤¡«»]||Â[\s£©®])/;
|
|
2376
|
+
var UNDER_CONSTRUCTION = /\b(under construction|coming soon|site is being (?:built|updated)|check back soon)\b/i;
|
|
2377
|
+
function textChecks(pages, businessName) {
|
|
2378
|
+
const out = [];
|
|
2379
|
+
const firstMatch = (re) => {
|
|
2380
|
+
for (const p of pages) {
|
|
2381
|
+
const m = re.exec(p.extract.text);
|
|
2382
|
+
if (m) {
|
|
2383
|
+
const at = Math.max(0, m.index - 40);
|
|
2384
|
+
return { page: p.url, snippet: `\u2026${p.extract.text.slice(at, m.index + 60).trim()}\u2026` };
|
|
2385
|
+
}
|
|
2386
|
+
}
|
|
2387
|
+
return null;
|
|
2388
|
+
};
|
|
2389
|
+
const cases = [
|
|
2390
|
+
[
|
|
2391
|
+
"lorem",
|
|
2392
|
+
"Placeholder Latin left in the copy",
|
|
2393
|
+
"Lorem ipsum on a live page tells a visitor the site was never finished.",
|
|
2394
|
+
"quick",
|
|
2395
|
+
LOREM
|
|
2396
|
+
],
|
|
2397
|
+
[
|
|
2398
|
+
"placeholder-copy",
|
|
2399
|
+
"Theme placeholder text left in the copy",
|
|
2400
|
+
"\u201CYour text here\u201D on a live page reads as a site nobody has looked at since it was built.",
|
|
2401
|
+
"quick",
|
|
2402
|
+
PLACEHOLDER_COPY
|
|
2403
|
+
],
|
|
2404
|
+
[
|
|
2405
|
+
"template-leakage",
|
|
2406
|
+
"Template code showing as text",
|
|
2407
|
+
"A visitor sees the machinery instead of the sentence, which reads as broken rather than unfinished.",
|
|
2408
|
+
"quick",
|
|
2409
|
+
TEMPLATE_LEAKAGE
|
|
2410
|
+
],
|
|
2411
|
+
[
|
|
2412
|
+
"mojibake",
|
|
2413
|
+
"Text with broken characters",
|
|
2414
|
+
"Apostrophes and quotes rendering as \xE2\u20AC\u2122 is the most visible sign of a migration nobody checked, and it is a one-line fix at the source.",
|
|
2415
|
+
"quick",
|
|
2416
|
+
MOJIBAKE
|
|
2417
|
+
]
|
|
2418
|
+
];
|
|
2419
|
+
for (const [key, label, why, scope, re] of cases) {
|
|
2420
|
+
if (pages.length === 0) {
|
|
2421
|
+
out.push(unknown(key, label, why, scope));
|
|
2422
|
+
continue;
|
|
2423
|
+
}
|
|
2424
|
+
const hit = firstMatch(re);
|
|
2425
|
+
out.push(
|
|
2426
|
+
check(
|
|
2427
|
+
key,
|
|
2428
|
+
label,
|
|
2429
|
+
why,
|
|
2430
|
+
scope,
|
|
2431
|
+
hit === null,
|
|
2432
|
+
hit === null ? `none across the ${pages.length} pages we read` : `${hit.page}: ${hit.snippet}`
|
|
2433
|
+
)
|
|
2434
|
+
);
|
|
2435
|
+
}
|
|
2436
|
+
out.push(underConstruction(pages));
|
|
2437
|
+
out.push(nameInTitle(pages, businessName));
|
|
2438
|
+
return out;
|
|
2439
|
+
}
|
|
2440
|
+
function underConstruction(pages) {
|
|
2441
|
+
const KEY = "under-construction";
|
|
2442
|
+
const LABEL = "Pages that say they are unfinished";
|
|
2443
|
+
const WHY = "A live page announcing it is coming soon is a page better unlinked until it is not.";
|
|
2444
|
+
if (pages.length === 0) return unknown(KEY, LABEL, WHY, "content");
|
|
2445
|
+
for (const p of pages) {
|
|
2446
|
+
const headlines = [p.extract.title ?? "", ...p.extract.headings.map((h) => h.text)];
|
|
2447
|
+
for (const line of headlines) {
|
|
2448
|
+
const m = UNDER_CONSTRUCTION.exec(line);
|
|
2449
|
+
if (m) {
|
|
2450
|
+
return check(KEY, LABEL, WHY, "content", false, `${p.url}: \u201C${line.trim()}\u201D`);
|
|
2451
|
+
}
|
|
2452
|
+
}
|
|
2453
|
+
}
|
|
2454
|
+
return check(
|
|
2455
|
+
KEY,
|
|
2456
|
+
LABEL,
|
|
2457
|
+
WHY,
|
|
2458
|
+
"content",
|
|
2459
|
+
true,
|
|
2460
|
+
`no page announces itself unfinished, across the ${pages.length} we read`
|
|
2461
|
+
);
|
|
2462
|
+
}
|
|
2463
|
+
var LEADING_ARTICLES = /* @__PURE__ */ new Set(["the"]);
|
|
2464
|
+
function nameInTitle(pages, businessName) {
|
|
2465
|
+
const WHY = "The title is the line in a search result, a browser tab and a shared link. Without your name in it, all three are anonymous.";
|
|
2466
|
+
const LABEL = "Your name in the page title";
|
|
2467
|
+
if (!businessName) return unknown("name-in-title", LABEL, WHY, "quick");
|
|
2468
|
+
const home = pages[0];
|
|
2469
|
+
if (!home || !home.extract.title) {
|
|
2470
|
+
return unknown("name-in-title", LABEL, WHY, "quick");
|
|
2471
|
+
}
|
|
2472
|
+
const words = businessName.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((w) => w.length > 2 && !LEADING_ARTICLES.has(w));
|
|
2473
|
+
const head = words[0];
|
|
2474
|
+
const title = home.extract.title.toLowerCase();
|
|
2475
|
+
const hit = head !== void 0 && title.includes(head);
|
|
2476
|
+
return check("name-in-title", LABEL, WHY, "quick", hit, `\u201C${home.extract.title}\u201D`);
|
|
2477
|
+
}
|
|
2478
|
+
function headingChecks(pages, businessName) {
|
|
2479
|
+
const WHY_PRESENT = "The h1 is the page's headline. A page without one gives a reader \u2014 and an assistant looking for something to quote \u2014 nothing that says what the page is about.";
|
|
2480
|
+
const WHY_ONE = "When everything on the page is marked as the page's headline, none of it is, and whatever gets quoted back is whichever one came first in the markup.";
|
|
2481
|
+
if (pages.length === 0) {
|
|
2482
|
+
return [
|
|
2483
|
+
unknown("h1-present", "A headline on every page", WHY_PRESENT, "content"),
|
|
2484
|
+
unknown("h1-one-headline", "One headline, not a list of them", WHY_ONE, "content"),
|
|
2485
|
+
unknown(
|
|
2486
|
+
"h1-not-name",
|
|
2487
|
+
"Headlines that say what the page is about",
|
|
2488
|
+
"A headline that is just your company name on every page describes none of them.",
|
|
2489
|
+
"content"
|
|
2490
|
+
),
|
|
2491
|
+
unknown(
|
|
2492
|
+
"h1-distinct",
|
|
2493
|
+
"A different headline on each page",
|
|
2494
|
+
"Identical headlines make every page look like the same page, to a reader and to a search engine.",
|
|
2495
|
+
"content"
|
|
2496
|
+
)
|
|
2497
|
+
];
|
|
2498
|
+
}
|
|
2499
|
+
const counted = contentPages(pages);
|
|
2500
|
+
if (counted.length === 0) {
|
|
2501
|
+
const HIDDEN = "every page we read asks not to be indexed, which is reported under meta-noindex";
|
|
2502
|
+
return [
|
|
2503
|
+
skip("h1-present", "A headline on every page", WHY_PRESENT, "content", HIDDEN),
|
|
2504
|
+
skip("h1-one-headline", "One headline, not a list of them", WHY_ONE, "content", HIDDEN),
|
|
2505
|
+
skip(
|
|
2506
|
+
"h1-not-name",
|
|
2507
|
+
"Headlines that say what the page is about",
|
|
2508
|
+
"A headline that is just your company name on every page describes none of them.",
|
|
2509
|
+
"content",
|
|
2510
|
+
HIDDEN
|
|
2511
|
+
),
|
|
2512
|
+
skip(
|
|
2513
|
+
"h1-distinct",
|
|
2514
|
+
"A different headline on each page",
|
|
2515
|
+
"Identical headlines make every page look like the same page, to a reader and to a search engine.",
|
|
2516
|
+
"content",
|
|
2517
|
+
HIDDEN
|
|
2518
|
+
)
|
|
2519
|
+
];
|
|
2520
|
+
}
|
|
2521
|
+
const h1sPerPage = counted.map((p) => ({
|
|
2522
|
+
url: p.url,
|
|
2523
|
+
h1s: p.extract.headings.filter((h) => h.level === 1).map((h) => h.text.trim())
|
|
2524
|
+
}));
|
|
2525
|
+
const missing = h1sPerPage.filter((p) => p.h1s.length === 0);
|
|
2526
|
+
const out = [
|
|
2527
|
+
check(
|
|
2528
|
+
"h1-present",
|
|
2529
|
+
"A headline on every page",
|
|
2530
|
+
WHY_PRESENT,
|
|
2531
|
+
"content",
|
|
2532
|
+
missing.length === 0,
|
|
2533
|
+
missing.length === 0 ? `all ${counted.length} pages carry one` : `${missing.length} of ${counted.length}: ${missing.slice(0, 3).map((p) => p.url).join(", ")}`
|
|
2534
|
+
)
|
|
2535
|
+
];
|
|
2536
|
+
const CROWDED_AT = 3;
|
|
2537
|
+
const crowded = h1sPerPage.filter((p) => p.h1s.length >= CROWDED_AT);
|
|
2538
|
+
out.push(
|
|
2539
|
+
check(
|
|
2540
|
+
"h1-one-headline",
|
|
2541
|
+
"One headline, not a list of them",
|
|
2542
|
+
WHY_ONE,
|
|
2543
|
+
"content",
|
|
2544
|
+
crowded.length === 0,
|
|
2545
|
+
crowded.length === 0 ? `no page marks more than two of its lines as the headline, across the ${counted.length} we read` : crowded.slice(0, 3).map((p) => `${p.url} has ${p.h1s.length}`).join(", ")
|
|
2546
|
+
)
|
|
2547
|
+
);
|
|
2548
|
+
const WHY_NAME = "A headline that is only your company name describes none of the page below it.";
|
|
2549
|
+
const LABEL_NAME = "Headlines that say what the page is about";
|
|
2550
|
+
const named = h1sPerPage.filter((p) => p.h1s.length === 1);
|
|
2551
|
+
if (!businessName || named.length === 0) {
|
|
2552
|
+
out.push(unknown("h1-not-name", LABEL_NAME, WHY_NAME, "content"));
|
|
2553
|
+
} else {
|
|
2554
|
+
const bare = named.filter(
|
|
2555
|
+
(p) => p.h1s[0].toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "") === businessName.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "")
|
|
2556
|
+
);
|
|
2557
|
+
out.push(
|
|
2558
|
+
check(
|
|
2559
|
+
"h1-not-name",
|
|
2560
|
+
LABEL_NAME,
|
|
2561
|
+
WHY_NAME,
|
|
2562
|
+
"content",
|
|
2563
|
+
bare.length === 0,
|
|
2564
|
+
bare.length === 0 ? "every headline describes its page" : `${bare.length} ${bare.length === 1 ? "page's headline is" : "pages' headlines are"} just \u201C${businessName}\u201D`
|
|
2565
|
+
)
|
|
2566
|
+
);
|
|
2567
|
+
}
|
|
2568
|
+
const WHY_DISTINCT = "Identical headlines make every page look like the same page, to a reader scanning tabs and to a search engine choosing which one to show.";
|
|
2569
|
+
const LABEL_DISTINCT = "A different headline on each page";
|
|
2570
|
+
if (counted.length < 2) {
|
|
2571
|
+
out.push(
|
|
2572
|
+
skip(
|
|
2573
|
+
"h1-distinct",
|
|
2574
|
+
LABEL_DISTINCT,
|
|
2575
|
+
WHY_DISTINCT,
|
|
2576
|
+
"content",
|
|
2577
|
+
"only one page was read, so there is nothing to compare it with"
|
|
2578
|
+
)
|
|
2579
|
+
);
|
|
2580
|
+
} else {
|
|
2581
|
+
const distinct = counted.filter(
|
|
2582
|
+
(p) => p.extract.headings.filter((h) => h.level === 1).length === 1
|
|
2583
|
+
);
|
|
2584
|
+
if (distinct.length < 2) {
|
|
2585
|
+
out.push(
|
|
2586
|
+
skip(
|
|
2587
|
+
"h1-distinct",
|
|
2588
|
+
LABEL_DISTINCT,
|
|
2589
|
+
WHY_DISTINCT,
|
|
2590
|
+
"content",
|
|
2591
|
+
distinct.length === 0 ? "no page we read carries exactly one headline, which is reported above" : "the pages we read all declare the same address, so there is nothing to compare"
|
|
2592
|
+
)
|
|
2593
|
+
);
|
|
2594
|
+
} else {
|
|
2595
|
+
const texts = distinct.map(
|
|
2596
|
+
(p) => p.extract.headings.find((h) => h.level === 1).text.trim().toLowerCase()
|
|
2597
|
+
);
|
|
2598
|
+
const dupes = texts.filter((t, i) => texts.indexOf(t) !== i);
|
|
2599
|
+
out.push(
|
|
2600
|
+
check(
|
|
2601
|
+
"h1-distinct",
|
|
2602
|
+
LABEL_DISTINCT,
|
|
2603
|
+
WHY_DISTINCT,
|
|
2604
|
+
"content",
|
|
2605
|
+
dupes.length === 0,
|
|
2606
|
+
dupes.length === 0 ? `${texts.length} pages, ${new Set(texts).size} different headlines` : `\u201C${[...new Set(dupes)].slice(0, 2).join("\u201D, \u201C")}\u201D appears on more than one page`
|
|
2607
|
+
)
|
|
2608
|
+
);
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
return out;
|
|
2612
|
+
}
|
|
2613
|
+
function collectNodes(node, into, depth = 0) {
|
|
2614
|
+
if (depth > 8) return;
|
|
2615
|
+
if (Array.isArray(node)) {
|
|
2616
|
+
for (const n of node) collectNodes(n, into, depth + 1);
|
|
2617
|
+
return;
|
|
2618
|
+
}
|
|
2619
|
+
if (!node || typeof node !== "object") return;
|
|
2620
|
+
const obj = node;
|
|
2621
|
+
if (obj["@type"]) into.push(obj);
|
|
2622
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
2623
|
+
if (k === "@type") continue;
|
|
2624
|
+
if (v && typeof v === "object") collectNodes(v, into, depth + 1);
|
|
2625
|
+
}
|
|
2626
|
+
}
|
|
2627
|
+
var typeOf = (n) => {
|
|
2628
|
+
const t = n["@type"];
|
|
2629
|
+
const raw = typeof t === "string" ? [t] : Array.isArray(t) ? t.filter((x) => typeof x === "string") : [];
|
|
2630
|
+
return raw.map((s) => String(s).replace(/^https?:\/\/schema\.org\//i, ""));
|
|
2631
|
+
};
|
|
2632
|
+
var isType = (n, names) => typeOf(n).some((t) => names.some((name) => t.toLowerCase() === name.toLowerCase()));
|
|
2633
|
+
var str = (v) => typeof v === "string" && v.trim() ? v.trim() : null;
|
|
2634
|
+
function schemaUrl(value, nodes, depth = 0) {
|
|
2635
|
+
if (depth > 3) return null;
|
|
2636
|
+
const direct = str(value);
|
|
2637
|
+
if (direct) return direct;
|
|
2638
|
+
if (Array.isArray(value)) {
|
|
2639
|
+
for (const v of value) {
|
|
2640
|
+
const hit = schemaUrl(v, nodes, depth + 1);
|
|
2641
|
+
if (hit) return hit;
|
|
2642
|
+
}
|
|
2643
|
+
return null;
|
|
2644
|
+
}
|
|
2645
|
+
if (!value || typeof value !== "object") return null;
|
|
2646
|
+
const obj = value;
|
|
2647
|
+
const inline = str(obj["url"]) ?? str(obj["contentUrl"]);
|
|
2648
|
+
if (inline) return inline;
|
|
2649
|
+
const id = str(obj["@id"]);
|
|
2650
|
+
if (!id) return null;
|
|
2651
|
+
const target = nodes.find((n) => n !== obj && str(n["@id"]) === id);
|
|
2652
|
+
return target ? schemaUrl(target, nodes, depth + 1) : null;
|
|
2653
|
+
}
|
|
2654
|
+
function schemaChecks(pages, origin, phones) {
|
|
2655
|
+
const nodes = [];
|
|
2656
|
+
const declaredOn = /* @__PURE__ */ new Map();
|
|
2657
|
+
let parsedAny = false;
|
|
2658
|
+
for (const p of pages) {
|
|
2659
|
+
for (const block of p.extract.jsonLd) {
|
|
2660
|
+
const before = nodes.length;
|
|
2661
|
+
try {
|
|
2662
|
+
collectNodes(JSON.parse(block), nodes);
|
|
2663
|
+
parsedAny = true;
|
|
2664
|
+
} catch {
|
|
2665
|
+
}
|
|
2666
|
+
for (let i = before; i < nodes.length; i++) declaredOn.set(nodes[i], p.url);
|
|
2667
|
+
}
|
|
2668
|
+
}
|
|
2669
|
+
const ORG = ["Organization", "LocalBusiness", "ProfessionalService", "Corporation"];
|
|
2670
|
+
const org = nodes.find((n) => isType(n, ORG));
|
|
2671
|
+
const local = nodes.find((n) => isType(n, ["LocalBusiness", "ProfessionalService"]));
|
|
2672
|
+
const noSchema = (key, label, why) => skip(
|
|
2673
|
+
key,
|
|
2674
|
+
label,
|
|
2675
|
+
why,
|
|
2676
|
+
"content",
|
|
2677
|
+
parsedAny ? "no such block in your structured data" : "this site publishes no structured data"
|
|
2678
|
+
);
|
|
2679
|
+
const out = [];
|
|
2680
|
+
const WHY_ORG = "This is the block an assistant reads to learn who you are. Half-filled, it answers half the question and the rest gets taken from a directory you do not control.";
|
|
2681
|
+
if (!org) {
|
|
2682
|
+
out.push(noSchema("schema-org-complete", "Your business details in structured data", WHY_ORG));
|
|
2683
|
+
} else {
|
|
2684
|
+
const missing = ["name", "url", "logo"].filter(
|
|
2685
|
+
(f) => f === "name" ? !str(org[f]) : !schemaUrl(org[f], nodes)
|
|
2686
|
+
);
|
|
2687
|
+
out.push(
|
|
2688
|
+
check(
|
|
2689
|
+
"schema-org-complete",
|
|
2690
|
+
"Your business details in structured data",
|
|
2691
|
+
WHY_ORG,
|
|
2692
|
+
"content",
|
|
2693
|
+
missing.length === 0,
|
|
2694
|
+
missing.length === 0 ? `name, url and logo all present on your ${typeOf(org)[0]} block` : `your ${typeOf(org)[0]} block is missing ${missing.join(", ")}`
|
|
2695
|
+
)
|
|
2696
|
+
);
|
|
2697
|
+
}
|
|
2698
|
+
const WHY_LOCAL = "A local business block without an address, a phone or opening hours is the one an assistant will fill in from elsewhere.";
|
|
2699
|
+
if (!local) {
|
|
2700
|
+
out.push(
|
|
2701
|
+
noSchema("schema-local-complete", "Your address and hours in structured data", WHY_LOCAL)
|
|
2702
|
+
);
|
|
2703
|
+
} else {
|
|
2704
|
+
const missing = ["address", "telephone", "openingHours"].filter(
|
|
2705
|
+
(f) => local[f] === void 0 && local[`${f}Specification`] === void 0
|
|
2706
|
+
);
|
|
2707
|
+
out.push(
|
|
2708
|
+
check(
|
|
2709
|
+
"schema-local-complete",
|
|
2710
|
+
"Your address and hours in structured data",
|
|
2711
|
+
WHY_LOCAL,
|
|
2712
|
+
"content",
|
|
2713
|
+
missing.length === 0,
|
|
2714
|
+
missing.length === 0 ? "address, telephone and opening hours all present" : `missing ${missing.join(", ")}`
|
|
2715
|
+
)
|
|
2716
|
+
);
|
|
2717
|
+
}
|
|
2718
|
+
const WHY_URL = "A structured-data url pointing at a domain you no longer use tells every assistant that reads it to go and look somewhere else.";
|
|
2719
|
+
const declaredUrl = org ? schemaUrl(org["url"], nodes) : null;
|
|
2720
|
+
if (!declaredUrl) {
|
|
2721
|
+
out.push(noSchema("schema-url-matches", "Structured data pointing at this site", WHY_URL));
|
|
2722
|
+
} else {
|
|
2723
|
+
const base = (org ? declaredOn.get(org) : null) ?? origin;
|
|
2724
|
+
const absolute = (() => {
|
|
2725
|
+
try {
|
|
2726
|
+
return new URL(declaredUrl, base).toString();
|
|
2727
|
+
} catch {
|
|
2728
|
+
return null;
|
|
2729
|
+
}
|
|
2730
|
+
})();
|
|
2731
|
+
const host = (u) => {
|
|
2732
|
+
try {
|
|
2733
|
+
return new URL(u).hostname.replace(/^www\./i, "").toLowerCase();
|
|
2734
|
+
} catch {
|
|
2735
|
+
return null;
|
|
2736
|
+
}
|
|
2737
|
+
};
|
|
2738
|
+
const declaredHost = absolute === null ? null : host(absolute);
|
|
2739
|
+
const sameHost = declaredHost !== null && declaredHost === host(origin);
|
|
2740
|
+
out.push(
|
|
2741
|
+
check(
|
|
2742
|
+
"schema-url-matches",
|
|
2743
|
+
"Structured data pointing at this site",
|
|
2744
|
+
WHY_URL,
|
|
2745
|
+
"content",
|
|
2746
|
+
sameHost,
|
|
2747
|
+
// The resolved address, since that is the one we compared. Printing the
|
|
2748
|
+
// raw "/" back at a reader who passed would read as a bug in us.
|
|
2749
|
+
absolute ?? declaredUrl
|
|
2750
|
+
)
|
|
2751
|
+
);
|
|
2752
|
+
}
|
|
2753
|
+
const WHY_PHONE = "When the number in your structured data is not the number on your page, an assistant may quote either one.";
|
|
2754
|
+
const schemaPhone = org ? str(org["telephone"]) : null;
|
|
2755
|
+
const digits = (s) => s.replace(/\D/g, "").replace(/^1(?=\d{10}$)/, "");
|
|
2756
|
+
if (!schemaPhone || phones.length === 0) {
|
|
2757
|
+
out.push(
|
|
2758
|
+
skip(
|
|
2759
|
+
"schema-phone-matches",
|
|
2760
|
+
"One phone number, everywhere",
|
|
2761
|
+
WHY_PHONE,
|
|
2762
|
+
"quick",
|
|
2763
|
+
!schemaPhone ? "no telephone in your structured data" : "we found no phone number in the page text to compare it with"
|
|
2764
|
+
)
|
|
2765
|
+
);
|
|
2766
|
+
} else {
|
|
2767
|
+
const match = phones.some((p) => digits(p) === digits(schemaPhone));
|
|
2768
|
+
out.push(
|
|
2769
|
+
check(
|
|
2770
|
+
"schema-phone-matches",
|
|
2771
|
+
"One phone number, everywhere",
|
|
2772
|
+
WHY_PHONE,
|
|
2773
|
+
"quick",
|
|
2774
|
+
match,
|
|
2775
|
+
match ? `${schemaPhone}, matching the page` : `${schemaPhone} in schema, not among the numbers on your pages`
|
|
2776
|
+
)
|
|
2777
|
+
);
|
|
2778
|
+
}
|
|
2779
|
+
const WHY_REVIEW = "Google does not show star ratings a business publishes about itself, so the markup does nothing but sit there.";
|
|
2780
|
+
const selfReview = nodes.filter(
|
|
2781
|
+
(n) => isType(n, ["AggregateRating", "Review"]) && org !== void 0
|
|
2782
|
+
);
|
|
2783
|
+
const selfServing = org ? selfReview.filter(() => org["aggregateRating"] !== void 0 || org["review"] !== void 0) : [];
|
|
2784
|
+
if (!org || selfReview.length === 0) {
|
|
2785
|
+
out.push(
|
|
2786
|
+
skip(
|
|
2787
|
+
"schema-self-review",
|
|
2788
|
+
"Ratings you publish about yourself",
|
|
2789
|
+
WHY_REVIEW,
|
|
2790
|
+
"content",
|
|
2791
|
+
"no review or rating markup on this site"
|
|
2792
|
+
)
|
|
2793
|
+
);
|
|
2794
|
+
} else {
|
|
2795
|
+
out.push(
|
|
2796
|
+
check(
|
|
2797
|
+
"schema-self-review",
|
|
2798
|
+
"Ratings you publish about yourself",
|
|
2799
|
+
WHY_REVIEW,
|
|
2800
|
+
"content",
|
|
2801
|
+
selfServing.length === 0,
|
|
2802
|
+
selfServing.length === 0 ? "your rating markup is attached to something other than your own organisation" : `an aggregateRating sits on your own ${typeOf(org)[0]} block`
|
|
2803
|
+
)
|
|
2804
|
+
);
|
|
2805
|
+
}
|
|
2806
|
+
return out;
|
|
2807
|
+
}
|
|
2808
|
+
var SECURITY_HEADER_CHECKS = [
|
|
2809
|
+
{
|
|
2810
|
+
header: "strict-transport-security",
|
|
2811
|
+
label: "Telling browsers to always use https",
|
|
2812
|
+
why: "Without it, the first visit of the day can still be made over plain http before the redirect happens."
|
|
2813
|
+
},
|
|
2814
|
+
{
|
|
2815
|
+
header: "content-security-policy",
|
|
2816
|
+
label: "A policy for what may run on your pages",
|
|
2817
|
+
why: "It is the difference between a compromised script running on your site and being blocked by the browser."
|
|
2818
|
+
},
|
|
2819
|
+
{
|
|
2820
|
+
header: "x-content-type-options",
|
|
2821
|
+
label: "Stopping browsers guessing file types",
|
|
2822
|
+
why: "One line that prevents a browser treating an uploaded file as something executable."
|
|
2823
|
+
},
|
|
2824
|
+
{
|
|
2825
|
+
header: "x-frame-options",
|
|
2826
|
+
label: "Stopping other sites framing yours",
|
|
2827
|
+
why: "Without it, anyone can put your site inside theirs and collect what visitors type into it."
|
|
2828
|
+
},
|
|
2829
|
+
{
|
|
2830
|
+
header: "permissions-policy",
|
|
2831
|
+
label: "Declaring what your pages may ask for",
|
|
2832
|
+
// NOT "it stops an embedded third party quietly asking for their camera",
|
|
2833
|
+
// which is what this said and is not true. Tested in a browser: a
|
|
2834
|
+
// cross-origin iframe is refused camera and geolocation with no header set
|
|
2835
|
+
// at all, and is GRANTED them with no header set the moment the site writes
|
|
2836
|
+
// `allow="geolocation"` on the iframe itself. The gate is the attribute the
|
|
2837
|
+
// site author types, so nothing here happens quietly. What the header adds
|
|
2838
|
+
// is the ability to say no to code already running on the page — including
|
|
2839
|
+
// code the owner did not write, if a dependency is ever compromised.
|
|
2840
|
+
why: "It limits what code running on your own pages may ask visitors for \u2014 including code you did not write, if a dependency is ever compromised.",
|
|
2841
|
+
// At least one directive whose allowlist is not `*`. `camera=()` denies it
|
|
2842
|
+
// outright, `camera=(self)` keeps it to your own code, `camera=*` grants it
|
|
2843
|
+
// to anything the page delegates to — which is the browser default, so a
|
|
2844
|
+
// header made only of those changes nothing.
|
|
2845
|
+
restricts: (value) => value.split(",").map((d) => d.trim()).filter(Boolean).some((d) => {
|
|
2846
|
+
const eq = d.indexOf("=");
|
|
2847
|
+
if (eq < 0) return false;
|
|
2848
|
+
const allow = d.slice(eq + 1).trim();
|
|
2849
|
+
return allow !== "*";
|
|
2850
|
+
})
|
|
2851
|
+
}
|
|
2852
|
+
];
|
|
2853
|
+
var VERSIONED = /\d+\.\d+/;
|
|
2854
|
+
var LEAKS_MORE_THAN_DEFAULT = /* @__PURE__ */ new Set(["unsafe-url", "no-referrer-when-downgrade"]);
|
|
2855
|
+
function referrerPolicyCheck(headers, measured) {
|
|
2856
|
+
const LABEL = "Not leaking more of your addresses than the browser already does";
|
|
2857
|
+
const WHY = "Browsers already trim the page address to your bare domain before handing it to a site a visitor clicks through to. Two values of this header undo that and send the whole address, query string included.";
|
|
2858
|
+
if (!measured) return unknown("header-referrer-policy", LABEL, WHY, "quick");
|
|
2859
|
+
const raw = headers["referrer-policy"];
|
|
2860
|
+
if (raw === void 0) {
|
|
2861
|
+
return skip(
|
|
2862
|
+
"header-referrer-policy",
|
|
2863
|
+
LABEL,
|
|
2864
|
+
WHY,
|
|
2865
|
+
"quick",
|
|
2866
|
+
"no referrer-policy header, so the browser's own default applies \u2014 which does not leak the address"
|
|
2867
|
+
);
|
|
2868
|
+
}
|
|
2869
|
+
const values = raw.split(",").map((v) => v.trim().toLowerCase()).filter(Boolean);
|
|
2870
|
+
const effective = [...values].reverse().find((v) => KNOWN_REFERRER_POLICIES.has(v)) ?? null;
|
|
2871
|
+
if (effective === null) {
|
|
2872
|
+
return skip(
|
|
2873
|
+
"header-referrer-policy",
|
|
2874
|
+
LABEL,
|
|
2875
|
+
WHY,
|
|
2876
|
+
"quick",
|
|
2877
|
+
`referrer-policy names no value a browser recognises (\u201C${raw.slice(0, 60)}\u201D), so the default applies`
|
|
2878
|
+
);
|
|
2879
|
+
}
|
|
2880
|
+
const leaks = LEAKS_MORE_THAN_DEFAULT.has(effective);
|
|
2881
|
+
return check(
|
|
2882
|
+
"header-referrer-policy",
|
|
2883
|
+
LABEL,
|
|
2884
|
+
WHY,
|
|
2885
|
+
"quick",
|
|
2886
|
+
!leaks,
|
|
2887
|
+
leaks ? `referrer-policy: ${effective} \u2014 that sends the full address, query string and all, to every site your visitors click through to` : `referrer-policy: ${effective}`
|
|
2888
|
+
);
|
|
2889
|
+
}
|
|
2890
|
+
var KNOWN_REFERRER_POLICIES = /* @__PURE__ */ new Set([
|
|
2891
|
+
"no-referrer",
|
|
2892
|
+
"no-referrer-when-downgrade",
|
|
2893
|
+
"origin",
|
|
2894
|
+
"origin-when-cross-origin",
|
|
2895
|
+
"same-origin",
|
|
2896
|
+
"strict-origin",
|
|
2897
|
+
"strict-origin-when-cross-origin",
|
|
2898
|
+
"unsafe-url"
|
|
2899
|
+
]);
|
|
2900
|
+
function headerChecks(headers, measured) {
|
|
2901
|
+
const out = [];
|
|
2902
|
+
for (const { header, label, why, restricts } of SECURITY_HEADER_CHECKS) {
|
|
2903
|
+
if (!measured) {
|
|
2904
|
+
out.push(unknown(`header-${header}`, label, why, "quick"));
|
|
2905
|
+
continue;
|
|
2906
|
+
}
|
|
2907
|
+
const value = headers[header];
|
|
2908
|
+
const present = value !== void 0;
|
|
2909
|
+
const ok = present && (restricts === void 0 || restricts(value));
|
|
2910
|
+
out.push(
|
|
2911
|
+
check(
|
|
2912
|
+
`header-${header}`,
|
|
2913
|
+
label,
|
|
2914
|
+
why,
|
|
2915
|
+
"quick",
|
|
2916
|
+
ok,
|
|
2917
|
+
!present ? `no ${header} header` : ok ? `${header} is set` : `${header} is set, but every directive in it is \u201C*\u201D \u2014 which is what the browser does without the header`
|
|
2918
|
+
)
|
|
2919
|
+
);
|
|
2920
|
+
}
|
|
2921
|
+
out.push(referrerPolicyCheck(headers, measured));
|
|
2922
|
+
const WHY_LEAK = "Naming the exact version you run tells anyone looking which published vulnerabilities to try first.";
|
|
2923
|
+
if (!measured) {
|
|
2924
|
+
out.push(
|
|
2925
|
+
unknown("header-version-leak", "Not advertising your software versions", WHY_LEAK, "quick")
|
|
2926
|
+
);
|
|
2927
|
+
} else {
|
|
2928
|
+
const leaks = ["server", "x-powered-by"].filter((h) => headers[h] && VERSIONED.test(headers[h])).map((h) => `${h}: ${headers[h]}`);
|
|
2929
|
+
out.push(
|
|
2930
|
+
check(
|
|
2931
|
+
"header-version-leak",
|
|
2932
|
+
"Not advertising your software versions",
|
|
2933
|
+
WHY_LEAK,
|
|
2934
|
+
"quick",
|
|
2935
|
+
leaks.length === 0,
|
|
2936
|
+
leaks.length === 0 ? "no version numbers in your response headers" : leaks.join(", ")
|
|
2937
|
+
)
|
|
2938
|
+
);
|
|
2939
|
+
}
|
|
2940
|
+
const WHY_GZIP = "Uncompressed HTML is roughly four times the bytes for the same page, paid for by every visitor on a phone.";
|
|
2941
|
+
if (!measured) {
|
|
2942
|
+
out.push(unknown("header-compression", "Pages sent compressed", WHY_GZIP, "quick"));
|
|
2943
|
+
} else {
|
|
2944
|
+
const enc = headers["content-encoding"] ?? "";
|
|
2945
|
+
const ok = /gzip|br|zstd|deflate/i.test(enc);
|
|
2946
|
+
out.push(
|
|
2947
|
+
check(
|
|
2948
|
+
"header-compression",
|
|
2949
|
+
"Pages sent compressed",
|
|
2950
|
+
WHY_GZIP,
|
|
2951
|
+
"quick",
|
|
2952
|
+
ok,
|
|
2953
|
+
ok ? `content-encoding: ${enc}` : "your HTML is served uncompressed"
|
|
2954
|
+
)
|
|
2955
|
+
);
|
|
2956
|
+
}
|
|
2957
|
+
const WHY_MAXAGE = "A max-age of a few minutes gives almost none of the protection the header exists for.";
|
|
2958
|
+
const hsts = headers["strict-transport-security"];
|
|
2959
|
+
const SIX_MONTHS = 15552e3;
|
|
2960
|
+
if (!measured || !hsts) {
|
|
2961
|
+
out.push(
|
|
2962
|
+
skip(
|
|
2963
|
+
"header-hsts-age",
|
|
2964
|
+
"An https policy that lasts",
|
|
2965
|
+
WHY_MAXAGE,
|
|
2966
|
+
"quick",
|
|
2967
|
+
"no strict-transport-security header to read a max-age from"
|
|
2968
|
+
)
|
|
2969
|
+
);
|
|
2970
|
+
} else {
|
|
2971
|
+
const m = /max-age\s*=\s*(\d+)/i.exec(hsts);
|
|
2972
|
+
const age = m ? Number(m[1]) : 0;
|
|
2973
|
+
out.push(
|
|
2974
|
+
check(
|
|
2975
|
+
"header-hsts-age",
|
|
2976
|
+
"An https policy that lasts",
|
|
2977
|
+
WHY_MAXAGE,
|
|
2978
|
+
"quick",
|
|
2979
|
+
age >= SIX_MONTHS,
|
|
2980
|
+
`max-age=${age}${age >= SIX_MONTHS ? "" : " \u2014 six months is the usual minimum"}`
|
|
2981
|
+
)
|
|
2982
|
+
);
|
|
2983
|
+
}
|
|
2984
|
+
return out;
|
|
2985
|
+
}
|
|
2986
|
+
function sidecarChecks(crawl, linkedPages) {
|
|
2987
|
+
const discoveredLinks = linkedPages.size;
|
|
2988
|
+
const out = [];
|
|
2989
|
+
const robotsMeasured = crawl.sidecarErrors.robots === null;
|
|
2990
|
+
const WHY_DISALLOW = "A blanket Disallow tells every search engine and every assistant to stay off the whole site. It is usually left over from a staging server.";
|
|
2991
|
+
if (!robotsMeasured) {
|
|
2992
|
+
out.push(
|
|
2993
|
+
unknown(
|
|
2994
|
+
"robots-not-blocking",
|
|
2995
|
+
"robots.txt not blocking your whole site",
|
|
2996
|
+
WHY_DISALLOW,
|
|
2997
|
+
"quick"
|
|
2998
|
+
)
|
|
2999
|
+
);
|
|
3000
|
+
} else if (crawl.robotsTxt === null) {
|
|
3001
|
+
out.push(
|
|
3002
|
+
check(
|
|
3003
|
+
"robots-not-blocking",
|
|
3004
|
+
"robots.txt not blocking your whole site",
|
|
3005
|
+
WHY_DISALLOW,
|
|
3006
|
+
"quick",
|
|
3007
|
+
true,
|
|
3008
|
+
"no robots.txt, so nothing is disallowed"
|
|
3009
|
+
)
|
|
3010
|
+
);
|
|
3011
|
+
} else {
|
|
3012
|
+
const blocked = /user-agent\s*:\s*\*[^]*?^\s*disallow\s*:\s*\/\s*$/im.test(crawl.robotsTxt);
|
|
3013
|
+
out.push(
|
|
3014
|
+
check(
|
|
3015
|
+
"robots-not-blocking",
|
|
3016
|
+
"robots.txt not blocking your whole site",
|
|
3017
|
+
WHY_DISALLOW,
|
|
3018
|
+
"quick",
|
|
3019
|
+
!blocked,
|
|
3020
|
+
blocked ? "Disallow: / applies to every crawler" : "nothing site-wide is disallowed"
|
|
3021
|
+
)
|
|
3022
|
+
);
|
|
3023
|
+
}
|
|
3024
|
+
const WHY_SITEMAP_LINE = "Naming the sitemap in robots.txt is how a crawler that has never seen your site finds the list of your pages.";
|
|
3025
|
+
if (!robotsMeasured || crawl.robotsTxt === null) {
|
|
3026
|
+
out.push(
|
|
3027
|
+
skip(
|
|
3028
|
+
"robots-names-sitemap",
|
|
3029
|
+
"robots.txt naming your sitemap",
|
|
3030
|
+
WHY_SITEMAP_LINE,
|
|
3031
|
+
"quick",
|
|
3032
|
+
robotsMeasured ? "this site has no robots.txt" : "we could not read robots.txt on this run"
|
|
3033
|
+
)
|
|
3034
|
+
);
|
|
3035
|
+
} else {
|
|
3036
|
+
const named = /^\s*sitemap\s*:/im.test(crawl.robotsTxt);
|
|
3037
|
+
out.push(
|
|
3038
|
+
check(
|
|
3039
|
+
"robots-names-sitemap",
|
|
3040
|
+
"robots.txt naming your sitemap",
|
|
3041
|
+
WHY_SITEMAP_LINE,
|
|
3042
|
+
"quick",
|
|
3043
|
+
named,
|
|
3044
|
+
named ? "a Sitemap: line is present" : "no Sitemap: line in robots.txt"
|
|
3045
|
+
)
|
|
3046
|
+
);
|
|
3047
|
+
}
|
|
3048
|
+
const sitemapMeasured = crawl.sidecarErrors.sitemap === null;
|
|
3049
|
+
const WHY_COVERAGE = "A sitemap listing fewer pages than your own navigation links to leaves the rest to be found by luck.";
|
|
3050
|
+
if (!sitemapMeasured || !crawl.sitemap.present) {
|
|
3051
|
+
out.push(
|
|
3052
|
+
skip(
|
|
3053
|
+
"sitemap-coverage",
|
|
3054
|
+
"A sitemap that lists your pages",
|
|
3055
|
+
WHY_COVERAGE,
|
|
3056
|
+
"content",
|
|
3057
|
+
sitemapMeasured ? "this site has no sitemap.xml" : "we could not read the sitemap on this run"
|
|
3058
|
+
)
|
|
3059
|
+
);
|
|
3060
|
+
} else if (discoveredLinks === 0) {
|
|
3061
|
+
out.push(
|
|
3062
|
+
unknown("sitemap-coverage", "A sitemap that lists your pages", WHY_COVERAGE, "content")
|
|
3063
|
+
);
|
|
3064
|
+
} else if (crawl.sitemap.truncated !== false) {
|
|
3065
|
+
out.push(
|
|
3066
|
+
unknown("sitemap-coverage", "A sitemap that lists your pages", WHY_COVERAGE, "content")
|
|
3067
|
+
);
|
|
3068
|
+
} else {
|
|
3069
|
+
const ok = crawl.sitemap.urlCount >= discoveredLinks;
|
|
3070
|
+
const sample = crawl.sitemap.sample;
|
|
3071
|
+
const whole = sample !== void 0 && crawl.sitemap.urlCount <= sample.length;
|
|
3072
|
+
const listed = new Set(sample?.map((u) => norm2(u) ?? u));
|
|
3073
|
+
const unlisted = whole ? [...linkedPages].filter((a) => !listed.has(a)) : [];
|
|
3074
|
+
out.push(
|
|
3075
|
+
check(
|
|
3076
|
+
"sitemap-coverage",
|
|
3077
|
+
"A sitemap that lists your pages",
|
|
3078
|
+
WHY_COVERAGE,
|
|
3079
|
+
"content",
|
|
3080
|
+
ok && unlisted.length === 0,
|
|
3081
|
+
unlisted.length > 0 ? `${unlisted.length} ${unlisted.length === 1 ? "page your own links point to is" : "pages your own links point to are"} missing from it \u2014 ${unlisted[0]}` : `${crawl.sitemap.urlCount} URLs listed, against ${discoveredLinks} distinct pages your own links point to`
|
|
3082
|
+
)
|
|
3083
|
+
);
|
|
3084
|
+
}
|
|
3085
|
+
const WHY_LLMS = "A short llms.txt states, in your own words, what you do and which pages matter. No answer engine has published that it reads one, so this is recorded rather than counted against you.";
|
|
3086
|
+
const llmsMeasured = crawl.sidecarErrors.llms === null;
|
|
3087
|
+
if (!llmsMeasured) {
|
|
3088
|
+
out.push(unknown("llms-txt", "A file written for AI assistants", WHY_LLMS, "content"));
|
|
3089
|
+
} else if (crawl.llmsTxt.present) {
|
|
3090
|
+
out.push(
|
|
3091
|
+
check(
|
|
3092
|
+
"llms-txt",
|
|
3093
|
+
"A file written for AI assistants",
|
|
3094
|
+
WHY_LLMS,
|
|
3095
|
+
"content",
|
|
3096
|
+
true,
|
|
3097
|
+
crawl.llmsTxt.firstLine ?? "present"
|
|
3098
|
+
)
|
|
3099
|
+
);
|
|
3100
|
+
} else {
|
|
3101
|
+
out.push(
|
|
3102
|
+
skip(
|
|
3103
|
+
"llms-txt",
|
|
3104
|
+
"A file written for AI assistants",
|
|
3105
|
+
WHY_LLMS,
|
|
3106
|
+
"content",
|
|
3107
|
+
"no llms.txt \u2014 recorded, not counted against you, because no engine has published that it reads one"
|
|
3108
|
+
)
|
|
3109
|
+
);
|
|
3110
|
+
}
|
|
3111
|
+
return out;
|
|
3112
|
+
}
|
|
3113
|
+
var FIRST_PARTY_BUNDLE = 10;
|
|
3114
|
+
var ANALYTICS_MARKERS = /(googletagmanager\.com|google-analytics\.com|plausible\.io|usefathom\.com|clarity\.ms|segment\.(?:com|io)|matomo|piwik|statcounter|hotjar\.com|posthog\.com|umami|simpleanalytics|cloudflareinsights\.com)/i;
|
|
3115
|
+
function analyticsCheck(pages) {
|
|
3116
|
+
const WHY = "Without any analytics you cannot tell whether a change helped, which visit turned into an enquiry, or whether the work was worth doing.";
|
|
3117
|
+
const LABEL = "Something measuring whether any of this works";
|
|
3118
|
+
const readable = pages.filter((p) => p.extract.scriptSrcs !== void 0);
|
|
3119
|
+
if (readable.length === 0) return unknown("analytics", LABEL, WHY, "quick");
|
|
3120
|
+
const loaded = readable.flatMap((p) => p.extract.scriptSrcs ?? []);
|
|
3121
|
+
const named = readable.flatMap((p) => p.extract.inlineScriptUrls ?? []);
|
|
3122
|
+
const hit = loaded.find((s) => ANALYTICS_MARKERS.test(s));
|
|
3123
|
+
const deferred = hit ? void 0 : named.find((h) => ANALYTICS_MARKERS.test(h));
|
|
3124
|
+
const thirdParty = loaded.filter((src) => /^https?:\/\//i.test(src));
|
|
3125
|
+
if (!hit && !deferred && thirdParty.length === 0 && loaded.length >= FIRST_PARTY_BUNDLE) {
|
|
3126
|
+
return unknown("analytics", LABEL, WHY, "quick");
|
|
3127
|
+
}
|
|
3128
|
+
return check(
|
|
3129
|
+
"analytics",
|
|
3130
|
+
LABEL,
|
|
3131
|
+
WHY,
|
|
3132
|
+
"quick",
|
|
3133
|
+
hit !== void 0 || deferred !== void 0,
|
|
3134
|
+
hit ?? (deferred ? `${deferred}, loaded on first interaction` : "no third-party analytics on the pages we read")
|
|
3135
|
+
);
|
|
3136
|
+
}
|
|
3137
|
+
function metaChecks(pages) {
|
|
3138
|
+
const out = [];
|
|
3139
|
+
const readable = pages.filter((p) => p.extract.metas !== void 0);
|
|
3140
|
+
const WHY_NOINDEX = "This one line tells every search engine and assistant to leave the page out of their results entirely. It is almost always left over from a staging site, and nothing on the page looks wrong.";
|
|
3141
|
+
const WHY_NOFOLLOW = "It tells search engines not to follow any link on the page, so nothing you link to gets credit from it.";
|
|
3142
|
+
const WHY_CHARSET = "Without a declared character set the browser guesses, and a wrong guess is what turns apostrophes into \xE2\u20AC\u2122.";
|
|
3143
|
+
if (readable.length === 0) {
|
|
3144
|
+
return [
|
|
3145
|
+
unknown("meta-noindex", "Pages search engines are allowed to list", WHY_NOINDEX, "quick"),
|
|
3146
|
+
unknown("meta-nofollow", "Links search engines are allowed to follow", WHY_NOFOLLOW, "quick"),
|
|
3147
|
+
unknown("meta-charset", "A declared character set", WHY_CHARSET, "quick"),
|
|
3148
|
+
unknown(
|
|
3149
|
+
"title-length",
|
|
3150
|
+
"Titles that survive a search result",
|
|
3151
|
+
"A title that is too long is cut off mid-word in the result, and one that is too short wastes the most valuable line you get.",
|
|
3152
|
+
"content"
|
|
3153
|
+
),
|
|
3154
|
+
unknown(
|
|
3155
|
+
"description-length",
|
|
3156
|
+
"Descriptions that survive a search result",
|
|
3157
|
+
"The description is the sentence under your link. Too long and it is truncated; too short and the engine writes its own.",
|
|
3158
|
+
"content"
|
|
3159
|
+
),
|
|
3160
|
+
unknown(
|
|
3161
|
+
"og-image-absolute",
|
|
3162
|
+
"A share image with a full web address",
|
|
3163
|
+
"A relative og:image renders no card at all on Slack, LinkedIn or iMessage \u2014 the link arrives as bare text.",
|
|
3164
|
+
"quick"
|
|
3165
|
+
),
|
|
3166
|
+
unknown(
|
|
3167
|
+
"duplicate-descriptions",
|
|
3168
|
+
"A different description on each page",
|
|
3169
|
+
"Identical descriptions make every page look like the same page in a search result.",
|
|
3170
|
+
"content"
|
|
3171
|
+
)
|
|
3172
|
+
];
|
|
3173
|
+
}
|
|
3174
|
+
const flagged = (token) => readable.filter((p) => token.test(p.extract.metas?.["robots"] ?? "")).map((p) => ({ url: p.url, value: p.extract.metas["robots"] }));
|
|
3175
|
+
const noindex = flagged(/\bnoindex\b/i);
|
|
3176
|
+
out.push(
|
|
3177
|
+
check(
|
|
3178
|
+
"meta-noindex",
|
|
3179
|
+
"Pages search engines are allowed to list",
|
|
3180
|
+
WHY_NOINDEX,
|
|
3181
|
+
"quick",
|
|
3182
|
+
noindex.length === 0,
|
|
3183
|
+
noindex.length === 0 ? `none of the ${readable.length} pages we read asks to be hidden` : `${noindex.length} ${noindex.length === 1 ? "page carries" : "pages carry"} noindex: ${noindex.slice(0, 3).map((n) => n.url).join(", ")}`
|
|
3184
|
+
)
|
|
3185
|
+
);
|
|
3186
|
+
const nofollow = flagged(/\bnofollow\b/i);
|
|
3187
|
+
out.push(
|
|
3188
|
+
check(
|
|
3189
|
+
"meta-nofollow",
|
|
3190
|
+
"Links search engines are allowed to follow",
|
|
3191
|
+
WHY_NOFOLLOW,
|
|
3192
|
+
"quick",
|
|
3193
|
+
nofollow.length === 0,
|
|
3194
|
+
nofollow.length === 0 ? "no page-wide nofollow" : `${nofollow.length} ${nofollow.length === 1 ? "page" : "pages"}`
|
|
3195
|
+
)
|
|
3196
|
+
);
|
|
3197
|
+
const noCharset = readable.filter((p) => !p.extract.metas?.["charset"]);
|
|
3198
|
+
out.push(
|
|
3199
|
+
check(
|
|
3200
|
+
"meta-charset",
|
|
3201
|
+
"A declared character set",
|
|
3202
|
+
WHY_CHARSET,
|
|
3203
|
+
"quick",
|
|
3204
|
+
noCharset.length === 0,
|
|
3205
|
+
noCharset.length === 0 ? `declared on all ${readable.length} pages` : `missing on ${noCharset.length} ${noCharset.length === 1 ? "page" : "pages"}`
|
|
3206
|
+
)
|
|
3207
|
+
);
|
|
3208
|
+
const TITLE_MIN = 10;
|
|
3209
|
+
const TITLE_MAX = 70;
|
|
3210
|
+
const titled = pages.filter((p) => p.extract.title);
|
|
3211
|
+
if (titled.length === 0) {
|
|
3212
|
+
out.push(
|
|
3213
|
+
skip(
|
|
3214
|
+
"title-length",
|
|
3215
|
+
"Titles that survive a search result",
|
|
3216
|
+
"A title that is too long is cut off mid-word in the result, and one that is too short wastes the most valuable line you get.",
|
|
3217
|
+
"content",
|
|
3218
|
+
"no page we read carries a title, which is reported separately"
|
|
3219
|
+
)
|
|
3220
|
+
);
|
|
3221
|
+
} else {
|
|
3222
|
+
const bad = titled.filter(
|
|
3223
|
+
(p) => p.extract.title.length < TITLE_MIN || p.extract.title.length > TITLE_MAX
|
|
3224
|
+
);
|
|
3225
|
+
out.push(
|
|
3226
|
+
check(
|
|
3227
|
+
"title-length",
|
|
3228
|
+
"Titles that survive a search result",
|
|
3229
|
+
"A title that is too long is cut off mid-word in the result, and one that is too short wastes the most valuable line you get.",
|
|
3230
|
+
"content",
|
|
3231
|
+
bad.length === 0,
|
|
3232
|
+
bad.length === 0 ? `all ${titled.length} are between ${TITLE_MIN} and ${TITLE_MAX} characters` : bad.slice(0, 2).map((p) => `${p.url}: ${p.extract.title.length} characters`).join(", ")
|
|
3233
|
+
)
|
|
3234
|
+
);
|
|
3235
|
+
}
|
|
3236
|
+
const DESC_MIN = 40;
|
|
3237
|
+
const DESC_MAX = 200;
|
|
3238
|
+
const described = contentPages(pages).filter((p) => p.extract.metaDescription);
|
|
3239
|
+
if (described.length === 0) {
|
|
3240
|
+
out.push(
|
|
3241
|
+
skip(
|
|
3242
|
+
"description-length",
|
|
3243
|
+
"Descriptions that survive a search result",
|
|
3244
|
+
"The description is the sentence under your link. Too long and it is truncated; too short and the engine writes its own.",
|
|
3245
|
+
"content",
|
|
3246
|
+
"no page we read carries a description, which is reported separately"
|
|
3247
|
+
)
|
|
3248
|
+
);
|
|
3249
|
+
} else {
|
|
3250
|
+
const bad = described.filter(
|
|
3251
|
+
(p) => p.extract.metaDescription.length < DESC_MIN || p.extract.metaDescription.length > DESC_MAX
|
|
3252
|
+
);
|
|
3253
|
+
out.push(
|
|
3254
|
+
check(
|
|
3255
|
+
"description-length",
|
|
3256
|
+
"Descriptions that survive a search result",
|
|
3257
|
+
"The description is the sentence under your link. Too long and it is truncated; too short and the engine writes its own.",
|
|
3258
|
+
"content",
|
|
3259
|
+
bad.length === 0,
|
|
3260
|
+
bad.length === 0 ? `all ${described.length} are between ${DESC_MIN} and ${DESC_MAX} characters` : bad.slice(0, 2).map((p) => `${p.url}: ${p.extract.metaDescription.length} characters`).join(", ")
|
|
3261
|
+
)
|
|
3262
|
+
);
|
|
3263
|
+
}
|
|
3264
|
+
const WHY_OG = "A relative og:image renders no card at all on Slack, LinkedIn or iMessage \u2014 the link arrives as bare text.";
|
|
3265
|
+
const withOg = pages.filter((p) => p.extract.social["og:image"]);
|
|
3266
|
+
if (withOg.length === 0) {
|
|
3267
|
+
out.push(
|
|
3268
|
+
skip(
|
|
3269
|
+
"og-image-absolute",
|
|
3270
|
+
"A share image with a full web address",
|
|
3271
|
+
WHY_OG,
|
|
3272
|
+
"quick",
|
|
3273
|
+
"no page we read declares an og:image"
|
|
3274
|
+
)
|
|
3275
|
+
);
|
|
3276
|
+
} else {
|
|
3277
|
+
const relative = withOg.filter((p) => !/^https?:\/\//i.test(p.extract.social["og:image"]));
|
|
3278
|
+
out.push(
|
|
3279
|
+
check(
|
|
3280
|
+
"og-image-absolute",
|
|
3281
|
+
"A share image with a full web address",
|
|
3282
|
+
WHY_OG,
|
|
3283
|
+
"quick",
|
|
3284
|
+
relative.length === 0,
|
|
3285
|
+
relative.length === 0 ? withOg[0].extract.social["og:image"] : `${relative[0].url} declares \u201C${relative[0].extract.social["og:image"]}\u201D, which has no host`
|
|
3286
|
+
)
|
|
3287
|
+
);
|
|
3288
|
+
}
|
|
3289
|
+
const WHY_DUP_DESC = "Identical descriptions make every page look like the same page in a search result, and the engine usually replaces them with a sentence of its own choosing.";
|
|
3290
|
+
const distinctDescribed = byDeclaredAddress(described);
|
|
3291
|
+
if (distinctDescribed.length < 2) {
|
|
3292
|
+
out.push(
|
|
3293
|
+
skip(
|
|
3294
|
+
"duplicate-descriptions",
|
|
3295
|
+
"A different description on each page",
|
|
3296
|
+
WHY_DUP_DESC,
|
|
3297
|
+
"content",
|
|
3298
|
+
described.length < 2 ? "fewer than two pages carry a description, so there is nothing to compare" : "the pages we read that carry a description all declare the same address, so there is nothing to compare"
|
|
3299
|
+
)
|
|
3300
|
+
);
|
|
3301
|
+
} else {
|
|
3302
|
+
const seen = /* @__PURE__ */ new Map();
|
|
3303
|
+
for (const p of distinctDescribed) {
|
|
3304
|
+
const key = p.extract.metaDescription.trim().toLowerCase();
|
|
3305
|
+
seen.set(key, [...seen.get(key) ?? [], p.url]);
|
|
3306
|
+
}
|
|
3307
|
+
const dupes = [...seen.entries()].filter(([, urls]) => urls.length > 1);
|
|
3308
|
+
out.push(
|
|
3309
|
+
check(
|
|
3310
|
+
"duplicate-descriptions",
|
|
3311
|
+
"A different description on each page",
|
|
3312
|
+
WHY_DUP_DESC,
|
|
3313
|
+
"content",
|
|
3314
|
+
dupes.length === 0,
|
|
3315
|
+
dupes.length === 0 ? `${distinctDescribed.length} pages, ${seen.size} different descriptions` : `${dupes[0][1].length} pages share one description`
|
|
3316
|
+
)
|
|
3317
|
+
);
|
|
3318
|
+
}
|
|
3319
|
+
return out;
|
|
3320
|
+
}
|
|
3321
|
+
function linkChecks(pages, origin, faviconProbed) {
|
|
3322
|
+
const readable = pages.filter((p) => p.extract.links !== void 0);
|
|
3323
|
+
const WHY_FAVICON = "Without one the browser shows a blank page icon in the tab and the bookmark, which is the smallest possible thing to fix and the most often seen.";
|
|
3324
|
+
const WHY_CANON_SELF = "A canonical naming another page tells search engines to index that page instead of this one. That is right for a genuine duplicate, and the fastest way to remove real pages from search results when it is not.";
|
|
3325
|
+
const WHY_CANON_ORIGIN = "A canonical on another domain hands every page's search ranking to that domain. It is usually a staging host or a site they used to own.";
|
|
3326
|
+
const WHY_HREFLANG = "An hreflang set that does not name itself is ignored, so the translated pages compete with each other instead of being offered to the right reader.";
|
|
3327
|
+
if (readable.length === 0) {
|
|
3328
|
+
return [
|
|
3329
|
+
unknown("favicon-declared", "An icon for the browser tab", WHY_FAVICON, "quick"),
|
|
3330
|
+
unknown("canonical-self", "Pages that point at themselves", WHY_CANON_SELF, "structural"),
|
|
3331
|
+
unknown(
|
|
3332
|
+
"canonical-origin",
|
|
3333
|
+
"Canonicals that stay on your domain",
|
|
3334
|
+
WHY_CANON_ORIGIN,
|
|
3335
|
+
"structural"
|
|
3336
|
+
),
|
|
3337
|
+
unknown("hreflang-self", "Language alternates that name themselves", WHY_HREFLANG, "content")
|
|
3338
|
+
];
|
|
3339
|
+
}
|
|
3340
|
+
const out = [];
|
|
3341
|
+
const icon = readable.flatMap((p) => p.extract.links ?? []).find((l) => /(^|\s)(icon|shortcut icon|apple-touch-icon)(\s|$)/.test(l.rel));
|
|
3342
|
+
out.push(
|
|
3343
|
+
faviconProbed ? skip(
|
|
3344
|
+
"favicon-declared",
|
|
3345
|
+
"An icon for the browser tab",
|
|
3346
|
+
WHY_FAVICON,
|
|
3347
|
+
"quick",
|
|
3348
|
+
icon ? "the icon your pages declare is the one we fetched, reported below" : "no icon is declared, so we fetched /favicon.ico instead \u2014 reported below"
|
|
3349
|
+
) : check(
|
|
3350
|
+
"favicon-declared",
|
|
3351
|
+
"An icon for the browser tab",
|
|
3352
|
+
WHY_FAVICON,
|
|
3353
|
+
"quick",
|
|
3354
|
+
icon !== void 0,
|
|
3355
|
+
icon ? `rel="${icon.rel}" \u2192 ${icon.href}` : "no icon link on the pages we read"
|
|
3356
|
+
)
|
|
3357
|
+
);
|
|
3358
|
+
const host = (u, base) => {
|
|
3359
|
+
try {
|
|
3360
|
+
return new URL(u, base).hostname.replace(/^www\./i, "").toLowerCase();
|
|
3361
|
+
} catch {
|
|
3362
|
+
return null;
|
|
3363
|
+
}
|
|
3364
|
+
};
|
|
3365
|
+
const ownHost = host(origin);
|
|
3366
|
+
const canonical = pages.filter((p) => p.extract.canonical);
|
|
3367
|
+
if (canonical.length === 0) {
|
|
3368
|
+
out.push(
|
|
3369
|
+
skip(
|
|
3370
|
+
"canonical-self",
|
|
3371
|
+
"Pages that point at themselves",
|
|
3372
|
+
WHY_CANON_SELF,
|
|
3373
|
+
"structural",
|
|
3374
|
+
"no page we read declares a canonical, which is reported separately"
|
|
3375
|
+
),
|
|
3376
|
+
skip(
|
|
3377
|
+
"canonical-origin",
|
|
3378
|
+
"Canonicals that stay on your domain",
|
|
3379
|
+
WHY_CANON_ORIGIN,
|
|
3380
|
+
"structural",
|
|
3381
|
+
"no page we read declares a canonical"
|
|
3382
|
+
)
|
|
3383
|
+
);
|
|
3384
|
+
} else {
|
|
3385
|
+
const norm = (u, base) => {
|
|
3386
|
+
try {
|
|
3387
|
+
const parsed = new URL(u, base);
|
|
3388
|
+
return `${parsed.hostname.replace(/^www\./i, "").toLowerCase()}${parsed.pathname.replace(/\/+$/, "") || "/"}`;
|
|
3389
|
+
} catch {
|
|
3390
|
+
return null;
|
|
3391
|
+
}
|
|
3392
|
+
};
|
|
3393
|
+
const notSelf = canonical.filter((p) => {
|
|
3394
|
+
const a = norm(p.extract.canonical, p.url);
|
|
3395
|
+
const b = norm(p.url, p.url);
|
|
3396
|
+
return a === null || b === null || a !== b;
|
|
3397
|
+
});
|
|
3398
|
+
const soleH1 = (p) => {
|
|
3399
|
+
const h1s = p.extract.headings.filter((h) => h.level === 1);
|
|
3400
|
+
return h1s.length === 1 ? h1s[0].text.trim().toLowerCase() : null;
|
|
3401
|
+
};
|
|
3402
|
+
const byAddress = /* @__PURE__ */ new Map();
|
|
3403
|
+
for (const p of pages) {
|
|
3404
|
+
const key = norm(p.url, p.url);
|
|
3405
|
+
if (key !== null && !byAddress.has(key)) byAddress.set(key, p);
|
|
3406
|
+
}
|
|
3407
|
+
const contradicted = notSelf.filter((p) => {
|
|
3408
|
+
const target = norm(p.extract.canonical, p.url);
|
|
3409
|
+
const other = target === null ? void 0 : byAddress.get(target);
|
|
3410
|
+
if (other === void 0) return false;
|
|
3411
|
+
const mine = soleH1(p);
|
|
3412
|
+
const theirs = soleH1(other);
|
|
3413
|
+
return mine !== null && theirs !== null && mine !== theirs;
|
|
3414
|
+
});
|
|
3415
|
+
const byTarget = /* @__PURE__ */ new Map();
|
|
3416
|
+
for (const p of notSelf) {
|
|
3417
|
+
const t = norm(p.extract.canonical, p.url);
|
|
3418
|
+
if (t !== null) byTarget.set(t, [...byTarget.get(t) ?? [], p]);
|
|
3419
|
+
}
|
|
3420
|
+
const disagreeing = [...byTarget.entries()].filter(([, claimants]) => {
|
|
3421
|
+
const heads = claimants.map(soleH1).filter((h) => h !== null);
|
|
3422
|
+
return heads.length > 1 && new Set(heads).size > 1;
|
|
3423
|
+
});
|
|
3424
|
+
const WIPEOUT_AT = 3;
|
|
3425
|
+
const wipeout = canonical.length >= WIPEOUT_AT && notSelf.length === canonical.length && byTarget.size === 1;
|
|
3426
|
+
const collapsing = contradicted.length > 0 || disagreeing.length > 0 || wipeout;
|
|
3427
|
+
out.push(
|
|
3428
|
+
check(
|
|
3429
|
+
"canonical-self",
|
|
3430
|
+
"Pages that point at themselves",
|
|
3431
|
+
WHY_CANON_SELF,
|
|
3432
|
+
"structural",
|
|
3433
|
+
!collapsing,
|
|
3434
|
+
collapsing ? contradicted.length > 0 ? `${contradicted[0].url} names ${contradicted[0].extract.canonical} canonical, and the two carry different headlines` : wipeout ? `all ${canonical.length} pages we read name ${[...byTarget.keys()][0]} canonical` : `${disagreeing[0][1].length} pages with different headlines all name ${disagreeing[0][0]} canonical` : notSelf.length === 0 ? `all ${canonical.length} canonicals point at their own page` : `${canonical.length - notSelf.length} of ${canonical.length} point at their own page; the ${notSelf.length === 1 ? "other names a page it duplicates" : `other ${notSelf.length} name a page they duplicate`}`
|
|
3435
|
+
)
|
|
3436
|
+
);
|
|
3437
|
+
const offOrigin = canonical.filter((p) => {
|
|
3438
|
+
const h = host(p.extract.canonical, p.url);
|
|
3439
|
+
return h !== null && h !== ownHost;
|
|
3440
|
+
});
|
|
3441
|
+
out.push(
|
|
3442
|
+
check(
|
|
3443
|
+
"canonical-origin",
|
|
3444
|
+
"Canonicals that stay on your domain",
|
|
3445
|
+
WHY_CANON_ORIGIN,
|
|
3446
|
+
"structural",
|
|
3447
|
+
offOrigin.length === 0,
|
|
3448
|
+
offOrigin.length === 0 ? "every canonical stays on this domain" : `${offOrigin[0].url} points at ${offOrigin[0].extract.canonical}`
|
|
3449
|
+
)
|
|
3450
|
+
);
|
|
3451
|
+
}
|
|
3452
|
+
const alternates = readable.filter(
|
|
3453
|
+
(p) => (p.extract.links ?? []).some((l) => l.rel === "alternate" && l.hreflang)
|
|
3454
|
+
);
|
|
3455
|
+
if (alternates.length === 0) {
|
|
3456
|
+
out.push(
|
|
3457
|
+
skip(
|
|
3458
|
+
"hreflang-self",
|
|
3459
|
+
"Language alternates that name themselves",
|
|
3460
|
+
WHY_HREFLANG,
|
|
3461
|
+
"content",
|
|
3462
|
+
"this site publishes no language alternates"
|
|
3463
|
+
)
|
|
3464
|
+
);
|
|
3465
|
+
} else {
|
|
3466
|
+
const own = alternates.filter((p) => {
|
|
3467
|
+
const c = p.extract.canonical ? norm2(p.extract.canonical, p.url) : null;
|
|
3468
|
+
return c === null || c === norm2(p.url);
|
|
3469
|
+
});
|
|
3470
|
+
if (own.length === 0) {
|
|
3471
|
+
out.push(
|
|
3472
|
+
skip(
|
|
3473
|
+
"hreflang-self",
|
|
3474
|
+
"Language alternates that name themselves",
|
|
3475
|
+
WHY_HREFLANG,
|
|
3476
|
+
"content",
|
|
3477
|
+
"every page we read that publishes alternates declares a different page canonical, so the set is not its own to name"
|
|
3478
|
+
)
|
|
3479
|
+
);
|
|
3480
|
+
return out;
|
|
3481
|
+
}
|
|
3482
|
+
const missingSelf = own.filter((p) => {
|
|
3483
|
+
const mine = norm2(p.url);
|
|
3484
|
+
return !(p.extract.links ?? []).some(
|
|
3485
|
+
(l) => l.rel === "alternate" && l.hreflang && norm2(l.href, p.url) === mine
|
|
3486
|
+
);
|
|
3487
|
+
});
|
|
3488
|
+
out.push(
|
|
3489
|
+
check(
|
|
3490
|
+
"hreflang-self",
|
|
3491
|
+
"Language alternates that name themselves",
|
|
3492
|
+
WHY_HREFLANG,
|
|
3493
|
+
"content",
|
|
3494
|
+
missingSelf.length === 0,
|
|
3495
|
+
missingSelf.length === 0 ? `${own.length} ${own.length === 1 ? "page lists" : "pages list"} themselves among their alternates` : `${missingSelf[0].url} is not among its own alternates`
|
|
3496
|
+
)
|
|
3497
|
+
);
|
|
3498
|
+
}
|
|
3499
|
+
return out;
|
|
3500
|
+
}
|
|
3501
|
+
function declaredAddress(p) {
|
|
3502
|
+
return p.extract.canonical ? norm2(p.extract.canonical, p.url) : norm2(p.url);
|
|
3503
|
+
}
|
|
3504
|
+
function byDeclaredAddress(pages) {
|
|
3505
|
+
const seen = /* @__PURE__ */ new Map();
|
|
3506
|
+
for (const p of pages) {
|
|
3507
|
+
const key = declaredAddress(p) ?? p.url;
|
|
3508
|
+
if (!seen.has(key)) seen.set(key, p);
|
|
3509
|
+
}
|
|
3510
|
+
return [...seen.values()];
|
|
3511
|
+
}
|
|
3512
|
+
function hiddenFromSearch(extract) {
|
|
3513
|
+
return /\bnoindex\b/i.test(extract.metas?.["robots"] ?? "");
|
|
3514
|
+
}
|
|
3515
|
+
function sameDocument(a, b) {
|
|
3516
|
+
const h1s = (e) => e.headings.filter((h) => h.level === 1).map((h) => h.text.trim()).join("\n");
|
|
3517
|
+
return a.extract.title === b.extract.title && a.extract.metaDescription === b.extract.metaDescription && h1s(a.extract) === h1s(b.extract);
|
|
3518
|
+
}
|
|
3519
|
+
function aliasAddress(p, read) {
|
|
3520
|
+
const declared = declaredAddress(p);
|
|
3521
|
+
if (!declared || declared === norm2(p.url)) return null;
|
|
3522
|
+
const target = read.get(declared);
|
|
3523
|
+
return target !== void 0 && sameDocument(target, p) ? declared : null;
|
|
3524
|
+
}
|
|
3525
|
+
function contentPages(pages) {
|
|
3526
|
+
const read = /* @__PURE__ */ new Map();
|
|
3527
|
+
for (const p of pages) {
|
|
3528
|
+
const at = norm2(p.url);
|
|
3529
|
+
if (at && !read.has(at)) read.set(at, p);
|
|
3530
|
+
}
|
|
3531
|
+
const counted = /* @__PURE__ */ new Set();
|
|
3532
|
+
const out = [];
|
|
3533
|
+
for (const p of pages) {
|
|
3534
|
+
if (hiddenFromSearch(p.extract)) continue;
|
|
3535
|
+
const at = aliasAddress(p, read) ?? norm2(p.url) ?? p.url;
|
|
3536
|
+
if (counted.has(at)) continue;
|
|
3537
|
+
counted.add(at);
|
|
3538
|
+
out.push(p);
|
|
3539
|
+
}
|
|
3540
|
+
return out;
|
|
3541
|
+
}
|
|
3542
|
+
function norm2(u, base) {
|
|
3543
|
+
try {
|
|
3544
|
+
const parsed = new URL(u, base);
|
|
3545
|
+
const path = parsed.pathname.replace(/\/+$/, "") || "/";
|
|
3546
|
+
let decoded;
|
|
3547
|
+
try {
|
|
3548
|
+
decoded = decodeURIComponent(path);
|
|
3549
|
+
} catch {
|
|
3550
|
+
decoded = path;
|
|
3551
|
+
}
|
|
3552
|
+
return `${parsed.hostname.replace(/^www\./i, "").toLowerCase()}${decoded}`;
|
|
3553
|
+
} catch {
|
|
3554
|
+
return null;
|
|
3555
|
+
}
|
|
3556
|
+
}
|
|
3557
|
+
var FORM_PROVIDERS = /(formspree\.io|hsforms\.(?:net|com)|typeform\.com|jotform\.com|wufoo\.com|formstack\.com|getform\.io|basin\.com|netlify|web3forms\.com|formsubmit\.co|google\.com\/forms)/i;
|
|
3558
|
+
function formChecks(pages, probes = []) {
|
|
3559
|
+
const WHY_TYPES = "An email field typed as plain text gives a phone the ordinary keyboard, so the visitor hunts for the @ at exactly the moment they had decided to write to you.";
|
|
3560
|
+
const WHY_AUTOCOMPLETE = "With these, a phone offers to fill the whole form in one tap. Without them it does not offer at all.";
|
|
3561
|
+
const WHY_METHOD = "A form that submits with GET puts everything the visitor typed into the address bar, and from there into your server logs and their browser history.";
|
|
3562
|
+
const WHY_ACTION = "A form posting to an endpoint nobody owns any more looks like it worked and silently goes nowhere.";
|
|
3563
|
+
const WHY_REQUIRED = "Without it the browser cannot stop a half-filled form being sent, and the visitor finds out something was missing only if somebody tells them.";
|
|
3564
|
+
const watched = probes.filter((p) => !!p);
|
|
3565
|
+
const jsControlled = new Set(
|
|
3566
|
+
watched.filter((p) => p.emptyRefused === true).map((p) => norm2(p.url) ?? p.url)
|
|
3567
|
+
);
|
|
3568
|
+
const scriptRuns = (url) => jsControlled.has(norm2(url) ?? url);
|
|
3569
|
+
const enquiries = pages.flatMap(
|
|
3570
|
+
(p) => (p.extract.forms ?? []).filter((f) => f.kind === "enquiry").map((f) => ({ form: f, url: p.url }))
|
|
3571
|
+
);
|
|
3572
|
+
const measured = enquiries.filter((e) => e.form.fields !== void 0);
|
|
3573
|
+
if (enquiries.length === 0) {
|
|
3574
|
+
const note = "no enquiry form on the pages we read, which is reported separately";
|
|
3575
|
+
return [
|
|
3576
|
+
skip("form-field-types", "Fields typed for a phone keyboard", WHY_TYPES, "content", note),
|
|
3577
|
+
skip(
|
|
3578
|
+
"form-autocomplete",
|
|
3579
|
+
"Fields a phone can fill in one tap",
|
|
3580
|
+
WHY_AUTOCOMPLETE,
|
|
3581
|
+
"content",
|
|
3582
|
+
note
|
|
3583
|
+
),
|
|
3584
|
+
skip("form-method", "A form that does not put answers in the URL", WHY_METHOD, "quick", note),
|
|
3585
|
+
skip(
|
|
3586
|
+
"form-action",
|
|
3587
|
+
"A form that posts somewhere we recognise",
|
|
3588
|
+
WHY_ACTION,
|
|
3589
|
+
"structural",
|
|
3590
|
+
note
|
|
3591
|
+
),
|
|
3592
|
+
skip("form-required", "Fields marked as required", WHY_REQUIRED, "quick", note)
|
|
3593
|
+
];
|
|
3594
|
+
}
|
|
3595
|
+
if (measured.length === 0) {
|
|
3596
|
+
return [
|
|
3597
|
+
unknown("form-field-types", "Fields typed for a phone keyboard", WHY_TYPES, "content"),
|
|
3598
|
+
unknown(
|
|
3599
|
+
"form-autocomplete",
|
|
3600
|
+
"Fields a phone can fill in one tap",
|
|
3601
|
+
WHY_AUTOCOMPLETE,
|
|
3602
|
+
"content"
|
|
3603
|
+
),
|
|
3604
|
+
unknown("form-method", "A form that does not put answers in the URL", WHY_METHOD, "quick"),
|
|
3605
|
+
unknown("form-action", "A form that posts somewhere we recognise", WHY_ACTION, "structural"),
|
|
3606
|
+
unknown("form-required", "Fields marked as required", WHY_REQUIRED, "quick")
|
|
3607
|
+
];
|
|
3608
|
+
}
|
|
3609
|
+
const out = [];
|
|
3610
|
+
const looksLikeEmail = (f) => /e-?mail/i.test(`${f.name ?? ""} ${f.autocomplete ?? ""}`);
|
|
3611
|
+
const looksLikePhone = (f) => /\b(phone|tel|mobile)\b/i.test(`${f.name ?? ""} ${f.autocomplete ?? ""}`);
|
|
3612
|
+
const mistyped = measured.flatMap(
|
|
3613
|
+
({ form, url }) => (form.fields ?? []).filter(
|
|
3614
|
+
(f) => looksLikeEmail(f) && f.type !== "email" || looksLikePhone(f) && f.type !== "tel"
|
|
3615
|
+
).map((f) => `${url}: ${f.name ?? "a field"} is type="${f.type}"`)
|
|
3616
|
+
);
|
|
3617
|
+
out.push(
|
|
3618
|
+
check(
|
|
3619
|
+
"form-field-types",
|
|
3620
|
+
"Fields typed for a phone keyboard",
|
|
3621
|
+
WHY_TYPES,
|
|
3622
|
+
"content",
|
|
3623
|
+
mistyped.length === 0,
|
|
3624
|
+
mistyped.length === 0 ? "your email and phone fields use the right input types" : mistyped.slice(0, 2).join(", ")
|
|
3625
|
+
)
|
|
3626
|
+
);
|
|
3627
|
+
const withAuto = measured.filter(({ form }) => (form.fields ?? []).some((f) => f.autocomplete));
|
|
3628
|
+
out.push(
|
|
3629
|
+
check(
|
|
3630
|
+
"form-autocomplete",
|
|
3631
|
+
"Fields a phone can fill in one tap",
|
|
3632
|
+
WHY_AUTOCOMPLETE,
|
|
3633
|
+
"content",
|
|
3634
|
+
withAuto.length === measured.length,
|
|
3635
|
+
withAuto.length === measured.length ? `${countOf(measured.length, "enquiry form", "enquiry forms")} ${measured.length === 1 ? "carries" : "carry"} autocomplete` : `${measured.length - withAuto.length} of ${measured.length} carry none`
|
|
3636
|
+
)
|
|
3637
|
+
);
|
|
3638
|
+
const getForms = measured.filter(({ form, url }) => form.method !== "post" && !scriptRuns(url));
|
|
3639
|
+
const getButScripted = measured.filter(
|
|
3640
|
+
({ form, url }) => form.method !== "post" && scriptRuns(url)
|
|
3641
|
+
);
|
|
3642
|
+
out.push(
|
|
3643
|
+
getForms.length === 0 && getButScripted.length > 0 ? skip(
|
|
3644
|
+
"form-method",
|
|
3645
|
+
"A form that does not put answers in the URL",
|
|
3646
|
+
WHY_METHOD,
|
|
3647
|
+
"quick",
|
|
3648
|
+
`${getButScripted[0].url} is marked GET, but we watched its script handle the submit instead, so the attribute never decides`
|
|
3649
|
+
) : check(
|
|
3650
|
+
"form-method",
|
|
3651
|
+
"A form that does not put answers in the URL",
|
|
3652
|
+
WHY_METHOD,
|
|
3653
|
+
"quick",
|
|
3654
|
+
getForms.length === 0,
|
|
3655
|
+
getForms.length === 0 ? "every enquiry form posts" : `${getForms[0].url} submits with ${getForms[0].form.method.toUpperCase()}`
|
|
3656
|
+
)
|
|
3657
|
+
);
|
|
3658
|
+
const offsite = measured.filter(({ form, url }) => {
|
|
3659
|
+
const action = form.action;
|
|
3660
|
+
if (!action) return false;
|
|
3661
|
+
const h = norm2(action, url);
|
|
3662
|
+
const own = norm2(url);
|
|
3663
|
+
return h !== null && own !== null && h.split("/")[0] !== own.split("/")[0];
|
|
3664
|
+
});
|
|
3665
|
+
const unrecognised = offsite.filter(({ form }) => !FORM_PROVIDERS.test(form.action ?? ""));
|
|
3666
|
+
out.push(
|
|
3667
|
+
unrecognised.length > 0 ? unknown("form-action", "A form that posts somewhere we recognise", WHY_ACTION, "structural") : check(
|
|
3668
|
+
"form-action",
|
|
3669
|
+
"A form that posts somewhere we recognise",
|
|
3670
|
+
WHY_ACTION,
|
|
3671
|
+
"structural",
|
|
3672
|
+
true,
|
|
3673
|
+
offsite.length === 0 ? "your forms post to your own site" : `posts to ${offsite[0].form.action}`
|
|
3674
|
+
)
|
|
3675
|
+
);
|
|
3676
|
+
const bare = measured.filter(({ form }) => !(form.fields ?? []).some((f) => f.required));
|
|
3677
|
+
const noRequired = bare.filter(({ url }) => !scriptRuns(url));
|
|
3678
|
+
const bareButChecked = bare.filter(({ url }) => scriptRuns(url));
|
|
3679
|
+
out.push(
|
|
3680
|
+
noRequired.length === 0 && bareButChecked.length > 0 ? skip(
|
|
3681
|
+
"form-required",
|
|
3682
|
+
"Fields marked as required",
|
|
3683
|
+
WHY_REQUIRED,
|
|
3684
|
+
"quick",
|
|
3685
|
+
`${bareButChecked[0].url} marks none, and we watched it refuse an empty submission anyway \u2014 its script does the checking`
|
|
3686
|
+
) : check(
|
|
3687
|
+
"form-required",
|
|
3688
|
+
"Fields marked as required",
|
|
3689
|
+
WHY_REQUIRED,
|
|
3690
|
+
"quick",
|
|
3691
|
+
noRequired.length === 0,
|
|
3692
|
+
noRequired.length === 0 ? `every enquiry form marks at least one field required` : `${noRequired[0].url} marks none`
|
|
3693
|
+
)
|
|
3694
|
+
);
|
|
3695
|
+
return out;
|
|
3696
|
+
}
|
|
3697
|
+
function browserChecks(pages) {
|
|
3698
|
+
const WHY_CONSOLE = "A script that throws stops running, so whatever it was doing \u2014 a menu, a form, a gallery \u2014 silently stops working for every visitor from that point on.";
|
|
3699
|
+
const WHY_REQUESTS = "A file the page asks for and does not get is a missing image, an unstyled section or a feature that does nothing.";
|
|
3700
|
+
const WHY_OVERFLOW = "Something on the page is wider than a phone screen, so the whole page scrolls sideways. Most of your visitors are on a phone, and this is the first thing they will notice.";
|
|
3701
|
+
const WHY_TINY = "Text under twelve pixels is uncomfortable on a desktop and unreadable on a phone without pinching.";
|
|
3702
|
+
const WHY_HEAVY = "These images are downloaded at several times the size they are shown at, so every visitor pays for pixels they never see.";
|
|
3703
|
+
const withVitals = pages.filter((p) => p.vitals != null);
|
|
3704
|
+
if (withVitals.length === 0) {
|
|
3705
|
+
return [
|
|
3706
|
+
unknown("console-errors", "Pages without JavaScript errors", WHY_CONSOLE, "structural"),
|
|
3707
|
+
unknown("failed-requests", "Files the page asks for and gets", WHY_REQUESTS, "quick"),
|
|
3708
|
+
unknown("mobile-overflow", "Pages that fit a phone screen", WHY_OVERFLOW, "structural"),
|
|
3709
|
+
unknown("tiny-text", "Text big enough to read", WHY_TINY, "content"),
|
|
3710
|
+
unknown("oversized-images", "Images sized for where they are shown", WHY_HEAVY, "quick")
|
|
3711
|
+
];
|
|
3712
|
+
}
|
|
3713
|
+
const out = [];
|
|
3714
|
+
const erroring = withVitals.filter((p) => p.vitals.consoleErrors.length > 0);
|
|
3715
|
+
out.push(
|
|
3716
|
+
check(
|
|
3717
|
+
"console-errors",
|
|
3718
|
+
"Pages without JavaScript errors",
|
|
3719
|
+
WHY_CONSOLE,
|
|
3720
|
+
"structural",
|
|
3721
|
+
erroring.length === 0,
|
|
3722
|
+
erroring.length === 0 ? `none on the ${withVitals.length} pages we opened` : `${erroring[0].url}: ${erroring[0].vitals.consoleErrors[0]}`
|
|
3723
|
+
)
|
|
3724
|
+
);
|
|
3725
|
+
const broken = withVitals.flatMap(
|
|
3726
|
+
(p) => p.vitals.failedRequests.filter((r) => r.firstParty).map((r) => ({ ...r, page: p.url }))
|
|
3727
|
+
);
|
|
3728
|
+
out.push(
|
|
3729
|
+
check(
|
|
3730
|
+
"failed-requests",
|
|
3731
|
+
"Files the page asks for and gets",
|
|
3732
|
+
WHY_REQUESTS,
|
|
3733
|
+
"quick",
|
|
3734
|
+
broken.length === 0,
|
|
3735
|
+
broken.length === 0 ? "every file your pages asked for came back" : `${broken.length}: ${broken.slice(0, 2).map((b) => `${b.url}${b.status ? ` (${b.status})` : ""}`).join(", ")}`
|
|
3736
|
+
)
|
|
3737
|
+
);
|
|
3738
|
+
const measuredOverflow = withVitals.filter((p) => p.vitals.overflowAt375 !== null);
|
|
3739
|
+
if (measuredOverflow.length === 0) {
|
|
3740
|
+
out.push(
|
|
3741
|
+
unknown("mobile-overflow", "Pages that fit a phone screen", WHY_OVERFLOW, "structural")
|
|
3742
|
+
);
|
|
3743
|
+
} else {
|
|
3744
|
+
const OVERFLOW_TOLERANCE = 4;
|
|
3745
|
+
const wide = measuredOverflow.filter((p) => p.vitals.overflowAt375 > OVERFLOW_TOLERANCE);
|
|
3746
|
+
out.push(
|
|
3747
|
+
check(
|
|
3748
|
+
"mobile-overflow",
|
|
3749
|
+
"Pages that fit a phone screen",
|
|
3750
|
+
WHY_OVERFLOW,
|
|
3751
|
+
"structural",
|
|
3752
|
+
wide.length === 0,
|
|
3753
|
+
wide.length === 0 ? `all ${measuredOverflow.length} fit a 375px screen` : `${wide[0].url} is ${wide[0].vitals.overflowAt375}px wider than the screen`
|
|
3754
|
+
)
|
|
3755
|
+
);
|
|
3756
|
+
}
|
|
3757
|
+
const measuredText = withVitals.filter((p) => p.vitals.tinyText !== null);
|
|
3758
|
+
if (measuredText.length === 0) {
|
|
3759
|
+
out.push(unknown("tiny-text", "Text big enough to read", WHY_TINY, "content"));
|
|
3760
|
+
} else {
|
|
3761
|
+
const tiny = measuredText.filter((p) => p.vitals.tinyText.count > 0);
|
|
3762
|
+
out.push(
|
|
3763
|
+
check(
|
|
3764
|
+
"tiny-text",
|
|
3765
|
+
"Text big enough to read",
|
|
3766
|
+
WHY_TINY,
|
|
3767
|
+
"content",
|
|
3768
|
+
tiny.length === 0,
|
|
3769
|
+
tiny.length === 0 ? "no text under 12px on the pages we opened" : `${tiny[0].url}: ${tiny[0].vitals.tinyText.count} ${tiny[0].vitals.tinyText.count === 1 ? "passage" : "passages"}, e.g. \u201C${tiny[0].vitals.tinyText.sample}\u201D`
|
|
3770
|
+
)
|
|
3771
|
+
);
|
|
3772
|
+
}
|
|
3773
|
+
const heavy = withVitals.flatMap((p) => p.vitals.oversizedImages);
|
|
3774
|
+
out.push(
|
|
3775
|
+
check(
|
|
3776
|
+
"oversized-images",
|
|
3777
|
+
"Images sized for where they are shown",
|
|
3778
|
+
WHY_HEAVY,
|
|
3779
|
+
"quick",
|
|
3780
|
+
heavy.length === 0,
|
|
3781
|
+
heavy.length === 0 ? "every image is close to the size it is drawn at" : `${heavy[0].src} is ${heavy[0].naturalWidth}px wide and drawn at ${heavy[0].renderedWidth}px`
|
|
3782
|
+
)
|
|
3783
|
+
);
|
|
3784
|
+
return out;
|
|
3785
|
+
}
|
|
3786
|
+
function clipRecord(record, max = 90) {
|
|
3787
|
+
const one = record.replace(/\s+/g, " ").trim();
|
|
3788
|
+
return one.length <= max ? one : `${one.slice(0, max - 1).trimEnd()}\u2026`;
|
|
3789
|
+
}
|
|
3790
|
+
function dnsChecks(dns) {
|
|
3791
|
+
const WHY_SPF = "Without it, anyone can send email that appears to come from your address, and your own mail is more likely to be filed as spam.";
|
|
3792
|
+
const WHY_DMARC = "SPF says who may send as you. DMARC is what tells the receiving mail server to act on that, and to report attempts to you. Without it the first one is advisory.";
|
|
3793
|
+
const WHY_MX = "There is no mail server listed for your domain, so email sent to any address at it is rejected.";
|
|
3794
|
+
const WHY_CONTACT_MX = "The address you publish is on a domain with no mail server, so anything sent to it bounces.";
|
|
3795
|
+
const WHY_EXPIRY = "Registration dates are public; whether it renews automatically is not, so this is one to confirm at your registrar rather than a fault we can see. When a registration does lapse the site and the email stop together, usually on a weekend, and getting them back is neither quick nor certain.";
|
|
3796
|
+
if (!dns || !dns.measured) {
|
|
3797
|
+
return [
|
|
3798
|
+
unknown("dns-spf", "An SPF record, so your email is trusted", WHY_SPF, "quick"),
|
|
3799
|
+
unknown("dns-dmarc", "A DMARC record, so SPF is enforced", WHY_DMARC, "quick"),
|
|
3800
|
+
unknown("dns-mx", "A mail server for your domain", WHY_MX, "structural"),
|
|
3801
|
+
unknown("dns-contact-mx", "An email address that can receive mail", WHY_CONTACT_MX, "quick"),
|
|
3802
|
+
unknown("domain-expiry", "A domain registration with room to spare", WHY_EXPIRY, "quick")
|
|
3803
|
+
];
|
|
3804
|
+
}
|
|
3805
|
+
const out = [];
|
|
3806
|
+
out.push(
|
|
3807
|
+
dns.spf === void 0 ? unknown("dns-spf", "An SPF record, so your email is trusted", WHY_SPF, "quick") : check(
|
|
3808
|
+
"dns-spf",
|
|
3809
|
+
"An SPF record, so your email is trusted",
|
|
3810
|
+
WHY_SPF,
|
|
3811
|
+
"quick",
|
|
3812
|
+
dns.spf !== null,
|
|
3813
|
+
(dns.spf ? clipRecord(dns.spf) : null) ?? `no v=spf1 record on ${dns.domain}`
|
|
3814
|
+
)
|
|
3815
|
+
);
|
|
3816
|
+
out.push(
|
|
3817
|
+
dns.dmarc === void 0 ? unknown("dns-dmarc", "A DMARC record, so SPF is enforced", WHY_DMARC, "quick") : check(
|
|
3818
|
+
"dns-dmarc",
|
|
3819
|
+
"A DMARC record, so SPF is enforced",
|
|
3820
|
+
WHY_DMARC,
|
|
3821
|
+
"quick",
|
|
3822
|
+
dns.dmarc !== null,
|
|
3823
|
+
(dns.dmarc ? clipRecord(dns.dmarc) : null) ?? `no record at _dmarc.${dns.domain}`
|
|
3824
|
+
)
|
|
3825
|
+
);
|
|
3826
|
+
out.push(
|
|
3827
|
+
dns.mx === void 0 ? unknown("dns-mx", "A mail server for your domain", WHY_MX, "structural") : check(
|
|
3828
|
+
"dns-mx",
|
|
3829
|
+
"A mail server for your domain",
|
|
3830
|
+
WHY_MX,
|
|
3831
|
+
"structural",
|
|
3832
|
+
dns.mx.length > 0,
|
|
3833
|
+
dns.mx.length > 0 ? dns.mx.slice(0, 2).join(", ") : `no MX record on ${dns.domain}`
|
|
3834
|
+
)
|
|
3835
|
+
);
|
|
3836
|
+
out.push(
|
|
3837
|
+
dns.contactMx === void 0 ? unknown("dns-contact-mx", "An email address that can receive mail", WHY_CONTACT_MX, "quick") : dns.contactMx === null ? skip(
|
|
3838
|
+
"dns-contact-mx",
|
|
3839
|
+
"An email address that can receive mail",
|
|
3840
|
+
WHY_CONTACT_MX,
|
|
3841
|
+
"quick",
|
|
3842
|
+
"the addresses you publish are on your own domain, which is checked above"
|
|
3843
|
+
) : check(
|
|
3844
|
+
"dns-contact-mx",
|
|
3845
|
+
"An email address that can receive mail",
|
|
3846
|
+
WHY_CONTACT_MX,
|
|
3847
|
+
"quick",
|
|
3848
|
+
dns.contactMx.hasMx,
|
|
3849
|
+
`${dns.contactMx.domain}${dns.contactMx.hasMx ? " accepts mail" : " has no mail server"}`
|
|
3850
|
+
)
|
|
3851
|
+
);
|
|
3852
|
+
const DAYS = 30;
|
|
3853
|
+
if (dns.expiresAt === void 0) {
|
|
3854
|
+
out.push(
|
|
3855
|
+
unknown("domain-expiry", "A domain registration with room to spare", WHY_EXPIRY, "quick")
|
|
3856
|
+
);
|
|
3857
|
+
} else {
|
|
3858
|
+
const on = dns.expiresAt.slice(0, 10);
|
|
3859
|
+
const days = Math.round((Date.parse(dns.expiresAt) - Date.now()) / 864e5);
|
|
3860
|
+
const evidence = days < 0 ? `the registration lapsed on ${on}` : days > DAYS ? `registered through ${on}` : `registered through ${on} \u2014 ${days === 1 ? "1 day" : `${days} days`} away`;
|
|
3861
|
+
out.push(
|
|
3862
|
+
check(
|
|
3863
|
+
"domain-expiry",
|
|
3864
|
+
"A domain registration with room to spare",
|
|
3865
|
+
WHY_EXPIRY,
|
|
3866
|
+
"quick",
|
|
3867
|
+
days > DAYS,
|
|
3868
|
+
evidence
|
|
3869
|
+
)
|
|
3870
|
+
);
|
|
3871
|
+
}
|
|
3872
|
+
return out;
|
|
3873
|
+
}
|
|
3874
|
+
function httpChecks(http) {
|
|
3875
|
+
const WHY_FAVICON = "The icon in the browser tab and the bookmark bar. When the file is missing the tab shows a blank sheet, which is the difference between a site that looks maintained and one that does not.";
|
|
3876
|
+
const WHY_UPGRADE = "Somebody typing your address without https, or following an old link, should land on the secure version in one step. Every extra redirect is delay on the first impression, and a chain that ends on http means the page is served in the clear.";
|
|
3877
|
+
const WHY_STABLE = "Two requests seconds apart got different answers, which is what an overloaded or half-deployed server looks like from outside. Some visitors are seeing the bad one.";
|
|
3878
|
+
const WHY_SLASH = "When /about and /about/ both answer with the same page and neither says which is real, search engines index both and split the credit between them.";
|
|
3879
|
+
const WHY_INDEX = "Your homepage answers at two addresses. Links, shares and search results scatter across both instead of accumulating on one.";
|
|
3880
|
+
const WHY_CASE = "The same page answers at two spellings, so the same content competes with itself in search results.";
|
|
3881
|
+
const WHY_SITEMAP_LIVE = "A sitemap is a list you hand to search engines saying 'these pages exist'. Entries that no longer answer waste the crawl budget you were trying to direct.";
|
|
3882
|
+
const WHY_EXTERNAL = "A link on your site that leads nowhere is a dead end for the visitor and a small mark against the page for a search engine.";
|
|
3883
|
+
const WHY_OG_SERVED = "This is the picture that appears when your page is shared in a message, on LinkedIn, or in a Slack channel. When the file does not load, the share renders as a bare grey box.";
|
|
3884
|
+
const WHY_OG_SIZE = `A share image under ${MIN_OG_IMAGE_EDGE} pixels on either edge is either cropped to nothing or dropped entirely by the platform showing it.`;
|
|
3885
|
+
const WHY_LOGO = "A logo that does not load leaves a broken-image icon at the top of every page, and it is the first thing on the page.";
|
|
3886
|
+
const WHY_CHAINS = "A link that redirects more than once makes every visitor wait through each hop, and search engines pass less credit along a chain than a direct link.";
|
|
3887
|
+
const ALL = [
|
|
3888
|
+
["favicon-served", "A favicon that actually loads", WHY_FAVICON, "quick"],
|
|
3889
|
+
["https-upgrade", "http:// reaching https:// without a detour", WHY_UPGRADE, "structural"],
|
|
3890
|
+
["home-stable", "A homepage that answers the same way twice", WHY_STABLE, "structural"],
|
|
3891
|
+
["trailing-slash", "One address per page, slash or no slash", WHY_SLASH, "structural"],
|
|
3892
|
+
["index-alias", "A homepage that lives at one address", WHY_INDEX, "structural"],
|
|
3893
|
+
["case-alias", "Paths that do not answer to two spellings", WHY_CASE, "structural"],
|
|
3894
|
+
["sitemap-urls-live", "A sitemap whose pages still exist", WHY_SITEMAP_LIVE, "quick"],
|
|
3895
|
+
["external-links-live", "Outbound links that still work", WHY_EXTERNAL, "quick"],
|
|
3896
|
+
["og-image-served", "A share image that loads", WHY_OG_SERVED, "quick"],
|
|
3897
|
+
["og-image-size", "A share image big enough to render", WHY_OG_SIZE, "quick"],
|
|
3898
|
+
["logo-served", "A logo that loads", WHY_LOGO, "quick"],
|
|
3899
|
+
["redirect-chains", "Internal links that go straight there", WHY_CHAINS, "quick"]
|
|
3900
|
+
];
|
|
3901
|
+
const meta = new Map(ALL.map(([key, label, why, scope]) => [key, { label, why, scope }]));
|
|
3902
|
+
const unmeasuredAll = () => ALL.map(([key, label, why, scope]) => unknown(key, label, why, scope));
|
|
3903
|
+
if (!http || !http.measured) return unmeasuredAll();
|
|
3904
|
+
const out = [];
|
|
3905
|
+
const add = (key, value, verdict) => {
|
|
3906
|
+
const m = meta.get(key);
|
|
3907
|
+
if (value === void 0) return void out.push(unknown(key, m.label, m.why, m.scope));
|
|
3908
|
+
const v = verdict();
|
|
3909
|
+
if (v === void 0) return void out.push(unknown(key, m.label, m.why, m.scope));
|
|
3910
|
+
if (v === null) return void out.push(skip(key, m.label, m.why, m.scope, notApplicable(key)));
|
|
3911
|
+
out.push(check(key, m.label, m.why, m.scope, v.ok, v.evidence));
|
|
3912
|
+
};
|
|
3913
|
+
const notApplicable = (key) => key === "index-alias" ? "/index.html does not answer, which is the behaviour we were hoping for" : key === "case-alias" ? "a differently-cased path 404s, which is the behaviour we were hoping for" : key === "og-image-served" || key === "og-image-size" ? "the homepage declares no share image, which is covered above" : key === "sitemap-urls-live" ? "there is no sitemap to check, which is covered above" : key === "external-links-live" ? "this site does not link out to anywhere else" : "there was nothing of this kind to check";
|
|
3914
|
+
const fromVerdict = (v, ok, bad) => v === "unverified" ? void 0 : { ok: v === "ok", evidence: v === "ok" ? ok : bad };
|
|
3915
|
+
add("favicon-served", http.favicon, () => {
|
|
3916
|
+
const f = http.favicon;
|
|
3917
|
+
return fromVerdict(
|
|
3918
|
+
f.verdict,
|
|
3919
|
+
f.declared ? "the declared icon is served" : "/favicon.ico is served",
|
|
3920
|
+
f.declared ? `the icon at ${f.url} does not load` : "no icon at /favicon.ico"
|
|
3921
|
+
);
|
|
3922
|
+
});
|
|
3923
|
+
add("https-upgrade", http.httpUpgrade, () => {
|
|
3924
|
+
const u = http.httpUpgrade;
|
|
3925
|
+
if (!u.https) return { ok: false, evidence: "http:// does not end up on https://" };
|
|
3926
|
+
const hops = u.hops;
|
|
3927
|
+
if (hops === null) return { ok: true, evidence: "http:// arrives on https://" };
|
|
3928
|
+
return {
|
|
3929
|
+
ok: hops <= 2,
|
|
3930
|
+
evidence: hops <= 2 ? `http:// arrives on https:// in ${hops === 1 ? "one hop" : `${hops} hops`}` : `http:// takes ${hops} redirects to reach https://`
|
|
3931
|
+
};
|
|
3932
|
+
});
|
|
3933
|
+
add("home-stable", http.homeSamples, () => {
|
|
3934
|
+
const [a, b] = http.homeSamples;
|
|
3935
|
+
if (a === null || b === null) return null;
|
|
3936
|
+
if (a === b)
|
|
3937
|
+
return a >= 200 && a < 400 ? { ok: true, evidence: `two requests, both ${a}` } : null;
|
|
3938
|
+
return { ok: false, evidence: `two requests seconds apart answered ${a} and ${b}` };
|
|
3939
|
+
});
|
|
3940
|
+
add("trailing-slash", http.trailingSlash, () => {
|
|
3941
|
+
const t = http.trailingSlash;
|
|
3942
|
+
return { ok: t.settled, evidence: `${t.path} \u2014 ${t.detail}` };
|
|
3943
|
+
});
|
|
3944
|
+
add("index-alias", http.indexAlias, () => {
|
|
3945
|
+
const i = http.indexAlias;
|
|
3946
|
+
return i === null ? null : { ok: !i.duplicate, evidence: `/index.html \u2014 ${i.detail}` };
|
|
3947
|
+
});
|
|
3948
|
+
add("case-alias", http.caseAlias, () => {
|
|
3949
|
+
const c = http.caseAlias;
|
|
3950
|
+
return c === null ? null : { ok: !c.duplicate, evidence: `${c.path} \u2014 ${c.detail}` };
|
|
3951
|
+
});
|
|
3952
|
+
add("sitemap-urls-live", http.sitemapUrls, () => {
|
|
3953
|
+
const s = http.sitemapUrls;
|
|
3954
|
+
if (s === null) return null;
|
|
3955
|
+
const { checked, total, broken } = s;
|
|
3956
|
+
return broken.length === 0 ? {
|
|
3957
|
+
ok: true,
|
|
3958
|
+
evidence: checked >= total ? `all ${total} answer` : `we sampled ${checked} of ${total}; all of them answer`
|
|
3959
|
+
} : {
|
|
3960
|
+
ok: false,
|
|
3961
|
+
evidence: `${broken.length} of the ${checked} we sampled are gone \u2014 ${broken[0].url}`
|
|
3962
|
+
};
|
|
3963
|
+
});
|
|
3964
|
+
add("external-links-live", http.externalLinks, () => {
|
|
3965
|
+
const e = http.externalLinks;
|
|
3966
|
+
if (e === null) return null;
|
|
3967
|
+
const { checked, total, broken } = e;
|
|
3968
|
+
return broken.length === 0 ? {
|
|
3969
|
+
ok: true,
|
|
3970
|
+
evidence: checked >= total ? `${countOf(total, "outbound link", "outbound links")} ${total === 1 ? "answers" : "answer"}` : `we sampled ${checked} of ${total} outbound links; all of them answer`
|
|
3971
|
+
} : {
|
|
3972
|
+
ok: false,
|
|
3973
|
+
evidence: `${broken.length} of the ${checked} we sampled are dead \u2014 ${broken[0].url}`
|
|
3974
|
+
};
|
|
3975
|
+
});
|
|
3976
|
+
add("og-image-served", http.ogImage, () => {
|
|
3977
|
+
const o = http.ogImage;
|
|
3978
|
+
if (o === null) return null;
|
|
3979
|
+
return fromVerdict(
|
|
3980
|
+
o.verdict,
|
|
3981
|
+
"the share image loads",
|
|
3982
|
+
`the share image does not load \u2014 ${o.url}`
|
|
3983
|
+
);
|
|
3984
|
+
});
|
|
3985
|
+
add("og-image-size", http.ogImage, () => {
|
|
3986
|
+
const o = http.ogImage;
|
|
3987
|
+
if (o === null) return null;
|
|
3988
|
+
if (o.width === null || o.height === null) return null;
|
|
3989
|
+
const small = o.width < MIN_OG_IMAGE_EDGE || o.height < MIN_OG_IMAGE_EDGE;
|
|
3990
|
+
return { ok: !small, evidence: `${o.width}\xD7${o.height}` };
|
|
3991
|
+
});
|
|
3992
|
+
add("logo-served", http.logo, () => {
|
|
3993
|
+
const l = http.logo;
|
|
3994
|
+
if (l === null) return null;
|
|
3995
|
+
return fromVerdict(l.verdict, `${l.how} loads`, `${l.how} does not load \u2014 ${l.url}`);
|
|
3996
|
+
});
|
|
3997
|
+
add("redirect-chains", http.redirectChains, () => {
|
|
3998
|
+
const r = http.redirectChains;
|
|
3999
|
+
return r.chained.length === 0 ? { ok: true, evidence: `${r.checked} internal links, none redirecting twice` } : {
|
|
4000
|
+
ok: false,
|
|
4001
|
+
evidence: `${r.chained.length} of ${r.checked} take ${r.chained[0].hops} hops \u2014 ${r.chained[0].url}`
|
|
4002
|
+
};
|
|
4003
|
+
});
|
|
4004
|
+
return out;
|
|
4005
|
+
}
|
|
4006
|
+
function formInteractionChecks(probes) {
|
|
4007
|
+
const WHY_EMPTY = "Somebody who taps send too early should be told what is missing. A form that submits anyway puts a blank enquiry in your inbox and tells them it worked, so they never follow up and you have nothing to follow up on.";
|
|
4008
|
+
const WHY_EMAIL = "A mistyped address is the one mistake that costs you the whole enquiry: the message arrives, and there is no way to answer it. Catching it while the visitor is still on the page is the only chance to fix it.";
|
|
4009
|
+
const probe = probes.find((p) => p) ?? null;
|
|
4010
|
+
const neverTried = probes.every((p) => p === void 0);
|
|
4011
|
+
if (neverTried) {
|
|
4012
|
+
const NOTE = "we did not try a form on this run";
|
|
4013
|
+
return [
|
|
4014
|
+
unknown(
|
|
4015
|
+
"form-rejects-empty",
|
|
4016
|
+
"A form that catches an empty submission",
|
|
4017
|
+
WHY_EMPTY,
|
|
4018
|
+
"quick",
|
|
4019
|
+
NOTE
|
|
4020
|
+
),
|
|
4021
|
+
unknown(
|
|
4022
|
+
"form-rejects-bad-email",
|
|
4023
|
+
"A form that catches a mistyped email",
|
|
4024
|
+
WHY_EMAIL,
|
|
4025
|
+
"quick",
|
|
4026
|
+
NOTE
|
|
4027
|
+
)
|
|
4028
|
+
];
|
|
4029
|
+
}
|
|
4030
|
+
if (!probe) {
|
|
4031
|
+
return [
|
|
4032
|
+
skip(
|
|
4033
|
+
"form-rejects-empty",
|
|
4034
|
+
"A form that catches an empty submission",
|
|
4035
|
+
WHY_EMPTY,
|
|
4036
|
+
"quick",
|
|
4037
|
+
"we found no enquiry form to try"
|
|
4038
|
+
),
|
|
4039
|
+
skip(
|
|
4040
|
+
"form-rejects-bad-email",
|
|
4041
|
+
"A form that catches a mistyped email",
|
|
4042
|
+
WHY_EMAIL,
|
|
4043
|
+
"quick",
|
|
4044
|
+
"we found no enquiry form to try"
|
|
4045
|
+
)
|
|
4046
|
+
];
|
|
4047
|
+
}
|
|
4048
|
+
const out = [
|
|
4049
|
+
probe.emptyRefused === void 0 ? unknown("form-rejects-empty", "A form that catches an empty submission", WHY_EMPTY, "quick") : check(
|
|
4050
|
+
"form-rejects-empty",
|
|
4051
|
+
"A form that catches an empty submission",
|
|
4052
|
+
WHY_EMPTY,
|
|
4053
|
+
"quick",
|
|
4054
|
+
probe.emptyRefused,
|
|
4055
|
+
probe.emptyHow ?? ""
|
|
4056
|
+
)
|
|
4057
|
+
];
|
|
4058
|
+
out.push(
|
|
4059
|
+
probe.invalidEmailRefused === null ? skip(
|
|
4060
|
+
"form-rejects-bad-email",
|
|
4061
|
+
"A form that catches a mistyped email",
|
|
4062
|
+
WHY_EMAIL,
|
|
4063
|
+
"quick",
|
|
4064
|
+
"this form does not ask for an email address"
|
|
4065
|
+
) : probe.invalidEmailRefused === void 0 ? unknown(
|
|
4066
|
+
"form-rejects-bad-email",
|
|
4067
|
+
"A form that catches a mistyped email",
|
|
4068
|
+
WHY_EMAIL,
|
|
4069
|
+
"quick"
|
|
4070
|
+
) : check(
|
|
4071
|
+
"form-rejects-bad-email",
|
|
4072
|
+
"A form that catches a mistyped email",
|
|
4073
|
+
WHY_EMAIL,
|
|
4074
|
+
"quick",
|
|
4075
|
+
probe.invalidEmailRefused,
|
|
4076
|
+
probe.invalidEmailHow ?? ""
|
|
4077
|
+
)
|
|
4078
|
+
);
|
|
4079
|
+
return out;
|
|
4080
|
+
}
|
|
4081
|
+
var TIER0_CHECK_KEYS = [
|
|
4082
|
+
"dead-links",
|
|
4083
|
+
"staging-links",
|
|
4084
|
+
"insecure-links",
|
|
4085
|
+
"link-text",
|
|
4086
|
+
"tel-dialable",
|
|
4087
|
+
"social-placeholders",
|
|
4088
|
+
"nav-consistency",
|
|
4089
|
+
"lorem",
|
|
4090
|
+
"placeholder-copy",
|
|
4091
|
+
"template-leakage",
|
|
4092
|
+
"mojibake",
|
|
4093
|
+
"under-construction",
|
|
4094
|
+
"name-in-title",
|
|
4095
|
+
"h1-present",
|
|
4096
|
+
"h1-one-headline",
|
|
4097
|
+
"h1-not-name",
|
|
4098
|
+
"h1-distinct",
|
|
4099
|
+
"schema-org-complete",
|
|
4100
|
+
"schema-local-complete",
|
|
4101
|
+
"schema-url-matches",
|
|
4102
|
+
"schema-phone-matches",
|
|
4103
|
+
"schema-self-review",
|
|
4104
|
+
...SECURITY_HEADER_CHECKS.map((h) => `header-${h.header}`),
|
|
4105
|
+
"header-version-leak",
|
|
4106
|
+
"header-compression",
|
|
4107
|
+
"header-hsts-age",
|
|
4108
|
+
"robots-not-blocking",
|
|
4109
|
+
"robots-names-sitemap",
|
|
4110
|
+
"sitemap-coverage",
|
|
4111
|
+
"llms-txt",
|
|
4112
|
+
"analytics"
|
|
4113
|
+
];
|
|
4114
|
+
function runSiteChecks(crawl, checks, businessName = null, dns = null, http = null) {
|
|
4115
|
+
const set = usablePages(crawl.pages);
|
|
4116
|
+
const pages = set.pages.map((p) => ({
|
|
4117
|
+
url: p.page.url,
|
|
4118
|
+
extract: p.extract,
|
|
4119
|
+
// Carried through so the browser checks can read what the render pass
|
|
4120
|
+
// observed without a second traversal of the capture list.
|
|
4121
|
+
vitals: p.page.vitals
|
|
4122
|
+
}));
|
|
4123
|
+
const read = /* @__PURE__ */ new Map();
|
|
4124
|
+
for (const p of pages) {
|
|
4125
|
+
const at = norm2(p.url);
|
|
4126
|
+
if (at && !read.has(at)) read.set(at, p);
|
|
4127
|
+
}
|
|
4128
|
+
const linked = /* @__PURE__ */ new Set();
|
|
4129
|
+
for (const { url, extract } of pages) {
|
|
4130
|
+
for (const a of extract.anchors ?? []) {
|
|
4131
|
+
const abs = resolveNavigable(a.href, url);
|
|
4132
|
+
if (!abs) continue;
|
|
4133
|
+
try {
|
|
4134
|
+
if (new URL(abs).origin !== crawl.origin) continue;
|
|
4135
|
+
} catch {
|
|
4136
|
+
continue;
|
|
4137
|
+
}
|
|
4138
|
+
const at = norm2(abs);
|
|
4139
|
+
if (!at) continue;
|
|
4140
|
+
const target = read.get(at);
|
|
4141
|
+
if (target && hiddenFromSearch(target.extract)) continue;
|
|
4142
|
+
linked.add(target ? aliasAddress(target, read) ?? at : at);
|
|
4143
|
+
}
|
|
4144
|
+
}
|
|
4145
|
+
const phones = (checks?.consistency?.phones ?? []).map((p) => p.normalized);
|
|
4146
|
+
const headersMeasured = Object.keys(crawl.homeHeaders ?? {}).length > 0;
|
|
4147
|
+
return [
|
|
4148
|
+
...anchorChecks(pages, crawl.origin),
|
|
4149
|
+
...textChecks(pages, businessName),
|
|
4150
|
+
...headingChecks(pages, businessName),
|
|
4151
|
+
...schemaChecks(pages, crawl.origin, phones),
|
|
4152
|
+
...metaChecks(pages),
|
|
4153
|
+
...linkChecks(
|
|
4154
|
+
pages,
|
|
4155
|
+
crawl.origin,
|
|
4156
|
+
http?.favicon !== void 0 && http.favicon.verdict !== "unverified"
|
|
4157
|
+
),
|
|
4158
|
+
...formChecks(
|
|
4159
|
+
pages,
|
|
4160
|
+
crawl.pages.map((p) => p.formProbe)
|
|
4161
|
+
),
|
|
4162
|
+
...browserChecks(pages),
|
|
4163
|
+
...dnsChecks(dns),
|
|
4164
|
+
...httpChecks(http),
|
|
4165
|
+
...formInteractionChecks(crawl.pages.map((p) => p.formProbe)),
|
|
4166
|
+
...headerChecks(crawl.homeHeaders ?? {}, headersMeasured),
|
|
4167
|
+
...sidecarChecks(crawl, linked),
|
|
4168
|
+
analyticsCheck(pages)
|
|
4169
|
+
];
|
|
4170
|
+
}
|
|
4171
|
+
|
|
4172
|
+
export {
|
|
4173
|
+
usablePages,
|
|
4174
|
+
summarizeAccessibility,
|
|
4175
|
+
AI_AGENTS,
|
|
4176
|
+
CLASSICAL_AGENTS,
|
|
4177
|
+
USER_AGENT,
|
|
4178
|
+
sleep,
|
|
4179
|
+
pacedEach,
|
|
4180
|
+
crawlSite,
|
|
4181
|
+
readCapped,
|
|
4182
|
+
defaultCrawlDeps,
|
|
4183
|
+
canonicalizeUrl,
|
|
4184
|
+
resolveNavigable,
|
|
4185
|
+
buildJourney,
|
|
4186
|
+
LAYER_ORDER,
|
|
4187
|
+
LAYER_LABELS,
|
|
4188
|
+
readStack,
|
|
4189
|
+
probeHttp,
|
|
4190
|
+
defaultHttpProbeDeps,
|
|
4191
|
+
tally,
|
|
4192
|
+
runSiteChecks
|
|
4193
|
+
};
|
|
4194
|
+
//# sourceMappingURL=chunk-VELWHWOW.js.map
|