@eightmedia/flightcheck 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/LICENSE +691 -0
- package/README.md +106 -2
- package/dist/analyze/fixManifest.d.ts +65 -0
- package/dist/analyze/fixManifest.d.ts.map +1 -0
- package/dist/analyze/fixManifest.js +588 -0
- package/dist/analyze/report.d.ts +283 -0
- package/dist/analyze/report.d.ts.map +1 -0
- package/dist/analyze/report.js +726 -0
- package/dist/analyze/reportTier1.d.ts +46 -0
- package/dist/analyze/reportTier1.d.ts.map +1 -0
- package/dist/analyze/reportTier1.js +251 -0
- package/dist/analyze/stack.d.ts +33 -0
- package/dist/analyze/stack.d.ts.map +1 -0
- package/dist/analyze/stack.js +188 -0
- package/dist/analyze/structuredReport.d.ts +62 -0
- package/dist/analyze/structuredReport.d.ts.map +1 -0
- package/dist/analyze/structuredReport.js +524 -0
- package/dist/analyze.d.ts +3 -0
- package/dist/analyze.d.ts.map +1 -0
- package/dist/analyze.js +40 -0
- package/dist/checks/agent-ready.d.ts +63 -0
- package/dist/checks/agent-ready.d.ts.map +1 -0
- package/dist/checks/agent-ready.js +219 -0
- package/dist/checks/analyze.d.ts +15 -0
- package/dist/checks/analyze.d.ts.map +1 -0
- package/dist/checks/analyze.js +63 -0
- package/dist/checks/crawl.d.ts +28 -0
- package/dist/checks/crawl.d.ts.map +1 -0
- package/dist/checks/crawl.js +86 -0
- package/dist/checks/dns.d.ts +25 -0
- package/dist/checks/dns.d.ts.map +1 -0
- package/dist/checks/dns.js +80 -0
- package/dist/checks/index.d.ts +24 -0
- package/dist/checks/index.d.ts.map +1 -0
- package/dist/checks/index.js +12 -0
- package/dist/checks/issues.d.ts +8 -0
- package/dist/checks/issues.d.ts.map +1 -0
- package/dist/checks/issues.js +3 -0
- package/dist/checks/redirect.d.ts +38 -0
- package/dist/checks/redirect.d.ts.map +1 -0
- package/dist/checks/redirect.js +209 -0
- package/dist/checks/registry.d.ts +8 -0
- package/dist/checks/registry.d.ts.map +1 -0
- package/dist/checks/registry.js +275 -0
- package/dist/checks/robots-txt.d.ts +16 -0
- package/dist/checks/robots-txt.d.ts.map +1 -0
- package/dist/checks/robots-txt.js +99 -0
- package/dist/checks/runner.d.ts +9 -0
- package/dist/checks/runner.d.ts.map +1 -0
- package/dist/checks/runner.js +110 -0
- package/dist/checks/security-headers.d.ts +12 -0
- package/dist/checks/security-headers.d.ts.map +1 -0
- package/dist/checks/security-headers.js +53 -0
- package/dist/checks/sitemap.d.ts +36 -0
- package/dist/checks/sitemap.d.ts.map +1 -0
- package/dist/checks/sitemap.js +140 -0
- package/dist/checks/types.d.ts +72 -0
- package/dist/checks/types.d.ts.map +1 -0
- package/dist/checks/types.js +20 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +125 -0
- package/dist/crawler/checkGroups.d.ts +7 -0
- package/dist/crawler/checkGroups.d.ts.map +1 -0
- package/dist/crawler/checkGroups.js +29 -0
- package/dist/crawler/crawlState.d.ts +7 -0
- package/dist/crawler/crawlState.d.ts.map +1 -0
- package/dist/crawler/crawlState.js +6 -0
- package/dist/crawler/defaults.d.ts +11 -0
- package/dist/crawler/defaults.d.ts.map +1 -0
- package/dist/crawler/defaults.js +10 -0
- package/dist/crawler/extractors.d.ts +5 -0
- package/dist/crawler/extractors.d.ts.map +1 -0
- package/dist/crawler/extractors.js +169 -0
- package/dist/crawler/jsAssets.d.ts +2 -0
- package/dist/crawler/jsAssets.d.ts.map +1 -0
- package/dist/crawler/jsAssets.js +22 -0
- package/dist/crawler/outputPaths.d.ts +14 -0
- package/dist/crawler/outputPaths.d.ts.map +1 -0
- package/dist/crawler/outputPaths.js +51 -0
- package/dist/crawler/reporter.d.ts +3 -0
- package/dist/crawler/reporter.d.ts.map +1 -0
- package/dist/crawler/reporter.js +12 -0
- package/dist/crawler/robots.d.ts +5 -0
- package/dist/crawler/robots.d.ts.map +1 -0
- package/dist/crawler/robots.js +29 -0
- package/dist/crawler/runCrawler.d.ts +22 -0
- package/dist/crawler/runCrawler.d.ts.map +1 -0
- package/dist/crawler/runCrawler.js +172 -0
- package/dist/crawler/schema.d.ts +87 -0
- package/dist/crawler/schema.d.ts.map +1 -0
- package/dist/crawler/schema.js +44 -0
- package/dist/crawler/secrets.d.ts +4 -0
- package/dist/crawler/secrets.d.ts.map +1 -0
- package/dist/crawler/secrets.js +70 -0
- package/dist/crawler/sitemap.d.ts +8 -0
- package/dist/crawler/sitemap.d.ts.map +1 -0
- package/dist/crawler/sitemap.js +105 -0
- package/dist/crawler/stack.d.ts +3 -0
- package/dist/crawler/stack.d.ts.map +1 -0
- package/dist/crawler/stack.js +1 -0
- package/dist/crawler/stackDetect.d.ts +24 -0
- package/dist/crawler/stackDetect.d.ts.map +1 -0
- package/dist/crawler/stackDetect.js +604 -0
- package/dist/crawler/tier1Extractors.d.ts +5 -0
- package/dist/crawler/tier1Extractors.d.ts.map +1 -0
- package/dist/crawler/tier1Extractors.js +255 -0
- package/dist/crawler/tracking.d.ts +8 -0
- package/dist/crawler/tracking.d.ts.map +1 -0
- package/dist/crawler/tracking.js +50 -0
- package/dist/crawler/urlUtils.d.ts +12 -0
- package/dist/crawler/urlUtils.d.ts.map +1 -0
- package/dist/crawler/urlUtils.js +53 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +7 -0
- package/docs/agent-fixes.md +17 -0
- package/docs/agent-ready.md +44 -0
- package/docs/analyze.md +103 -0
- package/docs/check-api.md +142 -0
- package/docs/checks.md +116 -0
- package/docs/crawler.md +158 -0
- package/docs/index.md +24 -0
- package/docs/qc-checks-plan.md +796 -0
- package/docs/stack.md +99 -0
- package/package.json +86 -4
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
function crawlIssue(id, severity, message) {
|
|
2
|
+
return { id, severity, message };
|
|
3
|
+
}
|
|
4
|
+
function parsePixelSize(value) {
|
|
5
|
+
if (!value)
|
|
6
|
+
return null;
|
|
7
|
+
const match = value.trim().match(/^(\d+(?:\.\d+)?)px$/i);
|
|
8
|
+
return match ? Number(match[1]) : null;
|
|
9
|
+
}
|
|
10
|
+
function isDecorativeImage($img) {
|
|
11
|
+
if ($img.attr("aria-hidden") === "true")
|
|
12
|
+
return true;
|
|
13
|
+
if (($img.attr("role") ?? "").toLowerCase() === "presentation")
|
|
14
|
+
return true;
|
|
15
|
+
const width = parsePixelSize($img.attr("width")) ?? parsePixelSize($img.attr("style")?.match(/width:\s*(\d+px)/i)?.[1]);
|
|
16
|
+
const height = parsePixelSize($img.attr("height")) ?? parsePixelSize($img.attr("style")?.match(/height:\s*(\d+px)/i)?.[1]);
|
|
17
|
+
if (width !== null && height !== null && width <= 2 && height <= 2)
|
|
18
|
+
return true;
|
|
19
|
+
return false;
|
|
20
|
+
}
|
|
21
|
+
const GENERIC_ALT = new Set(["image", "photo", "picture", "img", "logo", "icon"]);
|
|
22
|
+
const IMAGE_EXTENSION_RE = /\.(?:jpe?g|png|gif|webp|avif|svg)$/i;
|
|
23
|
+
const IMAGE_FILENAME_IN_TEXT_RE = /([^\/?#\s&,=]+\.(?:jpe?g|png|gif|webp|avif|svg))/gi;
|
|
24
|
+
function tryDecodeURIComponent(value) {
|
|
25
|
+
try {
|
|
26
|
+
return decodeURIComponent(value);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return value;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
function looksUrlEncoded(value) {
|
|
33
|
+
return /%[0-9a-f]{2}/i.test(value);
|
|
34
|
+
}
|
|
35
|
+
function extractProxyImageUrl(value) {
|
|
36
|
+
if (!/[?&]url=/i.test(value))
|
|
37
|
+
return null;
|
|
38
|
+
const match = value.match(/[?&]url=([^&]+)/i);
|
|
39
|
+
if (!match?.[1])
|
|
40
|
+
return null;
|
|
41
|
+
return tryDecodeURIComponent(match[1]);
|
|
42
|
+
}
|
|
43
|
+
function isValidImageFilename(filename) {
|
|
44
|
+
if (!filename || filename.includes("&") || filename.includes("="))
|
|
45
|
+
return false;
|
|
46
|
+
if (filename.includes("%") || filename.includes("://"))
|
|
47
|
+
return false;
|
|
48
|
+
if (!IMAGE_EXTENSION_RE.test(filename))
|
|
49
|
+
return false;
|
|
50
|
+
return !/^\d+$/.test(filename.replace(IMAGE_EXTENSION_RE, ""));
|
|
51
|
+
}
|
|
52
|
+
function filenameFromUrl(url) {
|
|
53
|
+
let working = url.trim();
|
|
54
|
+
if (!working || working.startsWith("data:") || working.startsWith("blob:"))
|
|
55
|
+
return null;
|
|
56
|
+
if (looksUrlEncoded(working)) {
|
|
57
|
+
working = tryDecodeURIComponent(working);
|
|
58
|
+
}
|
|
59
|
+
const proxyUrl = extractProxyImageUrl(working);
|
|
60
|
+
if (proxyUrl) {
|
|
61
|
+
working = proxyUrl;
|
|
62
|
+
}
|
|
63
|
+
let pathname;
|
|
64
|
+
try {
|
|
65
|
+
pathname = working.includes("://") ? new URL(working).pathname : working.split(/[?#]/)[0] ?? "";
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
pathname = working.split(/[?#]/)[0] ?? "";
|
|
69
|
+
}
|
|
70
|
+
const filename = pathname.split("/").filter(Boolean).pop() ?? "";
|
|
71
|
+
return isValidImageFilename(filename) ? filename : null;
|
|
72
|
+
}
|
|
73
|
+
function filenamesInText(text) {
|
|
74
|
+
const seen = new Set();
|
|
75
|
+
const filenames = [];
|
|
76
|
+
const decoded = looksUrlEncoded(text) ? tryDecodeURIComponent(text) : text;
|
|
77
|
+
for (const source of [text, decoded]) {
|
|
78
|
+
for (const match of source.matchAll(IMAGE_FILENAME_IN_TEXT_RE)) {
|
|
79
|
+
const filename = match[1];
|
|
80
|
+
if (!isValidImageFilename(filename) || seen.has(filename))
|
|
81
|
+
continue;
|
|
82
|
+
seen.add(filename);
|
|
83
|
+
filenames.push(filename);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return filenames;
|
|
87
|
+
}
|
|
88
|
+
function srcsetCandidates(srcset) {
|
|
89
|
+
return srcset
|
|
90
|
+
.split(/,\s*(?=(?:https?:)?\/)/i)
|
|
91
|
+
.map((entry) => entry.trim().split(/\s+/)[0] ?? "")
|
|
92
|
+
.filter(Boolean);
|
|
93
|
+
}
|
|
94
|
+
function imageReferenceFromValue(value, pageUrl) {
|
|
95
|
+
const trimmed = value.trim();
|
|
96
|
+
if (!trimmed)
|
|
97
|
+
return null;
|
|
98
|
+
const candidates = /\d+w|\d+x/i.test(trimmed) ? srcsetCandidates(trimmed) : [trimmed];
|
|
99
|
+
for (const candidate of candidates) {
|
|
100
|
+
const decoded = tryDecodeURIComponent(candidate);
|
|
101
|
+
const proxyUrl = extractProxyImageUrl(candidate) ?? extractProxyImageUrl(decoded);
|
|
102
|
+
const lookups = [proxyUrl, decoded, candidate].filter((entry) => Boolean(entry));
|
|
103
|
+
for (const lookup of lookups) {
|
|
104
|
+
const filename = filenameFromUrl(lookup) ?? filenamesInText(lookup)[0];
|
|
105
|
+
if (!filename)
|
|
106
|
+
continue;
|
|
107
|
+
const url = toAbsoluteImageUrl(lookup, pageUrl);
|
|
108
|
+
if (!url)
|
|
109
|
+
continue;
|
|
110
|
+
return { filename, url };
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
function toAbsoluteImageUrl(value, pageUrl) {
|
|
116
|
+
let working = value.trim();
|
|
117
|
+
if (!working || working.startsWith("data:") || working.startsWith("blob:"))
|
|
118
|
+
return null;
|
|
119
|
+
if (looksUrlEncoded(working)) {
|
|
120
|
+
working = tryDecodeURIComponent(working);
|
|
121
|
+
}
|
|
122
|
+
const proxyUrl = extractProxyImageUrl(working);
|
|
123
|
+
if (proxyUrl) {
|
|
124
|
+
working = proxyUrl;
|
|
125
|
+
}
|
|
126
|
+
try {
|
|
127
|
+
return working.includes("://") ? new URL(working).href : new URL(working, pageUrl).href;
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
function collectImageReference($, $img, pageUrl) {
|
|
134
|
+
const candidates = [];
|
|
135
|
+
const src = ($img.attr("src") ?? "").trim();
|
|
136
|
+
if (src)
|
|
137
|
+
candidates.push(src);
|
|
138
|
+
const srcset = ($img.attr("srcset") ?? "").trim();
|
|
139
|
+
if (srcset)
|
|
140
|
+
candidates.push(srcset);
|
|
141
|
+
const $picture = $img.closest("picture");
|
|
142
|
+
if ($picture.length > 0) {
|
|
143
|
+
$picture.find("source").each((_, source) => {
|
|
144
|
+
const $source = $(source);
|
|
145
|
+
const sourceSrc = ($source.attr("src") ?? "").trim();
|
|
146
|
+
if (sourceSrc)
|
|
147
|
+
candidates.push(sourceSrc);
|
|
148
|
+
const sourceSrcset = ($source.attr("srcset") ?? "").trim();
|
|
149
|
+
if (sourceSrcset)
|
|
150
|
+
candidates.push(sourceSrcset);
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
for (const candidate of candidates) {
|
|
154
|
+
const reference = imageReferenceFromValue(candidate, pageUrl);
|
|
155
|
+
if (reference)
|
|
156
|
+
return reference;
|
|
157
|
+
}
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
function imageIssueMessage(base, reference) {
|
|
161
|
+
if (!reference)
|
|
162
|
+
return base;
|
|
163
|
+
return `${base} ([${reference.filename}](${reference.url}))`;
|
|
164
|
+
}
|
|
165
|
+
function isGenericAlt(alt, src) {
|
|
166
|
+
const normalized = alt.trim().toLowerCase();
|
|
167
|
+
if (!normalized)
|
|
168
|
+
return false;
|
|
169
|
+
if (GENERIC_ALT.has(normalized))
|
|
170
|
+
return true;
|
|
171
|
+
const filename = src.split("/").pop()?.split("?")[0] ?? "";
|
|
172
|
+
const basename = filename.replace(/\.[^.]+$/, "").toLowerCase();
|
|
173
|
+
return basename.length > 0 && normalized === basename;
|
|
174
|
+
}
|
|
175
|
+
export function extractImageIssues($, pageUrl) {
|
|
176
|
+
const issues = [];
|
|
177
|
+
$("img").each((_, element) => {
|
|
178
|
+
const $img = $(element);
|
|
179
|
+
if (isDecorativeImage($img))
|
|
180
|
+
return;
|
|
181
|
+
const src = ($img.attr("src") ?? "").trim();
|
|
182
|
+
const reference = collectImageReference($, $img, pageUrl);
|
|
183
|
+
const hasAlt = $img.get(0)?.attribs && Object.prototype.hasOwnProperty.call($img.get(0).attribs, "alt");
|
|
184
|
+
const alt = ($img.attr("alt") ?? "").trim();
|
|
185
|
+
if (!hasAlt) {
|
|
186
|
+
issues.push(crawlIssue("missing-alt", "error", imageIssueMessage("Image is missing an alt attribute.", reference)));
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
if (alt.length === 0) {
|
|
190
|
+
issues.push(crawlIssue("empty-alt", "warning", imageIssueMessage("Image has an empty alt attribute.", reference)));
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
if (isGenericAlt(alt, src)) {
|
|
194
|
+
issues.push(crawlIssue("generic-alt", "warning", imageIssueMessage("Image uses generic alt text.", reference)));
|
|
195
|
+
}
|
|
196
|
+
});
|
|
197
|
+
return issues;
|
|
198
|
+
}
|
|
199
|
+
function headingLevel(tagName) {
|
|
200
|
+
const match = tagName.match(/^h([1-6])$/i);
|
|
201
|
+
return match ? Number(match[1]) : null;
|
|
202
|
+
}
|
|
203
|
+
function hasAssociatedLabel($, $input) {
|
|
204
|
+
const id = ($input.attr("id") ?? "").trim();
|
|
205
|
+
if (id && $(`label[for="${id}"]`).length > 0)
|
|
206
|
+
return true;
|
|
207
|
+
if (($input.attr("aria-label") ?? "").trim())
|
|
208
|
+
return true;
|
|
209
|
+
if (($input.attr("aria-labelledby") ?? "").trim())
|
|
210
|
+
return true;
|
|
211
|
+
if ($input.parents("label").length > 0)
|
|
212
|
+
return true;
|
|
213
|
+
if (($input.attr("title") ?? "").trim())
|
|
214
|
+
return true;
|
|
215
|
+
return false;
|
|
216
|
+
}
|
|
217
|
+
export function extractA11yStaticIssues($) {
|
|
218
|
+
const issues = [];
|
|
219
|
+
const headings = $("h1, h2, h3, h4, h5, h6")
|
|
220
|
+
.toArray()
|
|
221
|
+
.map((element) => headingLevel("name" in element ? String(element.name) : ""))
|
|
222
|
+
.filter((level) => level !== null);
|
|
223
|
+
if ($("h1").length > 1) {
|
|
224
|
+
issues.push(crawlIssue("multiple-h1", "warning", "Page has multiple h1 elements."));
|
|
225
|
+
}
|
|
226
|
+
let previousLevel = null;
|
|
227
|
+
for (const level of headings) {
|
|
228
|
+
if (previousLevel !== null && level > previousLevel + 1) {
|
|
229
|
+
issues.push(crawlIssue("heading-skip", "warning", "Heading levels skip a rank."));
|
|
230
|
+
break;
|
|
231
|
+
}
|
|
232
|
+
previousLevel = level;
|
|
233
|
+
}
|
|
234
|
+
const lang = ($("html").first().attr("lang") ?? "").trim();
|
|
235
|
+
if (!lang) {
|
|
236
|
+
issues.push(crawlIssue("missing-lang", "error", "html element is missing a lang attribute."));
|
|
237
|
+
}
|
|
238
|
+
$("input, select, textarea")
|
|
239
|
+
.not('[type="hidden"], [type="submit"], [type="button"], [type="reset"], [type="image"]')
|
|
240
|
+
.each((_, element) => {
|
|
241
|
+
const $input = $(element);
|
|
242
|
+
if (!hasAssociatedLabel($, $input)) {
|
|
243
|
+
issues.push(crawlIssue("unlabeled-input", "error", "Form control is missing an associated label."));
|
|
244
|
+
}
|
|
245
|
+
});
|
|
246
|
+
$("[tabindex]")
|
|
247
|
+
.toArray()
|
|
248
|
+
.forEach((element) => {
|
|
249
|
+
const value = Number.parseInt($(element).attr("tabindex") ?? "", 10);
|
|
250
|
+
if (Number.isFinite(value) && value > 0) {
|
|
251
|
+
issues.push(crawlIssue("positive-tabindex", "warning", "Element uses a positive tabindex."));
|
|
252
|
+
}
|
|
253
|
+
});
|
|
254
|
+
return issues;
|
|
255
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tracking.d.ts","sourceRoot":"","sources":["../../src/crawler/tracking.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,KAAK,cAAc,GAAG;IACpB,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,SAAS,EAAE,YAAY,EAAE,CAAC;CAC3B,CAAC;AAMF,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,CAkD5D"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
function uniq(values) {
|
|
2
|
+
return [...new Set(values)];
|
|
3
|
+
}
|
|
4
|
+
export function extractTracking(html) {
|
|
5
|
+
const fullText = html;
|
|
6
|
+
const tagManager = [];
|
|
7
|
+
const analytics = [];
|
|
8
|
+
const gtmMatches = fullText.match(/GTM-[A-Z0-9]+/gi) ?? [];
|
|
9
|
+
const ga4Matches = fullText.match(/\bG-[A-Z0-9]{6,}\b/gi) ?? [];
|
|
10
|
+
const uaMatches = fullText.match(/\bUA-\d+-\d+\b/gi) ?? [];
|
|
11
|
+
const matomoSiteIdMatches = fullText.match(/(?:_paq\.push\(\[['"]setSiteId['"],\s*['"]?(\d+)['"]?\]\)|siteId['"]?\s*[:=]\s*['"]?(\d+)['"]?)/gi) ?? [];
|
|
12
|
+
const pixelMatches = fullText.match(/\bfbq\(['"]init['"],\s*['"](\d{6,})['"]\)/gi) ?? [];
|
|
13
|
+
const plausibleMatch = /plausible\.io\/js\/script\.js/i.test(fullText);
|
|
14
|
+
for (const id of uniq(gtmMatches)) {
|
|
15
|
+
tagManager.push({ provider: "google-tag-manager", id });
|
|
16
|
+
}
|
|
17
|
+
for (const id of uniq(ga4Matches)) {
|
|
18
|
+
analytics.push({ provider: "google-analytics-ga4", id });
|
|
19
|
+
}
|
|
20
|
+
for (const id of uniq(uaMatches)) {
|
|
21
|
+
analytics.push({ provider: "google-analytics-ua", id });
|
|
22
|
+
}
|
|
23
|
+
for (const raw of uniq(matomoSiteIdMatches)) {
|
|
24
|
+
const idMatch = raw.match(/(\d+)/);
|
|
25
|
+
if (idMatch)
|
|
26
|
+
analytics.push({ provider: "matomo", id: idMatch[1] });
|
|
27
|
+
}
|
|
28
|
+
for (const raw of uniq(pixelMatches)) {
|
|
29
|
+
const idMatch = raw.match(/(\d{6,})/);
|
|
30
|
+
if (idMatch)
|
|
31
|
+
analytics.push({ provider: "facebook-pixel", id: idMatch[1] });
|
|
32
|
+
}
|
|
33
|
+
if (plausibleMatch) {
|
|
34
|
+
analytics.push({ provider: "plausible", id: "" });
|
|
35
|
+
}
|
|
36
|
+
const dedupeByProviderAndId = (items) => {
|
|
37
|
+
const seen = new Set();
|
|
38
|
+
return items.filter((item) => {
|
|
39
|
+
const key = `${item.provider}:${item.id}`;
|
|
40
|
+
if (seen.has(key))
|
|
41
|
+
return false;
|
|
42
|
+
seen.add(key);
|
|
43
|
+
return true;
|
|
44
|
+
});
|
|
45
|
+
};
|
|
46
|
+
return {
|
|
47
|
+
tagManager: dedupeByProviderAndId(tagManager),
|
|
48
|
+
analytics: dedupeByProviderAndId(analytics),
|
|
49
|
+
};
|
|
50
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare function sleep(ms: number): Promise<void>;
|
|
2
|
+
/**
|
|
3
|
+
* True for Vercel or Netlify platform default/preview hostnames.
|
|
4
|
+
* Custom production domains mapped onto those platforms return false.
|
|
5
|
+
*/
|
|
6
|
+
export declare function isPlatformPreviewHostname(hostname: string): boolean;
|
|
7
|
+
export declare function isHttpUrl(urlString: string): boolean;
|
|
8
|
+
export declare function normalizeUrl(urlString: string): string;
|
|
9
|
+
export declare function resolveUrlMaybe(value: string, baseUrl: string): string;
|
|
10
|
+
export declare function parseExcludePatterns(rawExcludes: string[]): string[];
|
|
11
|
+
export declare function matchesExcludePattern(urlString: string, patterns: string[]): boolean;
|
|
12
|
+
//# sourceMappingURL=urlUtils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"urlUtils.d.ts","sourceRoot":"","sources":["../../src/crawler/urlUtils.ts"],"names":[],"mappings":"AAAA,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/C;AAKD;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAGnE;AAED,wBAAgB,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAOpD;AAED,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAUtD;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAOtE;AAED,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAGpE;AAED,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAKpF"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export function sleep(ms) {
|
|
2
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3
|
+
}
|
|
4
|
+
/** Vercel / Netlify platform hostnames (*.vercel.app, *.netlify.app) — not custom domains. */
|
|
5
|
+
const PLATFORM_PREVIEW_SUFFIXES = [".vercel.app", ".netlify.app"];
|
|
6
|
+
/**
|
|
7
|
+
* True for Vercel or Netlify platform default/preview hostnames.
|
|
8
|
+
* Custom production domains mapped onto those platforms return false.
|
|
9
|
+
*/
|
|
10
|
+
export function isPlatformPreviewHostname(hostname) {
|
|
11
|
+
const lower = hostname.toLowerCase();
|
|
12
|
+
return PLATFORM_PREVIEW_SUFFIXES.some((suffix) => lower === suffix.slice(1) || lower.endsWith(suffix));
|
|
13
|
+
}
|
|
14
|
+
export function isHttpUrl(urlString) {
|
|
15
|
+
try {
|
|
16
|
+
const url = new URL(urlString);
|
|
17
|
+
return url.protocol === "http:" || url.protocol === "https:";
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export function normalizeUrl(urlString) {
|
|
24
|
+
const url = new URL(urlString);
|
|
25
|
+
url.hash = "";
|
|
26
|
+
url.search = "";
|
|
27
|
+
if (url.pathname === "") {
|
|
28
|
+
url.pathname = "/";
|
|
29
|
+
}
|
|
30
|
+
return url.toString();
|
|
31
|
+
}
|
|
32
|
+
export function resolveUrlMaybe(value, baseUrl) {
|
|
33
|
+
if (!value)
|
|
34
|
+
return "";
|
|
35
|
+
try {
|
|
36
|
+
return normalizeUrl(new URL(value, baseUrl).toString());
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return value;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export function parseExcludePatterns(rawExcludes) {
|
|
43
|
+
if (rawExcludes.length === 0)
|
|
44
|
+
return [];
|
|
45
|
+
return rawExcludes.flatMap((item) => item.split(",")).map((item) => item.trim()).filter(Boolean);
|
|
46
|
+
}
|
|
47
|
+
export function matchesExcludePattern(urlString, patterns) {
|
|
48
|
+
if (patterns.length === 0)
|
|
49
|
+
return false;
|
|
50
|
+
const normalized = normalizeUrl(urlString);
|
|
51
|
+
const pathname = new URL(normalized).pathname;
|
|
52
|
+
return patterns.some((pattern) => normalized.includes(pattern) || pathname.includes(pattern));
|
|
53
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export * from "./checks/index.js";
|
|
2
|
+
export { buildMarkdown, parseReport, reportSchema, buildReportAnalysis } from "./analyze/report.js";
|
|
3
|
+
export { buildStructuredReport, renderReportMarkdown, FLIGHTCHECK_REPORT_VERSION, } from "./analyze/structuredReport.js";
|
|
4
|
+
export type { ReportRecord, Tier1ReportData, BuildMarkdownOptions, ReportAnalysis } from "./analyze/report.js";
|
|
5
|
+
export type { FlightcheckReport, FlightcheckReportAnalysis, ReportFinding, FindingSeverity, FindingCategory, FindingScope, } from "./analyze/structuredReport.js";
|
|
6
|
+
/** @deprecated Prefer structured report JSON; kept for transitional imports. */
|
|
7
|
+
export { buildFixManifest, FIX_MANIFEST_VERSION } from "./analyze/fixManifest.js";
|
|
8
|
+
/** @deprecated Prefer structured report JSON; kept for transitional imports. */
|
|
9
|
+
export type { FixManifest, WorkPackage, ManifestIssue, LocationDiscovery, FixSurface, } from "./analyze/fixManifest.js";
|
|
10
|
+
export { runCrawler } from "./crawler/runCrawler.js";
|
|
11
|
+
export type { VerificationRecord } from "./crawler/schema.js";
|
|
12
|
+
export { CHECK_GROUPS, resolveActiveGroups } from "./crawler/checkGroups.js";
|
|
13
|
+
export type { CheckGroup } from "./crawler/checkGroups.js";
|
|
14
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AACpG,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,0BAA0B,GAC3B,MAAM,+BAA+B,CAAC;AACvC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC/G,YAAY,EACV,iBAAiB,EACjB,yBAAyB,EACzB,aAAa,EACb,eAAe,EACf,eAAe,EACf,YAAY,GACb,MAAM,+BAA+B,CAAC;AACvC,gFAAgF;AAChF,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAClF,gFAAgF;AAChF,YAAY,EACV,WAAW,EACX,WAAW,EACX,aAAa,EACb,iBAAiB,EACjB,UAAU,GACX,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,YAAY,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC7E,YAAY,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export * from "./checks/index.js";
|
|
2
|
+
export { buildMarkdown, parseReport, reportSchema, buildReportAnalysis } from "./analyze/report.js";
|
|
3
|
+
export { buildStructuredReport, renderReportMarkdown, FLIGHTCHECK_REPORT_VERSION, } from "./analyze/structuredReport.js";
|
|
4
|
+
/** @deprecated Prefer structured report JSON; kept for transitional imports. */
|
|
5
|
+
export { buildFixManifest, FIX_MANIFEST_VERSION } from "./analyze/fixManifest.js";
|
|
6
|
+
export { runCrawler } from "./crawler/runCrawler.js";
|
|
7
|
+
export { CHECK_GROUPS, resolveActiveGroups } from "./crawler/checkGroups.js";
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Agent fixes (deprecated)
|
|
2
|
+
|
|
3
|
+
**Status:** deprecated. Analyze no longer writes `*-fix-manifest.json` or
|
|
4
|
+
`*-agent-jobs/`.
|
|
5
|
+
|
|
6
|
+
Use the structured report instead:
|
|
7
|
+
|
|
8
|
+
- [Analyze reports](./analyze.md) — `*-report.json` + `*-report.md`
|
|
9
|
+
- [Design](./superpowers/specs/2026-10-27-structured-report-design.md)
|
|
10
|
+
|
|
11
|
+
The previous agent-jobs / fix-manifest workflow (work packages, `prompt.md` per
|
|
12
|
+
package, `flightcheck-orchestrate`) remains in the repo for transitional imports
|
|
13
|
+
but is not part of the default output. Prefer building any new agent automation
|
|
14
|
+
on `findings[]` in `*-report.json`.
|
|
15
|
+
|
|
16
|
+
Stack detection still feeds the report `analysis.stack` block. See
|
|
17
|
+
[stack.md](./stack.md).
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Agent readiness
|
|
2
|
+
|
|
3
|
+
Flightcheck can audit how ready a site is for AI agents, aligned with the practical
|
|
4
|
+
checks from [isitagentready.com](https://isitagentready.com/).
|
|
5
|
+
|
|
6
|
+
**Status:** implemented as standalone check `agent-ready` (default pipeline).
|
|
7
|
+
|
|
8
|
+
## What it checks
|
|
9
|
+
|
|
10
|
+
| Area | Signal | Severity when missing |
|
|
11
|
+
| --- | --- | --- |
|
|
12
|
+
| Bot access | `Content-Signal` in robots.txt | warning |
|
|
13
|
+
| Bot access | Explicit AI bot `User-agent` rules (GPTBot, ClaudeBot, …) | warning |
|
|
14
|
+
| Content | `/llms.txt` plain-text overview with `#` title | warning / review if incomplete |
|
|
15
|
+
| Content | Markdown negotiation (`Accept: text/markdown` → `text/markdown`) | warning |
|
|
16
|
+
| Discoverability | Homepage `Link` headers (`api-catalog`, `service-doc`, …) | review |
|
|
17
|
+
| Protocols | Well-known MCP / Agent Skills / API Catalog / ARD / A2A | review (informational for content sites) |
|
|
18
|
+
|
|
19
|
+
robots.txt existence and sitemaps remain in the existing `robots-txt` and `sitemap` checks.
|
|
20
|
+
|
|
21
|
+
## Architecture
|
|
22
|
+
|
|
23
|
+
- Module: `src/checks/agent-ready.ts`
|
|
24
|
+
- Pure analyzer: `analyzeAgentReady()` (unit-tested)
|
|
25
|
+
- Network runner: `runAgentReadyCheck()` — parallel fetches of robots, llms.txt, homepage
|
|
26
|
+
(HTML + markdown Accept), and well-known endpoints
|
|
27
|
+
- Wave 1 (parallel with crawl / DNS / headers / robots)
|
|
28
|
+
- Report section: `## Agent readiness` via `buildAgentReadySection()` in `reportTier1.ts`
|
|
29
|
+
- Fix package: `agent-ready` → `prompts/fix/agent-ready.md`
|
|
30
|
+
|
|
31
|
+
## CLI
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pnpm run flightcheck -- https://example.com
|
|
35
|
+
pnpm run flightcheck -- https://example.com --checks agent-ready,analyze
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Out of scope (for now)
|
|
39
|
+
|
|
40
|
+
- Commerce protocols (x402, UCP, ACP, MPP)
|
|
41
|
+
- Web Bot Auth
|
|
42
|
+
- DNS-AID
|
|
43
|
+
- WebMCP (needs a browser)
|
|
44
|
+
- Full OAuth / Auth.md discovery as hard failures
|
package/docs/analyze.md
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# Flightcheck Analyze Script
|
|
2
|
+
|
|
3
|
+
This script reads a crawl JSON file and produces a structured report:
|
|
4
|
+
|
|
5
|
+
- `*-report.json` — machine-readable analysis + findings
|
|
6
|
+
- `*-report.md` — plain-language Markdown rendered from that JSON
|
|
7
|
+
|
|
8
|
+
The main CLI (`flightcheck`) generates both by default after every crawl. Use this
|
|
9
|
+
standalone script only when you want to re-analyze an existing JSON file without re-crawling.
|
|
10
|
+
|
|
11
|
+
## Purpose
|
|
12
|
+
|
|
13
|
+
- Give tools a stable JSON schema (`analysis` + flat `findings[]`).
|
|
14
|
+
- Give people a readable Markdown report generated from the same model.
|
|
15
|
+
- Keep severity, category, and scope on every finding.
|
|
16
|
+
|
|
17
|
+
## Usage
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pnpm run analyze -- ./flightcheck/20260618.1413-example.com-raw.json
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Include only critical/error findings in the Markdown view:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
pnpm run analyze -- ./flightcheck/20260618.1413-example.com-raw.json --errors-only
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Without `--output`, paths are derived from the input: `-raw.json` becomes
|
|
30
|
+
`-report.md` and `-report.json`.
|
|
31
|
+
|
|
32
|
+
## Output structure
|
|
33
|
+
|
|
34
|
+
### `*-report.json`
|
|
35
|
+
|
|
36
|
+
```json
|
|
37
|
+
{
|
|
38
|
+
"version": 1,
|
|
39
|
+
"generatedAt": "2026-10-27T12:00:00.000Z",
|
|
40
|
+
"source": "./flightcheck/example-raw.json",
|
|
41
|
+
"redirectOnly": false,
|
|
42
|
+
"analysis": {
|
|
43
|
+
"pagesAnalyzed": 24,
|
|
44
|
+
"pagesWithIssues": 13,
|
|
45
|
+
"findingCounts": {
|
|
46
|
+
"critical": 1,
|
|
47
|
+
"error": 12,
|
|
48
|
+
"warning": 8,
|
|
49
|
+
"review": 3
|
|
50
|
+
},
|
|
51
|
+
"stack": [
|
|
52
|
+
{
|
|
53
|
+
"name": "Next.js",
|
|
54
|
+
"categories": ["JavaScript frameworks"],
|
|
55
|
+
"confidence": "high",
|
|
56
|
+
"evidence": ["Next.js runtime markers in HTML"],
|
|
57
|
+
"pageCount": 24
|
|
58
|
+
}
|
|
59
|
+
],
|
|
60
|
+
"templates": ["work", "blog"],
|
|
61
|
+
"reviewPages": []
|
|
62
|
+
},
|
|
63
|
+
"findings": [
|
|
64
|
+
{
|
|
65
|
+
"id": "missing-hsts",
|
|
66
|
+
"severity": "error",
|
|
67
|
+
"category": "infra",
|
|
68
|
+
"scope": "origin",
|
|
69
|
+
"title": "Missing Hsts",
|
|
70
|
+
"message": "Strict-Transport-Security header is missing on HTTPS."
|
|
71
|
+
}
|
|
72
|
+
]
|
|
73
|
+
}
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Findings are a flat list (no severity buckets), sorted critical → review. Each finding
|
|
77
|
+
includes metadata so consumers can filter or regroup.
|
|
78
|
+
|
|
79
|
+
| Field | Meaning |
|
|
80
|
+
| --- | --- |
|
|
81
|
+
| `severity` | `critical` \| `error` \| `warning` \| `review` |
|
|
82
|
+
| `category` | `infra` \| `seo` \| `a11y` \| `best-practices` \| `security` \| `performance` |
|
|
83
|
+
| `scope` | `site-wide` \| `template` \| `page` \| `origin` |
|
|
84
|
+
| `title` | Short human title |
|
|
85
|
+
| `message` | What to do / what was found |
|
|
86
|
+
| `template` | Present when `scope` is `template` |
|
|
87
|
+
| `urls` / `count` | Affected pages when relevant |
|
|
88
|
+
|
|
89
|
+
### `*-report.md`
|
|
90
|
+
|
|
91
|
+
Generated from the JSON. Structure:
|
|
92
|
+
|
|
93
|
+
1. **At a glance** — page counts and finding totals
|
|
94
|
+
2. **Stack / DNS / redirect** — factual context when available
|
|
95
|
+
3. **Findings** — grouped by severity for reading order, with plain-language titles
|
|
96
|
+
|
|
97
|
+
## Notes
|
|
98
|
+
|
|
99
|
+
- The script validates crawl input against the expected flightcheck report schema.
|
|
100
|
+
- On schema errors, the script stops with a clear error message.
|
|
101
|
+
- Fix-manifest and agent-jobs output are no longer generated. Prefer `*-report.json`
|
|
102
|
+
for machine consumers. See the design note in
|
|
103
|
+
[superpowers/specs/2026-10-27-structured-report-design.md](./superpowers/specs/2026-10-27-structured-report-design.md).
|