pagegraph 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -4
- package/dist/audit.d.ts +8 -3
- package/dist/audit.js +1931 -1
- package/dist/audit.js.map +1 -0
- package/dist/{graph-BlLoEOw2.d.ts → checks-BfsQtKga.d.ts} +43 -2
- package/dist/cli.js +41973 -35
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +7 -1
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts +131 -14
- package/dist/index.js +618 -2
- package/dist/index.js.map +1 -1
- package/dist/links-sGbLkl-7.js +184 -0
- package/dist/links-sGbLkl-7.js.map +1 -0
- package/package.json +13 -8
- package/dist/audit-B96V1x3q.js +0 -1929
- package/dist/audit-B96V1x3q.js.map +0 -1
- package/dist/inspect-html-CHuoiO2s.js +0 -452
- package/dist/inspect-html-CHuoiO2s.js.map +0 -1
- package/dist/main-GFEobQTH.js +0 -750
- package/dist/main-GFEobQTH.js.map +0 -1
package/dist/audit.js
CHANGED
|
@@ -1,2 +1,1932 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { r as extractAnchors } from "./links-sGbLkl-7.js";
|
|
2
|
+
import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import * as Context from "effect/Context";
|
|
5
|
+
import * as Data$1 from "effect/Data";
|
|
6
|
+
import * as Effect from "effect/Effect";
|
|
7
|
+
import * as Layer from "effect/Layer";
|
|
8
|
+
import { Data, Schema } from "effect";
|
|
9
|
+
import { createHash } from "node:crypto";
|
|
10
|
+
import { createServer, request } from "node:http";
|
|
11
|
+
import { request as request$1 } from "node:https";
|
|
12
|
+
import { BlockList, connect, isIP } from "node:net";
|
|
13
|
+
import { checkServerIdentity } from "node:tls";
|
|
14
|
+
import { lookup } from "node:dns/promises";
|
|
15
|
+
import { spawn } from "node:child_process";
|
|
16
|
+
import { createRequire } from "node:module";
|
|
17
|
+
import { tmpdir } from "node:os";
|
|
18
|
+
import * as Option from "effect/Option";
|
|
19
|
+
import * as Schema$1 from "effect/Schema";
|
|
20
|
+
//#region src/audit/rules.ts
|
|
21
|
+
const record = (value) => typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
|
|
22
|
+
const stringAt = (value, ...keys) => {
|
|
23
|
+
let current = value;
|
|
24
|
+
for (const key of keys) current = record(current)?.[key];
|
|
25
|
+
return typeof current === "string" ? current : void 0;
|
|
26
|
+
};
|
|
27
|
+
const firstString = (value, paths) => {
|
|
28
|
+
for (const path of paths) {
|
|
29
|
+
const found = stringAt(value, ...path);
|
|
30
|
+
if (found !== void 0) return found;
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
const pageEvidence = (value) => {
|
|
34
|
+
if (!Array.isArray(value)) return value;
|
|
35
|
+
return value.find((entry) => stringAt(entry, "kind") === "page-html") ?? value.find((entry) => stringAt(entry, "kind") === "page-head") ?? value[0];
|
|
36
|
+
};
|
|
37
|
+
const finding = (input, rule, message, fix, observed, severity = "structural") => ({
|
|
38
|
+
scanner: "rules",
|
|
39
|
+
target: input.target.url,
|
|
40
|
+
severity,
|
|
41
|
+
rule,
|
|
42
|
+
message,
|
|
43
|
+
fix,
|
|
44
|
+
...observed === void 0 ? {} : { observed }
|
|
45
|
+
});
|
|
46
|
+
/** Generic page-quality rules. They accept the normalized evidence shape used
|
|
47
|
+
* by HTTP and hosted scanners, while remaining pure and dependency-free. */
|
|
48
|
+
const pageRules = [
|
|
49
|
+
{
|
|
50
|
+
id: "http-status",
|
|
51
|
+
evaluate: (observation, input) => {
|
|
52
|
+
const status = record(pageEvidence(observation.evidence))?.status;
|
|
53
|
+
return typeof status === "number" && status >= 400 ? [finding(input, "http-status", `Fetch returned HTTP ${status}.`, "Return a successful HTTP status.", status)] : [];
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
id: "title-length",
|
|
58
|
+
evaluate: (observation, input) => {
|
|
59
|
+
const evidence = pageEvidence(observation.evidence);
|
|
60
|
+
const title = firstString(evidence, [
|
|
61
|
+
["title"],
|
|
62
|
+
["document", "title"],
|
|
63
|
+
["head", "title"]
|
|
64
|
+
]);
|
|
65
|
+
const policy = input.options.policy;
|
|
66
|
+
const min = policy?.titleMinLength ?? 10;
|
|
67
|
+
const max = policy?.titleMaxLength ?? 70;
|
|
68
|
+
if (title === void 0 || title.trim() === "") return [finding(input, "missing-title", "Page is missing a usable title.", "Add a descriptive <title>.")];
|
|
69
|
+
if (title.length < min || title.length > max) return [finding(input, "title-length", `Title length ${title.length} is outside ${min}-${max} characters.`, `Keep the title between ${min} and ${max} characters.`, title, "editorial")];
|
|
70
|
+
return [];
|
|
71
|
+
}
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
id: "description-length",
|
|
75
|
+
evaluate: (observation, input) => {
|
|
76
|
+
const evidence = pageEvidence(observation.evidence);
|
|
77
|
+
const description = firstString(evidence, [
|
|
78
|
+
["description"],
|
|
79
|
+
["metaDescription"],
|
|
80
|
+
["document", "description"],
|
|
81
|
+
["head", "description"]
|
|
82
|
+
]);
|
|
83
|
+
const policy = input.options.policy;
|
|
84
|
+
const min = policy?.descriptionMinLength ?? 50;
|
|
85
|
+
const max = policy?.descriptionMaxLength ?? 160;
|
|
86
|
+
if (description === void 0 || description.trim() === "") return [finding(input, "missing-description", "Page is missing a meta description.", "Add a concise description that summarizes the page.")];
|
|
87
|
+
if (description.length < min || description.length > max) return [finding(input, "description-length", `Meta description length ${description.length} is outside ${min}-${max} characters.`, `Keep the meta description between ${min} and ${max} characters.`, description, "editorial")];
|
|
88
|
+
return [];
|
|
89
|
+
}
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
id: "canonical",
|
|
93
|
+
evaluate: (observation, input) => {
|
|
94
|
+
if (input.options.policy?.requireCanonical === false) return [];
|
|
95
|
+
const canonical = firstString(pageEvidence(observation.evidence), [
|
|
96
|
+
["canonical"],
|
|
97
|
+
["canonicalUrl"],
|
|
98
|
+
["document", "canonical"],
|
|
99
|
+
["document", "canonicalUrl"],
|
|
100
|
+
["head", "canonical"]
|
|
101
|
+
]);
|
|
102
|
+
if (canonical === void 0) return [finding(input, "missing-canonical", "Page is missing a canonical URL.", "Add a canonical link that identifies the preferred URL.")];
|
|
103
|
+
try {
|
|
104
|
+
const observed = new URL(canonical, input.target.url);
|
|
105
|
+
const expected = new URL(firstString(pageEvidence(observation.evidence), [["finalUrl"]]) ?? input.target.url);
|
|
106
|
+
observed.hash = "";
|
|
107
|
+
expected.hash = "";
|
|
108
|
+
return observed.href === expected.href ? [] : [finding(input, "canonical-mismatch", `Canonical points to ${observed.href} instead of the audited URL.`, "Use a self-referencing canonical unless this page is intentionally consolidated.", observed.href)];
|
|
109
|
+
} catch {
|
|
110
|
+
return [finding(input, "invalid-canonical", "Page canonical is not a valid URL.", "Use an absolute or resolvable canonical URL.", canonical)];
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
];
|
|
115
|
+
const evaluateRules = (rules, observation, input) => rules.flatMap((rule) => rule.evaluate(observation, input));
|
|
116
|
+
/** Default policy for a web-page audit. */
|
|
117
|
+
const defaultAuditOptions = {
|
|
118
|
+
concurrency: 4,
|
|
119
|
+
policy: {
|
|
120
|
+
titleMinLength: 10,
|
|
121
|
+
titleMaxLength: 70,
|
|
122
|
+
descriptionMinLength: 50,
|
|
123
|
+
descriptionMaxLength: 160,
|
|
124
|
+
requireCanonical: true
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
//#endregion
|
|
128
|
+
//#region src/audit/scanner-registry.ts
|
|
129
|
+
let ScannerRegistry;
|
|
130
|
+
(function(_ScannerRegistry) {
|
|
131
|
+
class Service extends Context.Service()("pagegraph/ScannerRegistry") {}
|
|
132
|
+
_ScannerRegistry.Service = Service;
|
|
133
|
+
})(ScannerRegistry || (ScannerRegistry = {}));
|
|
134
|
+
const makeScannerRegistry = (scanners) => {
|
|
135
|
+
const values = [...scanners];
|
|
136
|
+
const byId = new Map(values.map((value) => [value.id, value]));
|
|
137
|
+
return {
|
|
138
|
+
all: () => values,
|
|
139
|
+
get: (id) => byId.get(id)
|
|
140
|
+
};
|
|
141
|
+
};
|
|
142
|
+
const ScannerRegistryLive = (scanners) => Layer.succeed(ScannerRegistry.Service, makeScannerRegistry(scanners));
|
|
143
|
+
//#endregion
|
|
144
|
+
//#region src/audit/audit.ts
|
|
145
|
+
var InvalidTarget = class extends Data$1.TaggedError("InvalidTarget") {};
|
|
146
|
+
let Audit;
|
|
147
|
+
(function(_Audit) {
|
|
148
|
+
class Service extends Context.Service()("pagegraph/Audit") {}
|
|
149
|
+
_Audit.Service = Service;
|
|
150
|
+
})(Audit || (Audit = {}));
|
|
151
|
+
const validateTarget = (target) => Effect.try({
|
|
152
|
+
try: () => {
|
|
153
|
+
const url = new URL(target);
|
|
154
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("only http and https URLs are supported");
|
|
155
|
+
if (url.username || url.password) throw new Error("credentials in URLs are not supported");
|
|
156
|
+
return { url: url.toString() };
|
|
157
|
+
},
|
|
158
|
+
catch: (cause) => new InvalidTarget({
|
|
159
|
+
target,
|
|
160
|
+
message: cause instanceof Error ? cause.message : "invalid URL"
|
|
161
|
+
})
|
|
162
|
+
});
|
|
163
|
+
const runOne = Effect.fn("Audit.runScanner")(function* (scanner, target, options) {
|
|
164
|
+
const input = {
|
|
165
|
+
target,
|
|
166
|
+
options
|
|
167
|
+
};
|
|
168
|
+
return yield* scanner.scan(input).pipe(Effect.map((observation) => ({
|
|
169
|
+
scanner: scanner.id,
|
|
170
|
+
target: target.url,
|
|
171
|
+
status: "ok",
|
|
172
|
+
...observation.evidence === void 0 ? {} : { evidence: observation.evidence },
|
|
173
|
+
findings: [...observation.findings ?? [], ...evaluateRules([...scanner.rules ?? [], ...scanner.id === "http" ? pageRules : []], observation, input)]
|
|
174
|
+
})), Effect.catchTag("ScannerFailure", (error) => Effect.succeed({
|
|
175
|
+
scanner: scanner.id,
|
|
176
|
+
target: target.url,
|
|
177
|
+
status: "error",
|
|
178
|
+
findings: [],
|
|
179
|
+
error: error instanceof Error ? error.message : String(error)
|
|
180
|
+
})));
|
|
181
|
+
});
|
|
182
|
+
const runAudit = Effect.fn("Audit.run")(function* (registry, request) {
|
|
183
|
+
const targets = yield* Effect.forEach(request.targets, validateTarget);
|
|
184
|
+
const options = {
|
|
185
|
+
...defaultOptions,
|
|
186
|
+
...request.options ?? {},
|
|
187
|
+
policy: {
|
|
188
|
+
...defaultOptions.policy,
|
|
189
|
+
...request.options?.policy ?? {}
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
const available = registry.all();
|
|
193
|
+
const scanners = request.scanners === void 0 ? available : available.filter((scanner) => request.scanners.includes(scanner.id));
|
|
194
|
+
const pairs = targets.flatMap((target) => scanners.map((scanner) => ({
|
|
195
|
+
target,
|
|
196
|
+
scanner
|
|
197
|
+
})));
|
|
198
|
+
const results = yield* Effect.forEach(pairs, ({ target, scanner }) => runOne(scanner, target, options), { concurrency: Math.max(1, Math.floor(options.concurrency ?? 4)) });
|
|
199
|
+
const findings = results.flatMap((result) => result.findings);
|
|
200
|
+
const warnings = results.filter((result) => result.status === "error").map((result) => `${result.scanner} failed for ${result.target}: ${result.error ?? "unknown error"}`);
|
|
201
|
+
return {
|
|
202
|
+
schemaVersion: 1,
|
|
203
|
+
generatedAt: yield* Effect.clockWith((clock) => Effect.map(clock.currentTimeMillis, (millis) => new Date(millis).toISOString())),
|
|
204
|
+
targets: targets.map(({ url }) => url),
|
|
205
|
+
results,
|
|
206
|
+
findings,
|
|
207
|
+
warnings
|
|
208
|
+
};
|
|
209
|
+
});
|
|
210
|
+
const defaultOptions = {
|
|
211
|
+
concurrency: 4,
|
|
212
|
+
policy: {
|
|
213
|
+
titleMinLength: 10,
|
|
214
|
+
titleMaxLength: 70,
|
|
215
|
+
descriptionMinLength: 50,
|
|
216
|
+
descriptionMaxLength: 160,
|
|
217
|
+
requireCanonical: true
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
const makeAudit = (registry) => ({ run: (request) => runAudit(registry, request) });
|
|
221
|
+
const AuditLive = Layer.effect(Audit.Service, Effect.map(ScannerRegistry.Service, makeAudit));
|
|
222
|
+
//#endregion
|
|
223
|
+
//#region src/audit/model.ts
|
|
224
|
+
const FindingSeverity = Schema.Literals(["structural", "editorial"]);
|
|
225
|
+
/** A machine-readable audit finding. Values are intentionally open-ended so
|
|
226
|
+
* scanner authors can retain the evidence that led to a finding. */
|
|
227
|
+
const AuditFinding = Schema.Struct({
|
|
228
|
+
scanner: Schema.String,
|
|
229
|
+
target: Schema.String,
|
|
230
|
+
severity: FindingSeverity,
|
|
231
|
+
rule: Schema.String,
|
|
232
|
+
message: Schema.String,
|
|
233
|
+
fix: Schema.optionalKey(Schema.String),
|
|
234
|
+
observed: Schema.optionalKey(Schema.Unknown),
|
|
235
|
+
expected: Schema.optionalKey(Schema.Unknown)
|
|
236
|
+
});
|
|
237
|
+
const ScannerStatus = Schema.Literals(["ok", "error"]);
|
|
238
|
+
/** The result of one scanner against one target. Scanner failures are data in
|
|
239
|
+
* a report rather than an un-attributed failure of the entire audit. */
|
|
240
|
+
const ScannerResult = Schema.Struct({
|
|
241
|
+
scanner: Schema.String,
|
|
242
|
+
target: Schema.String,
|
|
243
|
+
status: ScannerStatus,
|
|
244
|
+
evidence: Schema.optionalKey(Schema.Unknown),
|
|
245
|
+
findings: Schema.Array(AuditFinding),
|
|
246
|
+
error: Schema.optionalKey(Schema.String)
|
|
247
|
+
});
|
|
248
|
+
const AuditPolicy = Schema.Struct({
|
|
249
|
+
titleMinLength: Schema.optionalKey(Schema.Number),
|
|
250
|
+
titleMaxLength: Schema.optionalKey(Schema.Number),
|
|
251
|
+
descriptionMinLength: Schema.optionalKey(Schema.Number),
|
|
252
|
+
descriptionMaxLength: Schema.optionalKey(Schema.Number),
|
|
253
|
+
requireCanonical: Schema.optionalKey(Schema.Boolean)
|
|
254
|
+
});
|
|
255
|
+
/** JSON/Markdown report contract for `pagegraph audit`. */
|
|
256
|
+
const AuditReport = Schema.Struct({
|
|
257
|
+
schemaVersion: Schema.Literal(1),
|
|
258
|
+
generatedAt: Schema.String,
|
|
259
|
+
targets: Schema.Array(Schema.String),
|
|
260
|
+
results: Schema.Array(ScannerResult),
|
|
261
|
+
findings: Schema.Array(AuditFinding),
|
|
262
|
+
warnings: Schema.Array(Schema.String)
|
|
263
|
+
});
|
|
264
|
+
//#endregion
|
|
265
|
+
//#region src/audit/diff.ts
|
|
266
|
+
const DiffFindingGroup = Schema.Struct({
|
|
267
|
+
target: Schema.String,
|
|
268
|
+
scanner: Schema.String,
|
|
269
|
+
rule: Schema.String,
|
|
270
|
+
severity: FindingSeverity,
|
|
271
|
+
findings: Schema.Array(AuditFinding)
|
|
272
|
+
});
|
|
273
|
+
const FindingAdded = Schema.TaggedStruct("FindingAdded", { group: DiffFindingGroup });
|
|
274
|
+
const FindingRemoved = Schema.TaggedStruct("FindingRemoved", { group: DiffFindingGroup });
|
|
275
|
+
const FindingChanged = Schema.TaggedStruct("FindingChanged", {
|
|
276
|
+
target: Schema.String,
|
|
277
|
+
scanner: Schema.String,
|
|
278
|
+
rule: Schema.String,
|
|
279
|
+
beforeSeverity: FindingSeverity,
|
|
280
|
+
afterSeverity: FindingSeverity,
|
|
281
|
+
before: Schema.Array(AuditFinding),
|
|
282
|
+
after: Schema.Array(AuditFinding)
|
|
283
|
+
});
|
|
284
|
+
const ScannerDegraded = Schema.TaggedStruct("ScannerDegraded", {
|
|
285
|
+
target: Schema.String,
|
|
286
|
+
scanner: Schema.String
|
|
287
|
+
});
|
|
288
|
+
const ScannerRecovered = Schema.TaggedStruct("ScannerRecovered", {
|
|
289
|
+
target: Schema.String,
|
|
290
|
+
scanner: Schema.String
|
|
291
|
+
});
|
|
292
|
+
const TargetAdded = Schema.TaggedStruct("TargetAdded", { target: Schema.String });
|
|
293
|
+
const TargetRemoved = Schema.TaggedStruct("TargetRemoved", { target: Schema.String });
|
|
294
|
+
const ScannerAdded = Schema.TaggedStruct("ScannerAdded", {
|
|
295
|
+
target: Schema.String,
|
|
296
|
+
scanner: Schema.String,
|
|
297
|
+
status: ScannerStatus
|
|
298
|
+
});
|
|
299
|
+
const ScannerRemoved = Schema.TaggedStruct("ScannerRemoved", {
|
|
300
|
+
target: Schema.String,
|
|
301
|
+
scanner: Schema.String
|
|
302
|
+
});
|
|
303
|
+
/** One normalized semantic change between two audit reports. */
|
|
304
|
+
const AuditDiffChange = Schema.Union([
|
|
305
|
+
FindingAdded,
|
|
306
|
+
FindingRemoved,
|
|
307
|
+
FindingChanged,
|
|
308
|
+
ScannerDegraded,
|
|
309
|
+
ScannerRecovered,
|
|
310
|
+
TargetAdded,
|
|
311
|
+
TargetRemoved,
|
|
312
|
+
ScannerAdded,
|
|
313
|
+
ScannerRemoved
|
|
314
|
+
]).pipe(Schema.toTaggedUnion("_tag"));
|
|
315
|
+
const AuditDiffOutcome = Schema.Literals([
|
|
316
|
+
"unchanged",
|
|
317
|
+
"changed",
|
|
318
|
+
"regressed"
|
|
319
|
+
]);
|
|
320
|
+
const AuditDiffSummary = Schema.Struct({
|
|
321
|
+
structuralRegressions: Schema.Number,
|
|
322
|
+
editorialRegressions: Schema.Number,
|
|
323
|
+
scannerRegressions: Schema.Number,
|
|
324
|
+
coverageRegressions: Schema.Number,
|
|
325
|
+
improvements: Schema.Number,
|
|
326
|
+
informationalChanges: Schema.Number
|
|
327
|
+
});
|
|
328
|
+
const AuditDiffSource = Schema.Struct({
|
|
329
|
+
reportSchemaVersion: Schema.Literal(1),
|
|
330
|
+
generatedAt: Schema.String,
|
|
331
|
+
targets: Schema.Array(Schema.String)
|
|
332
|
+
});
|
|
333
|
+
/** Versioned machine-readable contract emitted by `pagegraph diff --json`. */
|
|
334
|
+
const AuditDiff = Schema.Struct({
|
|
335
|
+
kind: Schema.Literal("audit-diff"),
|
|
336
|
+
schemaVersion: Schema.Literal(1),
|
|
337
|
+
before: AuditDiffSource,
|
|
338
|
+
after: AuditDiffSource,
|
|
339
|
+
outcome: AuditDiffOutcome,
|
|
340
|
+
summary: AuditDiffSummary,
|
|
341
|
+
changes: Schema.Array(AuditDiffChange),
|
|
342
|
+
ignored: Schema.Tuple([
|
|
343
|
+
Schema.Literal("generatedAt"),
|
|
344
|
+
Schema.Literal("warnings"),
|
|
345
|
+
Schema.Literal("results[].evidence")
|
|
346
|
+
])
|
|
347
|
+
});
|
|
348
|
+
const AuditComparisonResult = Data.taggedEnum();
|
|
349
|
+
const compareStrings = (left, right) => left < right ? -1 : left > right ? 1 : 0;
|
|
350
|
+
const canonicalize = (value) => {
|
|
351
|
+
if (Array.isArray(value)) return value.map(canonicalize);
|
|
352
|
+
if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).sort(([left], [right]) => compareStrings(left, right)).map(([key, entry]) => [key, canonicalize(entry)]));
|
|
353
|
+
return value;
|
|
354
|
+
};
|
|
355
|
+
const canonicalJson = (value) => JSON.stringify(canonicalize(value));
|
|
356
|
+
const tupleKey = (...parts) => JSON.stringify(parts);
|
|
357
|
+
const severityOf = (findings) => findings.some((finding) => finding.severity === "structural") ? "structural" : "editorial";
|
|
358
|
+
const findingsByIdentity = (findings) => {
|
|
359
|
+
const groups = /* @__PURE__ */ new Map();
|
|
360
|
+
for (const finding of findings) {
|
|
361
|
+
const key = tupleKey(finding.target, finding.scanner, finding.rule);
|
|
362
|
+
const group = groups.get(key);
|
|
363
|
+
if (group) group.push(finding);
|
|
364
|
+
else groups.set(key, [finding]);
|
|
365
|
+
}
|
|
366
|
+
return new Map([...groups.entries()].map(([key, group]) => {
|
|
367
|
+
const sorted = group.toSorted((left, right) => compareStrings(canonicalJson(left), canonicalJson(right)));
|
|
368
|
+
const first = sorted[0];
|
|
369
|
+
return [key, {
|
|
370
|
+
target: first.target,
|
|
371
|
+
scanner: first.scanner,
|
|
372
|
+
rule: first.rule,
|
|
373
|
+
severity: severityOf(sorted),
|
|
374
|
+
findings: sorted
|
|
375
|
+
}];
|
|
376
|
+
}));
|
|
377
|
+
};
|
|
378
|
+
const validateReport = (label, report) => {
|
|
379
|
+
const issues = [];
|
|
380
|
+
const targets = /* @__PURE__ */ new Set();
|
|
381
|
+
for (const target of report.targets) {
|
|
382
|
+
if (targets.has(target)) issues.push(`${label}: duplicate target ${target}`);
|
|
383
|
+
targets.add(target);
|
|
384
|
+
}
|
|
385
|
+
const results = /* @__PURE__ */ new Set();
|
|
386
|
+
for (const result of report.results) {
|
|
387
|
+
if (!targets.has(result.target)) issues.push(`${label}: scanner ${result.scanner} references undeclared target ${result.target}`);
|
|
388
|
+
const key = tupleKey(result.target, result.scanner);
|
|
389
|
+
if (results.has(key)) issues.push(`${label}: duplicate scanner result ${result.scanner} for ${result.target}`);
|
|
390
|
+
results.add(key);
|
|
391
|
+
for (const finding of result.findings) if (finding.target !== result.target) issues.push(`${label}: finding ${finding.rule} targets ${finding.target} inside result for ${result.target}`);
|
|
392
|
+
}
|
|
393
|
+
const flattened = report.results.flatMap((result) => result.findings).map(canonicalJson).sort(compareStrings);
|
|
394
|
+
const topLevel = report.findings.map(canonicalJson).sort(compareStrings);
|
|
395
|
+
if (canonicalJson(flattened) !== canonicalJson(topLevel)) issues.push(`${label}: top-level findings do not match result findings`);
|
|
396
|
+
return issues;
|
|
397
|
+
};
|
|
398
|
+
const resultMap = (report) => new Map(report.results.map((result) => [tupleKey(result.target, result.scanner), result]));
|
|
399
|
+
const changeSortKey = (change) => {
|
|
400
|
+
switch (change._tag) {
|
|
401
|
+
case "FindingAdded":
|
|
402
|
+
case "FindingRemoved": return tupleKey(change.group.target, change.group.scanner, change.group.rule, change._tag);
|
|
403
|
+
case "FindingChanged": return tupleKey(change.target, change.scanner, change.rule, change._tag);
|
|
404
|
+
case "ScannerAdded":
|
|
405
|
+
case "ScannerRemoved":
|
|
406
|
+
case "ScannerDegraded":
|
|
407
|
+
case "ScannerRecovered": return tupleKey(change.target, change.scanner, "", change._tag);
|
|
408
|
+
case "TargetAdded":
|
|
409
|
+
case "TargetRemoved": return tupleKey(change.target, "", "", change._tag);
|
|
410
|
+
}
|
|
411
|
+
};
|
|
412
|
+
const summarize = (changes) => {
|
|
413
|
+
let structuralRegressions = 0;
|
|
414
|
+
let editorialRegressions = 0;
|
|
415
|
+
let scannerRegressions = 0;
|
|
416
|
+
let coverageRegressions = 0;
|
|
417
|
+
let improvements = 0;
|
|
418
|
+
let informationalChanges = 0;
|
|
419
|
+
for (const change of changes) switch (change._tag) {
|
|
420
|
+
case "FindingAdded":
|
|
421
|
+
if (change.group.severity === "structural") structuralRegressions++;
|
|
422
|
+
else editorialRegressions++;
|
|
423
|
+
break;
|
|
424
|
+
case "FindingRemoved":
|
|
425
|
+
case "ScannerRecovered":
|
|
426
|
+
improvements++;
|
|
427
|
+
break;
|
|
428
|
+
case "FindingChanged":
|
|
429
|
+
if (change.beforeSeverity === "editorial" && change.afterSeverity === "structural") structuralRegressions++;
|
|
430
|
+
else if (change.beforeSeverity === "structural" && change.afterSeverity === "editorial") improvements++;
|
|
431
|
+
else informationalChanges++;
|
|
432
|
+
break;
|
|
433
|
+
case "ScannerDegraded":
|
|
434
|
+
scannerRegressions++;
|
|
435
|
+
break;
|
|
436
|
+
case "TargetRemoved":
|
|
437
|
+
case "ScannerRemoved":
|
|
438
|
+
coverageRegressions++;
|
|
439
|
+
break;
|
|
440
|
+
case "TargetAdded":
|
|
441
|
+
informationalChanges++;
|
|
442
|
+
break;
|
|
443
|
+
case "ScannerAdded": if (change.status === "error") scannerRegressions++;
|
|
444
|
+
else informationalChanges++;
|
|
445
|
+
}
|
|
446
|
+
return {
|
|
447
|
+
structuralRegressions,
|
|
448
|
+
editorialRegressions,
|
|
449
|
+
scannerRegressions,
|
|
450
|
+
coverageRegressions,
|
|
451
|
+
improvements,
|
|
452
|
+
informationalChanges
|
|
453
|
+
};
|
|
454
|
+
};
|
|
455
|
+
const outcomeOf = (summary, changes) => {
|
|
456
|
+
if (summary.structuralRegressions > 0 || summary.scannerRegressions > 0 || summary.coverageRegressions > 0) return "regressed";
|
|
457
|
+
return changes.length > 0 ? "changed" : "unchanged";
|
|
458
|
+
};
|
|
459
|
+
/** Compare two decoded audit reports while ignoring non-semantic scanner evidence. */
|
|
460
|
+
const compareAuditReports = (before, after) => {
|
|
461
|
+
const issues = [...validateReport("before", before), ...validateReport("after", after)];
|
|
462
|
+
if (issues.length > 0) return AuditComparisonResult.InvalidReport({ issues });
|
|
463
|
+
const changes = [];
|
|
464
|
+
const beforeTargets = new Set(before.targets);
|
|
465
|
+
const afterTargets = new Set(after.targets);
|
|
466
|
+
for (const target of beforeTargets) if (!afterTargets.has(target)) changes.push({
|
|
467
|
+
_tag: "TargetRemoved",
|
|
468
|
+
target
|
|
469
|
+
});
|
|
470
|
+
for (const target of afterTargets) if (!beforeTargets.has(target)) changes.push({
|
|
471
|
+
_tag: "TargetAdded",
|
|
472
|
+
target
|
|
473
|
+
});
|
|
474
|
+
const beforeResults = resultMap(before);
|
|
475
|
+
const afterResults = resultMap(after);
|
|
476
|
+
for (const [key, result] of beforeResults) {
|
|
477
|
+
const next = afterResults.get(key);
|
|
478
|
+
if (next === void 0) changes.push({
|
|
479
|
+
_tag: "ScannerRemoved",
|
|
480
|
+
target: result.target,
|
|
481
|
+
scanner: result.scanner
|
|
482
|
+
});
|
|
483
|
+
else if (result.status === "ok" && next.status === "error") changes.push({
|
|
484
|
+
_tag: "ScannerDegraded",
|
|
485
|
+
target: result.target,
|
|
486
|
+
scanner: result.scanner
|
|
487
|
+
});
|
|
488
|
+
else if (result.status === "error" && next.status === "ok") changes.push({
|
|
489
|
+
_tag: "ScannerRecovered",
|
|
490
|
+
target: result.target,
|
|
491
|
+
scanner: result.scanner
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
for (const [key, result] of afterResults) if (!beforeResults.has(key)) changes.push({
|
|
495
|
+
_tag: "ScannerAdded",
|
|
496
|
+
target: result.target,
|
|
497
|
+
scanner: result.scanner,
|
|
498
|
+
status: result.status
|
|
499
|
+
});
|
|
500
|
+
const beforeFindings = findingsByIdentity(before.findings);
|
|
501
|
+
const afterFindings = findingsByIdentity(after.findings);
|
|
502
|
+
for (const [key, group] of beforeFindings) {
|
|
503
|
+
const next = afterFindings.get(key);
|
|
504
|
+
if (next === void 0) changes.push({
|
|
505
|
+
_tag: "FindingRemoved",
|
|
506
|
+
group
|
|
507
|
+
});
|
|
508
|
+
else if (canonicalJson(group.findings) !== canonicalJson(next.findings)) changes.push({
|
|
509
|
+
_tag: "FindingChanged",
|
|
510
|
+
target: group.target,
|
|
511
|
+
scanner: group.scanner,
|
|
512
|
+
rule: group.rule,
|
|
513
|
+
beforeSeverity: group.severity,
|
|
514
|
+
afterSeverity: next.severity,
|
|
515
|
+
before: group.findings,
|
|
516
|
+
after: next.findings
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
for (const [key, group] of afterFindings) if (!beforeFindings.has(key)) changes.push({
|
|
520
|
+
_tag: "FindingAdded",
|
|
521
|
+
group
|
|
522
|
+
});
|
|
523
|
+
changes.sort((left, right) => compareStrings(changeSortKey(left), changeSortKey(right)));
|
|
524
|
+
const summary = summarize(changes);
|
|
525
|
+
const targets = (report) => report.targets.toSorted(compareStrings);
|
|
526
|
+
return AuditComparisonResult.Success({ diff: {
|
|
527
|
+
kind: "audit-diff",
|
|
528
|
+
schemaVersion: 1,
|
|
529
|
+
before: {
|
|
530
|
+
reportSchemaVersion: 1,
|
|
531
|
+
generatedAt: before.generatedAt,
|
|
532
|
+
targets: targets(before)
|
|
533
|
+
},
|
|
534
|
+
after: {
|
|
535
|
+
reportSchemaVersion: 1,
|
|
536
|
+
generatedAt: after.generatedAt,
|
|
537
|
+
targets: targets(after)
|
|
538
|
+
},
|
|
539
|
+
outcome: outcomeOf(summary, changes),
|
|
540
|
+
summary,
|
|
541
|
+
changes,
|
|
542
|
+
ignored: [
|
|
543
|
+
"generatedAt",
|
|
544
|
+
"warnings",
|
|
545
|
+
"results[].evidence"
|
|
546
|
+
]
|
|
547
|
+
} });
|
|
548
|
+
};
|
|
549
|
+
//#endregion
|
|
550
|
+
//#region src/audit/layers.ts
|
|
551
|
+
/** Compose the default audit services with an explicit scanner set. */
|
|
552
|
+
const AuditLayer = (scanners) => AuditLive.pipe(Layer.provide(ScannerRegistryLive(scanners)));
|
|
553
|
+
//#endregion
|
|
554
|
+
//#region src/audit/scanner.ts
|
|
555
|
+
/** A typed, attributed failure from a scanner. The audit service catches this
|
|
556
|
+
* at the scanner boundary and preserves it in the report. */
|
|
557
|
+
var ScannerFailure = class extends Data$1.TaggedError("ScannerFailure") {};
|
|
558
|
+
const scanner = (definition) => definition;
|
|
559
|
+
//#endregion
|
|
560
|
+
//#region src/audit/render.ts
|
|
561
|
+
const cell = (value) => String(value ?? "—").replaceAll("|", "\\|").replaceAll("\n", " ");
|
|
562
|
+
const renderAuditMarkdown = (report) => {
|
|
563
|
+
const structural = report.findings.filter((finding) => finding.severity === "structural");
|
|
564
|
+
const editorial = report.findings.filter((finding) => finding.severity === "editorial");
|
|
565
|
+
const lines = [
|
|
566
|
+
"# SEO audit",
|
|
567
|
+
"",
|
|
568
|
+
`Generated: ${report.generatedAt}`,
|
|
569
|
+
`Targets: ${report.targets.length} · Structural: ${structural.length} · Editorial: ${editorial.length}`,
|
|
570
|
+
"",
|
|
571
|
+
"## Scanner results",
|
|
572
|
+
"",
|
|
573
|
+
"| Scanner | Target | Status | Findings |",
|
|
574
|
+
"| --- | --- | --- | ---: |",
|
|
575
|
+
...report.results.map((result) => `| ${cell(result.scanner)} | ${cell(result.target)} | ${cell(result.status)} | ${result.findings.length} |`)
|
|
576
|
+
];
|
|
577
|
+
if (report.findings.length > 0) lines.push("", "## Findings", "", "| Severity | Rule | Target | Finding |", "| --- | --- | --- | --- |", ...report.findings.map((finding) => `| ${cell(finding.severity)} | ${cell(finding.rule)} | ${cell(finding.target)} | ${cell(finding.message)} |`));
|
|
578
|
+
if (report.warnings.length > 0) lines.push("", "## Warnings", "", ...report.warnings.map((warning) => `- ${warning}`));
|
|
579
|
+
return `${lines.join("\n")}\n`;
|
|
580
|
+
};
|
|
581
|
+
const renderDiffChange = (change) => {
|
|
582
|
+
switch (change._tag) {
|
|
583
|
+
case "FindingAdded": return `+ [${change.group.severity}] ${change.group.rule} · ${change.group.target} (${change.group.scanner})`;
|
|
584
|
+
case "FindingRemoved": return `- [${change.group.severity}] ${change.group.rule} · ${change.group.target} (${change.group.scanner})`;
|
|
585
|
+
case "FindingChanged": return `~ [${change.beforeSeverity} → ${change.afterSeverity}] ${change.rule} · ${change.target} (${change.scanner})`;
|
|
586
|
+
case "ScannerDegraded": return `✗ scanner ${change.scanner} degraded · ${change.target}`;
|
|
587
|
+
case "ScannerRecovered": return `✓ scanner ${change.scanner} recovered · ${change.target}`;
|
|
588
|
+
case "TargetAdded": return `+ target coverage · ${change.target}`;
|
|
589
|
+
case "TargetRemoved": return `- target coverage · ${change.target}`;
|
|
590
|
+
case "ScannerAdded": return `+ scanner coverage ${change.scanner} (${change.status}) · ${change.target}`;
|
|
591
|
+
case "ScannerRemoved": return `- scanner coverage ${change.scanner} · ${change.target}`;
|
|
592
|
+
}
|
|
593
|
+
};
|
|
594
|
+
const diffSection = (title, changes) => changes.length === 0 ? [] : [
|
|
595
|
+
"",
|
|
596
|
+
`## ${title}`,
|
|
597
|
+
"",
|
|
598
|
+
...changes.map(renderDiffChange)
|
|
599
|
+
];
|
|
600
|
+
/** Human-readable rendering of the semantic comparison contract. */
|
|
601
|
+
const renderAuditDiff = (diff) => {
|
|
602
|
+
const regressions = [];
|
|
603
|
+
const editorial = [];
|
|
604
|
+
const improvements = [];
|
|
605
|
+
const informational = [];
|
|
606
|
+
for (const change of diff.changes) switch (change._tag) {
|
|
607
|
+
case "FindingAdded":
|
|
608
|
+
if (change.group.severity === "structural") regressions.push(change);
|
|
609
|
+
else editorial.push(change);
|
|
610
|
+
break;
|
|
611
|
+
case "FindingChanged":
|
|
612
|
+
if (change.beforeSeverity === "editorial" && change.afterSeverity === "structural") regressions.push(change);
|
|
613
|
+
else if (change.beforeSeverity === "structural" && change.afterSeverity === "editorial") improvements.push(change);
|
|
614
|
+
else informational.push(change);
|
|
615
|
+
break;
|
|
616
|
+
case "ScannerDegraded":
|
|
617
|
+
case "TargetRemoved":
|
|
618
|
+
case "ScannerRemoved":
|
|
619
|
+
regressions.push(change);
|
|
620
|
+
break;
|
|
621
|
+
case "FindingRemoved":
|
|
622
|
+
case "ScannerRecovered":
|
|
623
|
+
improvements.push(change);
|
|
624
|
+
break;
|
|
625
|
+
case "TargetAdded":
|
|
626
|
+
informational.push(change);
|
|
627
|
+
break;
|
|
628
|
+
case "ScannerAdded": if (change.status === "error") regressions.push(change);
|
|
629
|
+
else informational.push(change);
|
|
630
|
+
}
|
|
631
|
+
const lines = [
|
|
632
|
+
"# SEO audit diff",
|
|
633
|
+
"",
|
|
634
|
+
`${diff.outcome === "regressed" ? "✗" : diff.outcome === "changed" ? "!" : "✓"} Outcome: ${diff.outcome}`,
|
|
635
|
+
`Before: ${diff.before.generatedAt} · ${diff.before.targets.length} target(s)`,
|
|
636
|
+
`After: ${diff.after.generatedAt} · ${diff.after.targets.length} target(s)`
|
|
637
|
+
];
|
|
638
|
+
if (diff.changes.length === 0) lines.push("", "No semantic changes. Volatile scanner evidence was ignored.");
|
|
639
|
+
else lines.push(...diffSection("Regressions", regressions), ...diffSection("Editorial changes", editorial), ...diffSection("Improvements", improvements), ...diffSection("Other changes", informational));
|
|
640
|
+
lines.push("", `Ignored: ${diff.ignored.join(", ")}`, "", `Summary: ${diff.summary.structuralRegressions} structural · ${diff.summary.editorialRegressions} editorial · ${diff.summary.scannerRegressions} scanner · ${diff.summary.coverageRegressions} coverage regression(s)`);
|
|
641
|
+
return `${lines.join("\n")}\n`;
|
|
642
|
+
};
|
|
643
|
+
const atomicWrite = async (path, contents) => {
|
|
644
|
+
const temporary = `${path}.${process.pid}.${crypto.randomUUID()}.tmp`;
|
|
645
|
+
await writeFile(temporary, contents, "utf8");
|
|
646
|
+
await rename(temporary, path);
|
|
647
|
+
};
|
|
648
|
+
const writeAuditFiles = async (report, outputDirectory) => {
|
|
649
|
+
await mkdir(outputDirectory, { recursive: true });
|
|
650
|
+
const stamp = report.generatedAt.replaceAll(":", "-");
|
|
651
|
+
const json = join(outputDirectory, `${stamp}.json`);
|
|
652
|
+
const markdown = join(outputDirectory, `${stamp}.md`);
|
|
653
|
+
await Promise.all([atomicWrite(json, `${JSON.stringify(report, null, 2)}\n`), atomicWrite(markdown, renderAuditMarkdown(report))]);
|
|
654
|
+
return {
|
|
655
|
+
json,
|
|
656
|
+
markdown
|
|
657
|
+
};
|
|
658
|
+
};
|
|
659
|
+
//#endregion
|
|
660
|
+
//#region src/audit/network.ts
|
|
661
|
+
const blockedIpv4Addresses = new BlockList();
|
|
662
|
+
blockedIpv4Addresses.addAddress("168.63.129.16", "ipv4");
|
|
663
|
+
for (const [network, prefix] of [
|
|
664
|
+
["0.0.0.0", 8],
|
|
665
|
+
["10.0.0.0", 8],
|
|
666
|
+
["100.64.0.0", 10],
|
|
667
|
+
["127.0.0.0", 8],
|
|
668
|
+
["169.254.0.0", 16],
|
|
669
|
+
["172.16.0.0", 12],
|
|
670
|
+
["192.0.0.0", 24],
|
|
671
|
+
["192.0.2.0", 24],
|
|
672
|
+
["192.88.99.0", 24],
|
|
673
|
+
["192.168.0.0", 16],
|
|
674
|
+
["198.18.0.0", 15],
|
|
675
|
+
["198.51.100.0", 24],
|
|
676
|
+
["203.0.113.0", 24],
|
|
677
|
+
["224.0.0.0", 4],
|
|
678
|
+
["240.0.0.0", 4]
|
|
679
|
+
]) blockedIpv4Addresses.addSubnet(network, prefix, "ipv4");
|
|
680
|
+
const ipv4Groups = (address) => {
|
|
681
|
+
const octets = address.split(".").map(Number);
|
|
682
|
+
if (octets.length !== 4 || octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255)) throw new Error(`Invalid embedded IPv4 address: ${address}`);
|
|
683
|
+
return [octets[0] << 8 | octets[1], octets[2] << 8 | octets[3]];
|
|
684
|
+
};
|
|
685
|
+
const ipv6Value = (address) => {
|
|
686
|
+
const [head = "", tail, extra] = address.toLowerCase().split("::");
|
|
687
|
+
if (extra !== void 0) throw new Error(`Invalid IPv6 address: ${address}`);
|
|
688
|
+
const parseGroups = (side) => side === "" ? [] : side.split(":").flatMap((group) => group.includes(".") ? ipv4Groups(group) : [Number.parseInt(group, 16)]);
|
|
689
|
+
const left = parseGroups(head);
|
|
690
|
+
const right = parseGroups(tail ?? "");
|
|
691
|
+
if (left.some((group) => !Number.isInteger(group) || group < 0 || group > 65535)) throw new Error(`Invalid IPv6 address: ${address}`);
|
|
692
|
+
const missing = 8 - left.length - right.length;
|
|
693
|
+
if (tail === void 0 && missing !== 0 || missing < 0) throw new Error(`Invalid IPv6 address: ${address}`);
|
|
694
|
+
return [
|
|
695
|
+
...left,
|
|
696
|
+
...Array(missing).fill(0),
|
|
697
|
+
...right
|
|
698
|
+
].reduce((value, group) => value << 16n | BigInt(group), 0n);
|
|
699
|
+
};
|
|
700
|
+
const blockedIpv6Ranges = [
|
|
701
|
+
["::", 128],
|
|
702
|
+
["::1", 128],
|
|
703
|
+
["::", 96],
|
|
704
|
+
["::ffff:0:0", 96],
|
|
705
|
+
["::ffff:0:0:0", 96],
|
|
706
|
+
["64:ff9b::", 96],
|
|
707
|
+
["64:ff9b:1::", 48],
|
|
708
|
+
["100::", 64],
|
|
709
|
+
["2001::", 23],
|
|
710
|
+
["2001:db8::", 32],
|
|
711
|
+
["2002::", 16],
|
|
712
|
+
["3fff::", 20],
|
|
713
|
+
["5f00::", 16],
|
|
714
|
+
["fc00::", 7],
|
|
715
|
+
["fe80::", 10],
|
|
716
|
+
["ff00::", 8]
|
|
717
|
+
];
|
|
718
|
+
const isBlockedIpv6 = (address) => {
|
|
719
|
+
try {
|
|
720
|
+
const value = ipv6Value(address);
|
|
721
|
+
return blockedIpv6Ranges.some(([network, prefix]) => {
|
|
722
|
+
const shift = BigInt(128 - prefix);
|
|
723
|
+
return value >> shift === ipv6Value(network) >> shift;
|
|
724
|
+
});
|
|
725
|
+
} catch {
|
|
726
|
+
return true;
|
|
727
|
+
}
|
|
728
|
+
};
|
|
729
|
+
const networkHostname = (hostname) => hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
|
|
730
|
+
const isPrivateAddress = (address) => {
|
|
731
|
+
const family = isIP(address);
|
|
732
|
+
if (family === 4) return blockedIpv4Addresses.check(address, "ipv4");
|
|
733
|
+
if (family === 6) return isBlockedIpv6(address);
|
|
734
|
+
return true;
|
|
735
|
+
};
|
|
736
|
+
const resolveHostname = async (hostname) => {
|
|
737
|
+
const normalized = networkHostname(hostname);
|
|
738
|
+
const literalFamily = isIP(normalized);
|
|
739
|
+
if (literalFamily === 4 || literalFamily === 6) return [{
|
|
740
|
+
address: normalized,
|
|
741
|
+
family: literalFamily
|
|
742
|
+
}];
|
|
743
|
+
return (await lookup(normalized, {
|
|
744
|
+
all: true,
|
|
745
|
+
verbatim: true
|
|
746
|
+
})).flatMap(({ address, family }) => family === 4 || family === 6 ? [{
|
|
747
|
+
address,
|
|
748
|
+
family
|
|
749
|
+
}] : []);
|
|
750
|
+
};
|
|
751
|
+
/** Resolve every DNS answer before connecting. A host is rejected if any answer is private. */
|
|
752
|
+
const resolveTargetUrl = async (url, allowPrivate, resolve = resolveHostname) => {
|
|
753
|
+
if (url.protocol !== "https:" && url.protocol !== "http:") throw new Error(`Unsupported URL protocol: ${url.protocol}`);
|
|
754
|
+
if (url.username || url.password) throw new Error("Target URLs must not contain credentials");
|
|
755
|
+
const hostname = networkHostname(url.hostname).toLowerCase();
|
|
756
|
+
if (!allowPrivate && (hostname === "localhost" || hostname.endsWith(".local"))) throw new Error(`Private target is not allowed: ${hostname}`);
|
|
757
|
+
const addresses = await resolve(hostname);
|
|
758
|
+
if (addresses.length === 0 || addresses.some(({ address, family }) => isIP(address) !== family) || !allowPrivate && addresses.some(({ address }) => isPrivateAddress(address))) throw new Error(`Private target is not allowed: ${hostname}`);
|
|
759
|
+
return addresses;
|
|
760
|
+
};
|
|
761
|
+
/** A lookup callback pinned to the already validated DNS answers. */
|
|
762
|
+
const pinnedLookup = (addresses) => {
|
|
763
|
+
const [first] = addresses;
|
|
764
|
+
if (first === void 0) throw new Error("Cannot create a pinned lookup without an address");
|
|
765
|
+
return (_hostname, options, callback) => {
|
|
766
|
+
if (options.all) {
|
|
767
|
+
callback(null, addresses.map(({ address, family }) => ({
|
|
768
|
+
address,
|
|
769
|
+
family
|
|
770
|
+
})));
|
|
771
|
+
return;
|
|
772
|
+
}
|
|
773
|
+
callback(null, first.address, first.family);
|
|
774
|
+
};
|
|
775
|
+
};
|
|
776
|
+
//#endregion
|
|
777
|
+
//#region src/audit/scanners/http.ts
|
|
778
|
+
var HttpProbeError = class extends Data$1.TaggedError("HttpProbeError") {};
|
|
779
|
+
const errorMessage$2 = (error) => error instanceof Error ? error.message : String(error);
|
|
780
|
+
const discoveryPaths = [
|
|
781
|
+
{
|
|
782
|
+
kind: "robots",
|
|
783
|
+
path: "/robots.txt",
|
|
784
|
+
accept: "text/plain, */*;q=0.1"
|
|
785
|
+
},
|
|
786
|
+
{
|
|
787
|
+
kind: "sitemap",
|
|
788
|
+
path: "/sitemap.xml",
|
|
789
|
+
accept: "application/xml, text/xml;q=0.9, */*;q=0.1"
|
|
790
|
+
},
|
|
791
|
+
{
|
|
792
|
+
kind: "llms",
|
|
793
|
+
path: "/llms.txt",
|
|
794
|
+
accept: "text/plain, text/markdown;q=0.9, */*;q=0.1"
|
|
795
|
+
}
|
|
796
|
+
];
|
|
797
|
+
const selectedHeaders = [
|
|
798
|
+
"cache-control",
|
|
799
|
+
"cf-cache-status",
|
|
800
|
+
"content-language",
|
|
801
|
+
"content-length",
|
|
802
|
+
"content-signal",
|
|
803
|
+
"content-type",
|
|
804
|
+
"link",
|
|
805
|
+
"server",
|
|
806
|
+
"vary",
|
|
807
|
+
"x-robots-tag"
|
|
808
|
+
];
|
|
809
|
+
const buildProbeRequests = (targets, extraDiscovery = []) => {
|
|
810
|
+
const requests = [];
|
|
811
|
+
const seenDiscoveryOrigins = /* @__PURE__ */ new Set();
|
|
812
|
+
for (const target of targets) {
|
|
813
|
+
requests.push({
|
|
814
|
+
kind: "page-head",
|
|
815
|
+
method: "HEAD",
|
|
816
|
+
accept: "text/html, */*;q=0.1",
|
|
817
|
+
url: target
|
|
818
|
+
}, {
|
|
819
|
+
kind: "page-html",
|
|
820
|
+
method: "GET",
|
|
821
|
+
accept: "text/html, */*;q=0.1",
|
|
822
|
+
url: target
|
|
823
|
+
}, {
|
|
824
|
+
kind: "page-markdown",
|
|
825
|
+
method: "GET",
|
|
826
|
+
accept: "text/markdown, text/plain;q=0.8, */*;q=0.1",
|
|
827
|
+
url: target
|
|
828
|
+
});
|
|
829
|
+
if (seenDiscoveryOrigins.has(target.origin)) continue;
|
|
830
|
+
seenDiscoveryOrigins.add(target.origin);
|
|
831
|
+
for (const discovery of [...discoveryPaths, ...extraDiscovery]) requests.push({
|
|
832
|
+
kind: discovery.kind,
|
|
833
|
+
method: "GET",
|
|
834
|
+
accept: discovery.accept,
|
|
835
|
+
url: new URL(discovery.path, target.origin)
|
|
836
|
+
});
|
|
837
|
+
}
|
|
838
|
+
return requests;
|
|
839
|
+
};
|
|
840
|
+
const responseHeaders = (headers) => {
|
|
841
|
+
const values = {};
|
|
842
|
+
for (const header of selectedHeaders) {
|
|
843
|
+
const value = headers[header];
|
|
844
|
+
if (value !== void 0) values[header] = Array.isArray(value) ? value.join(", ") : value;
|
|
845
|
+
}
|
|
846
|
+
return values;
|
|
847
|
+
};
|
|
848
|
+
const attribute = (tag, name) => {
|
|
849
|
+
const match = new RegExp(`${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i").exec(tag);
|
|
850
|
+
return match?.[1] ?? match?.[2] ?? match?.[3] ?? null;
|
|
851
|
+
};
|
|
852
|
+
const metaContent = (html, key) => {
|
|
853
|
+
for (const match of html.matchAll(/<meta\b[^>]*>/gi)) {
|
|
854
|
+
const tag = match[0];
|
|
855
|
+
if (attribute(tag, "name")?.toLowerCase() === key) return attribute(tag, "content");
|
|
856
|
+
}
|
|
857
|
+
return null;
|
|
858
|
+
};
|
|
859
|
+
const canonicalUrl = (html) => {
|
|
860
|
+
for (const match of html.matchAll(/<link\b[^>]*>/gi)) {
|
|
861
|
+
const tag = match[0];
|
|
862
|
+
if ((attribute(tag, "rel")?.toLowerCase().split(/\s+/) ?? []).includes("canonical")) return attribute(tag, "href");
|
|
863
|
+
}
|
|
864
|
+
return null;
|
|
865
|
+
};
|
|
866
|
+
const countMatches = (value, pattern) => [...value.matchAll(pattern)].length;
|
|
867
|
+
const extractDocumentSignals = (body, contentType, requestedUrl) => {
|
|
868
|
+
const isHtml = contentType.includes("text/html");
|
|
869
|
+
const isMarkdown = contentType.includes("text/markdown");
|
|
870
|
+
const looksJson = contentType.includes("json") || requestedUrl.pathname.endsWith(".json");
|
|
871
|
+
const titleMatch = isHtml ? /<title\b[^>]*>([\s\S]*?)<\/title>/i.exec(body) : null;
|
|
872
|
+
let jsonValid = null;
|
|
873
|
+
if (looksJson && body.trim().length > 0) try {
|
|
874
|
+
JSON.parse(body);
|
|
875
|
+
jsonValid = true;
|
|
876
|
+
} catch {
|
|
877
|
+
jsonValid = false;
|
|
878
|
+
}
|
|
879
|
+
return {
|
|
880
|
+
title: titleMatch?.[1]?.replace(/\s+/g, " ").trim() ?? null,
|
|
881
|
+
description: isHtml ? metaContent(body, "description") : null,
|
|
882
|
+
canonicalUrl: isHtml ? canonicalUrl(body) : null,
|
|
883
|
+
robots: isHtml ? metaContent(body, "robots") : null,
|
|
884
|
+
jsonLdCount: isHtml ? countMatches(body, /<script\b[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>/gi) : 0,
|
|
885
|
+
modulePreloadCount: isHtml ? countMatches(body, /<link\b[^>]*rel\s*=\s*["'][^"']*modulepreload[^"']*["'][^>]*>/gi) : 0,
|
|
886
|
+
scriptCount: isHtml ? countMatches(body, /<script\b/gi) : 0,
|
|
887
|
+
stylesheetCount: isHtml ? countMatches(body, /<link\b[^>]*rel\s*=\s*["'][^"']*stylesheet[^"']*["'][^>]*>/gi) : 0,
|
|
888
|
+
wordCount: isMarkdown ? body.trim().split(/\s+/).filter(Boolean).length : null,
|
|
889
|
+
jsonValid
|
|
890
|
+
};
|
|
891
|
+
};
|
|
892
|
+
const readBoundedBody = async (response, maxBodyBytes) => {
|
|
893
|
+
const chunks = [];
|
|
894
|
+
let length = 0;
|
|
895
|
+
let truncated = false;
|
|
896
|
+
for await (const chunk of response) {
|
|
897
|
+
const value = new Uint8Array(chunk);
|
|
898
|
+
const remaining = maxBodyBytes - length;
|
|
899
|
+
if (value.byteLength > remaining) {
|
|
900
|
+
if (remaining > 0) chunks.push(value.slice(0, remaining));
|
|
901
|
+
length += Math.max(remaining, 0);
|
|
902
|
+
truncated = true;
|
|
903
|
+
response.destroy();
|
|
904
|
+
break;
|
|
905
|
+
}
|
|
906
|
+
chunks.push(value);
|
|
907
|
+
length += value.byteLength;
|
|
908
|
+
}
|
|
909
|
+
const bytes = new Uint8Array(length);
|
|
910
|
+
let offset = 0;
|
|
911
|
+
for (const chunk of chunks) {
|
|
912
|
+
bytes.set(chunk, offset);
|
|
913
|
+
offset += chunk.byteLength;
|
|
914
|
+
}
|
|
915
|
+
return {
|
|
916
|
+
bytes,
|
|
917
|
+
truncated
|
|
918
|
+
};
|
|
919
|
+
};
|
|
920
|
+
const bodyExcerpt = (body, contentType) => {
|
|
921
|
+
if (contentType.includes("text/html") || body.length === 0) return null;
|
|
922
|
+
return [...body].filter((character) => {
|
|
923
|
+
const code = character.charCodeAt(0);
|
|
924
|
+
return code === 9 || code === 10 || code === 13 || code >= 32;
|
|
925
|
+
}).join("").slice(0, 600);
|
|
926
|
+
};
|
|
927
|
+
const requestPinned = async (url, request$4, options) => {
|
|
928
|
+
const addresses = await resolveTargetUrl(url, options.allowPrivate, options.resolve);
|
|
929
|
+
const transport = url.protocol === "https:" ? request$1 : request;
|
|
930
|
+
const hostname = networkHostname(url.hostname);
|
|
931
|
+
return new Promise((resolve, reject) => {
|
|
932
|
+
const outgoing = transport({
|
|
933
|
+
hostname,
|
|
934
|
+
port: url.port || void 0,
|
|
935
|
+
path: `${url.pathname}${url.search}`,
|
|
936
|
+
method: request$4.method,
|
|
937
|
+
headers: {
|
|
938
|
+
accept: request$4.accept,
|
|
939
|
+
host: url.host,
|
|
940
|
+
"user-agent": "pagegraph/0.4 (+https://tanstack.com)"
|
|
941
|
+
},
|
|
942
|
+
agent: false,
|
|
943
|
+
lookup: pinnedLookup(addresses),
|
|
944
|
+
signal: AbortSignal.timeout(options.timeoutMs),
|
|
945
|
+
...url.protocol === "https:" ? {
|
|
946
|
+
servername: isIP(hostname) === 0 ? hostname : void 0,
|
|
947
|
+
checkServerIdentity: (_host, certificate) => checkServerIdentity(hostname, certificate)
|
|
948
|
+
} : void 0
|
|
949
|
+
});
|
|
950
|
+
outgoing.once("response", resolve);
|
|
951
|
+
outgoing.once("error", reject);
|
|
952
|
+
outgoing.end();
|
|
953
|
+
});
|
|
954
|
+
};
|
|
955
|
+
const probeHttp = async (request, options) => {
|
|
956
|
+
const startedAt = performance.now();
|
|
957
|
+
const redirects = [];
|
|
958
|
+
let headersAt = null;
|
|
959
|
+
try {
|
|
960
|
+
let currentUrl = request.url;
|
|
961
|
+
let response = null;
|
|
962
|
+
for (let redirectCount = 0; redirectCount <= 10; redirectCount += 1) {
|
|
963
|
+
response = await requestPinned(currentUrl, request, options);
|
|
964
|
+
headersAt = performance.now();
|
|
965
|
+
const location = response.headers.location;
|
|
966
|
+
const status = response.statusCode ?? 0;
|
|
967
|
+
if (status < 300 || status >= 400 || location === void 0) break;
|
|
968
|
+
if (redirectCount === 10) throw new Error("Redirect limit exceeded");
|
|
969
|
+
response.destroy();
|
|
970
|
+
currentUrl = new URL(location, currentUrl);
|
|
971
|
+
redirects.push(currentUrl.href);
|
|
972
|
+
}
|
|
973
|
+
if (response === null) throw new Error("Request did not produce a response");
|
|
974
|
+
const bounded = request.method === "HEAD" ? {
|
|
975
|
+
bytes: /* @__PURE__ */ new Uint8Array(),
|
|
976
|
+
truncated: false
|
|
977
|
+
} : await readBoundedBody(response, options.maxBodyBytes);
|
|
978
|
+
const contentType = typeof response.headers["content-type"] === "string" ? response.headers["content-type"] : "";
|
|
979
|
+
const body = new TextDecoder().decode(bounded.bytes);
|
|
980
|
+
const anchors = options.captureAnchors === true && contentType.includes("text/html") ? extractAnchors(body, currentUrl.href) : void 0;
|
|
981
|
+
return {
|
|
982
|
+
kind: request.kind,
|
|
983
|
+
method: request.method,
|
|
984
|
+
accept: request.accept,
|
|
985
|
+
requestedUrl: request.url.href,
|
|
986
|
+
finalUrl: currentUrl.href,
|
|
987
|
+
redirects,
|
|
988
|
+
status: response.statusCode ?? null,
|
|
989
|
+
ok: response.statusCode !== void 0 && response.statusCode >= 200 && response.statusCode < 300,
|
|
990
|
+
headersMs: headersAt === null ? null : Math.round((headersAt - startedAt) * 10) / 10,
|
|
991
|
+
totalMs: Math.round((performance.now() - startedAt) * 10) / 10,
|
|
992
|
+
responseHeaders: responseHeaders(response.headers),
|
|
993
|
+
capturedBodyBytes: bounded.bytes.byteLength,
|
|
994
|
+
capturedBodySha256: bounded.bytes.byteLength === 0 ? null : createHash("sha256").update(bounded.bytes).digest("hex"),
|
|
995
|
+
bodyTruncated: bounded.truncated,
|
|
996
|
+
bodyExcerpt: bodyExcerpt(body, contentType),
|
|
997
|
+
document: body.length === 0 ? null : extractDocumentSignals(body, contentType, currentUrl),
|
|
998
|
+
anchors,
|
|
999
|
+
error: null
|
|
1000
|
+
};
|
|
1001
|
+
} catch (error) {
|
|
1002
|
+
return {
|
|
1003
|
+
kind: request.kind,
|
|
1004
|
+
method: request.method,
|
|
1005
|
+
accept: request.accept,
|
|
1006
|
+
requestedUrl: request.url.href,
|
|
1007
|
+
finalUrl: null,
|
|
1008
|
+
redirects,
|
|
1009
|
+
status: null,
|
|
1010
|
+
ok: false,
|
|
1011
|
+
headersMs: headersAt === null ? null : Math.round((headersAt - startedAt) * 10) / 10,
|
|
1012
|
+
totalMs: Math.round((performance.now() - startedAt) * 10) / 10,
|
|
1013
|
+
responseHeaders: {},
|
|
1014
|
+
capturedBodyBytes: 0,
|
|
1015
|
+
capturedBodySha256: null,
|
|
1016
|
+
bodyTruncated: false,
|
|
1017
|
+
bodyExcerpt: null,
|
|
1018
|
+
document: null,
|
|
1019
|
+
error: errorMessage$2(error)
|
|
1020
|
+
};
|
|
1021
|
+
}
|
|
1022
|
+
};
|
|
1023
|
+
const probeAll = async (requests, options) => {
|
|
1024
|
+
const results = Array.from({ length: requests.length });
|
|
1025
|
+
const pending = requests.map((request, index) => ({
|
|
1026
|
+
index,
|
|
1027
|
+
request
|
|
1028
|
+
}));
|
|
1029
|
+
const workers = Array.from({ length: Math.min(4, Math.max(1, pending.length)) }, async () => {
|
|
1030
|
+
while (pending.length > 0) {
|
|
1031
|
+
const item = pending.shift();
|
|
1032
|
+
if (item !== void 0) results[item.index] = await probeHttp(item.request, options);
|
|
1033
|
+
}
|
|
1034
|
+
});
|
|
1035
|
+
await Promise.all(workers);
|
|
1036
|
+
return results.map((result, index) => {
|
|
1037
|
+
if (result === void 0) throw new Error(`Probe ${index} did not produce a result`);
|
|
1038
|
+
return result;
|
|
1039
|
+
});
|
|
1040
|
+
};
|
|
1041
|
+
Effect.fn("SeoAudit.probeHttp")(function* (request, options) {
|
|
1042
|
+
return yield* Effect.tryPromise({
|
|
1043
|
+
try: () => probeHttp(request, options),
|
|
1044
|
+
catch: (cause) => new HttpProbeError({
|
|
1045
|
+
message: errorMessage$2(cause),
|
|
1046
|
+
cause
|
|
1047
|
+
})
|
|
1048
|
+
});
|
|
1049
|
+
});
|
|
1050
|
+
const makeHttpScanner = (options) => ({
|
|
1051
|
+
id: "http",
|
|
1052
|
+
description: "HTTP metadata, discovery, and content-negotiation probes",
|
|
1053
|
+
scan: (input) => Effect.fn("SeoAudit.scanHttp")(function* () {
|
|
1054
|
+
const target = new URL(input.target.url);
|
|
1055
|
+
const evidence = yield* Effect.tryPromise({
|
|
1056
|
+
try: () => probeAll(buildProbeRequests([target], options.discovery), options),
|
|
1057
|
+
catch: (cause) => new ScannerFailure({
|
|
1058
|
+
scanner: "http",
|
|
1059
|
+
target: target.href,
|
|
1060
|
+
message: errorMessage$2(cause),
|
|
1061
|
+
cause
|
|
1062
|
+
})
|
|
1063
|
+
});
|
|
1064
|
+
const acquisitionErrors = evidence.flatMap((probe) => probe.status === null && probe.error !== null ? [probe.error] : []);
|
|
1065
|
+
if (acquisitionErrors.length === evidence.length) return yield* new ScannerFailure({
|
|
1066
|
+
scanner: "http",
|
|
1067
|
+
target: target.href,
|
|
1068
|
+
message: `Every HTTP probe failed: ${[...new Set(acquisitionErrors)].join("; ")}`
|
|
1069
|
+
});
|
|
1070
|
+
return { evidence };
|
|
1071
|
+
})()
|
|
1072
|
+
});
|
|
1073
|
+
//#endregion
|
|
1074
|
+
//#region src/audit/proxy.ts
|
|
1075
|
+
const proxyRequestHeaders = (headers, target) => {
|
|
1076
|
+
const forwarded = {
|
|
1077
|
+
...headers,
|
|
1078
|
+
host: target.host
|
|
1079
|
+
};
|
|
1080
|
+
delete forwarded["proxy-authorization"];
|
|
1081
|
+
delete forwarded["proxy-connection"];
|
|
1082
|
+
return forwarded;
|
|
1083
|
+
};
|
|
1084
|
+
const connectAuthority = (authority) => {
|
|
1085
|
+
const target = new URL(`http://${authority}`);
|
|
1086
|
+
if (target.username || target.password || target.pathname !== "/" || target.search || target.hash) throw new Error("Invalid proxy CONNECT authority");
|
|
1087
|
+
return target;
|
|
1088
|
+
};
|
|
1089
|
+
const connectPinned = (target, addresses, port, timeoutMs, track) => new Promise((resolve, reject) => {
|
|
1090
|
+
const socket = connect({
|
|
1091
|
+
host: networkHostname(target.hostname),
|
|
1092
|
+
port,
|
|
1093
|
+
autoSelectFamily: true,
|
|
1094
|
+
lookup: pinnedLookup(addresses)
|
|
1095
|
+
});
|
|
1096
|
+
track(socket);
|
|
1097
|
+
socket.setTimeout(timeoutMs, () => socket.destroy(/* @__PURE__ */ new Error("Proxy connection timed out")));
|
|
1098
|
+
socket.once("connect", () => resolve(socket));
|
|
1099
|
+
socket.once("error", reject);
|
|
1100
|
+
});
|
|
1101
|
+
/**
|
|
1102
|
+
* Start a loopback-only validating proxy for browser scanners. Every request,
|
|
1103
|
+
* including CONNECT and every redirect hop, is DNS-pinned and revalidated.
|
|
1104
|
+
*/
|
|
1105
|
+
const startAuditProxy = async (options) => {
|
|
1106
|
+
const sockets = /* @__PURE__ */ new Set();
|
|
1107
|
+
const timeoutMs = options.timeoutMs ?? 45e3;
|
|
1108
|
+
let closed = false;
|
|
1109
|
+
let closePromise;
|
|
1110
|
+
const track = (socket) => {
|
|
1111
|
+
sockets.add(socket);
|
|
1112
|
+
socket.once("close", () => sockets.delete(socket));
|
|
1113
|
+
};
|
|
1114
|
+
const server = createServer(async (request$3, response) => {
|
|
1115
|
+
try {
|
|
1116
|
+
if (request$3.url === void 0) throw new Error("Proxy request URL is missing");
|
|
1117
|
+
const target = new URL(request$3.url);
|
|
1118
|
+
if (target.protocol !== "http:") throw new Error("HTTPS proxy requests must use CONNECT");
|
|
1119
|
+
const addresses = await resolveTargetUrl(target, options.allowPrivate, options.resolve);
|
|
1120
|
+
if (closed || request$3.destroyed) throw new Error("Audit proxy is closed");
|
|
1121
|
+
const upstream = request({
|
|
1122
|
+
hostname: networkHostname(target.hostname),
|
|
1123
|
+
port: target.port || void 0,
|
|
1124
|
+
path: `${target.pathname}${target.search}`,
|
|
1125
|
+
method: request$3.method,
|
|
1126
|
+
headers: proxyRequestHeaders(request$3.headers, target),
|
|
1127
|
+
agent: false,
|
|
1128
|
+
lookup: pinnedLookup(addresses)
|
|
1129
|
+
});
|
|
1130
|
+
upstream.once("socket", track);
|
|
1131
|
+
upstream.setTimeout(timeoutMs, () => upstream.destroy(/* @__PURE__ */ new Error("Proxy request timed out")));
|
|
1132
|
+
upstream.once("response", (upstreamResponse) => {
|
|
1133
|
+
response.writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers);
|
|
1134
|
+
upstreamResponse.pipe(response);
|
|
1135
|
+
});
|
|
1136
|
+
upstream.once("error", (error) => {
|
|
1137
|
+
if (!response.headersSent) response.writeHead(502, { "content-type": "text/plain" });
|
|
1138
|
+
response.end(error.message);
|
|
1139
|
+
});
|
|
1140
|
+
request$3.pipe(upstream);
|
|
1141
|
+
} catch (error) {
|
|
1142
|
+
response.writeHead(403, { "content-type": "text/plain" });
|
|
1143
|
+
response.end(error instanceof Error ? error.message : String(error));
|
|
1144
|
+
}
|
|
1145
|
+
});
|
|
1146
|
+
server.on("connection", track);
|
|
1147
|
+
server.on("connect", async (request, client, head) => {
|
|
1148
|
+
try {
|
|
1149
|
+
if (request.url === void 0) throw new Error("Proxy CONNECT authority is missing");
|
|
1150
|
+
const target = connectAuthority(request.url);
|
|
1151
|
+
const port = target.port === "" ? 443 : Number(target.port);
|
|
1152
|
+
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("Invalid proxy CONNECT port");
|
|
1153
|
+
const addresses = await resolveTargetUrl(target, options.allowPrivate, options.resolve);
|
|
1154
|
+
if (closed || client.destroyed) throw new Error("Audit proxy is closed");
|
|
1155
|
+
const upstream = await connectPinned(target, addresses, port, timeoutMs, track);
|
|
1156
|
+
client.write("HTTP/1.1 200 Connection Established\r\n\r\n");
|
|
1157
|
+
if (head.byteLength > 0) upstream.write(head);
|
|
1158
|
+
upstream.pipe(client);
|
|
1159
|
+
client.pipe(upstream);
|
|
1160
|
+
} catch (error) {
|
|
1161
|
+
client.end(`HTTP/1.1 403 Forbidden\r\nConnection: close\r\nContent-Type: text/plain\r\n\r\n${error instanceof Error ? error.message : String(error)}`);
|
|
1162
|
+
}
|
|
1163
|
+
});
|
|
1164
|
+
await new Promise((resolve, reject) => {
|
|
1165
|
+
server.once("error", reject);
|
|
1166
|
+
server.listen(0, "127.0.0.1", resolve);
|
|
1167
|
+
});
|
|
1168
|
+
const address = server.address();
|
|
1169
|
+
if (address === null || typeof address === "string") throw new Error("Audit proxy did not bind");
|
|
1170
|
+
return {
|
|
1171
|
+
url: `http://127.0.0.1:${address.port}`,
|
|
1172
|
+
close: () => {
|
|
1173
|
+
closePromise ??= new Promise((resolve, reject) => {
|
|
1174
|
+
closed = true;
|
|
1175
|
+
for (const socket of sockets) socket.destroy();
|
|
1176
|
+
server.close((error) => error ? reject(error) : resolve());
|
|
1177
|
+
});
|
|
1178
|
+
return closePromise;
|
|
1179
|
+
}
|
|
1180
|
+
};
|
|
1181
|
+
};
|
|
1182
|
+
//#endregion
|
|
1183
|
+
//#region src/audit/scanners/lighthouse.ts
|
|
1184
|
+
const require = createRequire(import.meta.url);
|
|
1185
|
+
const metricIds = [
|
|
1186
|
+
"first-contentful-paint",
|
|
1187
|
+
"largest-contentful-paint",
|
|
1188
|
+
"total-blocking-time",
|
|
1189
|
+
"cumulative-layout-shift",
|
|
1190
|
+
"speed-index"
|
|
1191
|
+
];
|
|
1192
|
+
const UnknownRecord = Schema$1.Record(Schema$1.String, Schema$1.Unknown);
|
|
1193
|
+
const decodeUnknownRecord = Schema$1.decodeUnknownOption(UnknownRecord);
|
|
1194
|
+
const asRecord = (value) => Option.getOrNull(decodeUnknownRecord(value));
|
|
1195
|
+
const asString = (value) => typeof value === "string" ? value : null;
|
|
1196
|
+
const asNumber = (value) => typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
1197
|
+
var LighthouseError = class extends Data$1.TaggedError("LighthouseError") {};
|
|
1198
|
+
const errorMessage$1 = (error) => error instanceof Error ? error.message : String(error);
|
|
1199
|
+
const resolveLighthouse = () => {
|
|
1200
|
+
try {
|
|
1201
|
+
return {
|
|
1202
|
+
packageJson: require.resolve("lighthouse/package.json"),
|
|
1203
|
+
cli: require.resolve("lighthouse/cli/index.js")
|
|
1204
|
+
};
|
|
1205
|
+
} catch (cause) {
|
|
1206
|
+
throw new LighthouseError({
|
|
1207
|
+
message: "Lighthouse is not installed. Install it as an optional dependency to enable browser audits.",
|
|
1208
|
+
cause
|
|
1209
|
+
});
|
|
1210
|
+
}
|
|
1211
|
+
};
|
|
1212
|
+
const parseCategories = (value) => {
|
|
1213
|
+
const categories = {};
|
|
1214
|
+
for (const [id, candidate] of Object.entries(asRecord(value) ?? {})) {
|
|
1215
|
+
const category = asRecord(candidate);
|
|
1216
|
+
if (category !== null) categories[id] = {
|
|
1217
|
+
id,
|
|
1218
|
+
title: asString(category.title) ?? id,
|
|
1219
|
+
score: asNumber(category.score)
|
|
1220
|
+
};
|
|
1221
|
+
}
|
|
1222
|
+
return categories;
|
|
1223
|
+
};
|
|
1224
|
+
const parseMetrics = (auditsValue) => {
|
|
1225
|
+
const metrics = {};
|
|
1226
|
+
const audits = asRecord(auditsValue) ?? {};
|
|
1227
|
+
for (const id of metricIds) {
|
|
1228
|
+
const audit = asRecord(audits[id]);
|
|
1229
|
+
if (audit !== null) metrics[id] = {
|
|
1230
|
+
id,
|
|
1231
|
+
title: asString(audit.title) ?? id,
|
|
1232
|
+
numericValue: asNumber(audit.numericValue),
|
|
1233
|
+
numericUnit: asString(audit.numericUnit),
|
|
1234
|
+
displayValue: asString(audit.displayValue),
|
|
1235
|
+
score: asNumber(audit.score)
|
|
1236
|
+
};
|
|
1237
|
+
}
|
|
1238
|
+
return metrics;
|
|
1239
|
+
};
|
|
1240
|
+
const parseOpportunities = (auditsValue) => {
|
|
1241
|
+
const opportunities = [];
|
|
1242
|
+
for (const [id, candidate] of Object.entries(asRecord(auditsValue) ?? {})) {
|
|
1243
|
+
const audit = asRecord(candidate);
|
|
1244
|
+
const details = asRecord(audit?.details);
|
|
1245
|
+
if (audit === null || details?.type !== "opportunity") continue;
|
|
1246
|
+
const savingsMs = asNumber(details.overallSavingsMs);
|
|
1247
|
+
const savingsBytes = asNumber(details.overallSavingsBytes);
|
|
1248
|
+
if ((savingsMs ?? 0) <= 0 && (savingsBytes ?? 0) <= 0) continue;
|
|
1249
|
+
opportunities.push({
|
|
1250
|
+
id,
|
|
1251
|
+
title: asString(audit.title) ?? id,
|
|
1252
|
+
displayValue: asString(audit.displayValue),
|
|
1253
|
+
savingsMs,
|
|
1254
|
+
savingsBytes
|
|
1255
|
+
});
|
|
1256
|
+
}
|
|
1257
|
+
return opportunities.sort((left, right) => (right.savingsMs ?? 0) - (left.savingsMs ?? 0) || (right.savingsBytes ?? 0) - (left.savingsBytes ?? 0));
|
|
1258
|
+
};
|
|
1259
|
+
const parseWarnings = (value) => Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
1260
|
+
const parseLighthouseResult = (value, requestedUrl, formFactor, run) => {
|
|
1261
|
+
const result = asRecord(value);
|
|
1262
|
+
if (result === null) throw new LighthouseError({ message: "Lighthouse returned a non-object result" });
|
|
1263
|
+
const environment = asRecord(result.environment);
|
|
1264
|
+
return {
|
|
1265
|
+
requestedUrl,
|
|
1266
|
+
finalUrl: asString(result.finalDisplayedUrl) ?? asString(result.finalUrl),
|
|
1267
|
+
formFactor,
|
|
1268
|
+
run,
|
|
1269
|
+
lighthouseVersion: asString(result.lighthouseVersion),
|
|
1270
|
+
userAgent: asString(environment?.hostUserAgent),
|
|
1271
|
+
fetchTime: asString(result.fetchTime),
|
|
1272
|
+
categories: parseCategories(result.categories),
|
|
1273
|
+
metrics: parseMetrics(result.audits),
|
|
1274
|
+
opportunities: parseOpportunities(result.audits),
|
|
1275
|
+
warnings: parseWarnings(result.runWarnings),
|
|
1276
|
+
error: null
|
|
1277
|
+
};
|
|
1278
|
+
};
|
|
1279
|
+
const lighthouseVersion = async () => {
|
|
1280
|
+
try {
|
|
1281
|
+
const { packageJson } = resolveLighthouse();
|
|
1282
|
+
const value = JSON.parse(await readFile(packageJson, "utf8"));
|
|
1283
|
+
return asString(asRecord(value)?.version);
|
|
1284
|
+
} catch {
|
|
1285
|
+
return null;
|
|
1286
|
+
}
|
|
1287
|
+
};
|
|
1288
|
+
const lighthouseChromeFlags = (proxyUrl) => [
|
|
1289
|
+
"--headless=new",
|
|
1290
|
+
`--proxy-server=${proxyUrl}`,
|
|
1291
|
+
"--proxy-bypass-list=<-loopback>",
|
|
1292
|
+
"--disable-quic",
|
|
1293
|
+
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp"
|
|
1294
|
+
].join(" ");
|
|
1295
|
+
const windowsProcessTreeKill = (pid) => ({
|
|
1296
|
+
command: "taskkill",
|
|
1297
|
+
arguments: [
|
|
1298
|
+
"/pid",
|
|
1299
|
+
String(pid),
|
|
1300
|
+
"/T",
|
|
1301
|
+
"/F"
|
|
1302
|
+
]
|
|
1303
|
+
});
|
|
1304
|
+
const runLighthouseProcess = async (cli, arguments_, timeoutMs, killGraceMs = 1e3) => {
|
|
1305
|
+
const child = spawn(process.execPath, [cli, ...arguments_], {
|
|
1306
|
+
detached: process.platform !== "win32",
|
|
1307
|
+
stdio: [
|
|
1308
|
+
"ignore",
|
|
1309
|
+
"ignore",
|
|
1310
|
+
"pipe"
|
|
1311
|
+
]
|
|
1312
|
+
});
|
|
1313
|
+
let stderr = "";
|
|
1314
|
+
child.stderr.setEncoding("utf8");
|
|
1315
|
+
child.stderr.on("data", (chunk) => {
|
|
1316
|
+
if (stderr.length < 16e3) stderr += chunk;
|
|
1317
|
+
});
|
|
1318
|
+
const signal = (name) => {
|
|
1319
|
+
if (child.pid === void 0) return;
|
|
1320
|
+
if (process.platform === "win32") {
|
|
1321
|
+
const taskkill = windowsProcessTreeKill(child.pid);
|
|
1322
|
+
const killer = spawn(taskkill.command, taskkill.arguments, {
|
|
1323
|
+
stdio: "ignore",
|
|
1324
|
+
windowsHide: true
|
|
1325
|
+
});
|
|
1326
|
+
const fallback = () => {
|
|
1327
|
+
try {
|
|
1328
|
+
child.kill("SIGKILL");
|
|
1329
|
+
} catch {}
|
|
1330
|
+
};
|
|
1331
|
+
killer.once("error", fallback);
|
|
1332
|
+
killer.once("exit", (code) => {
|
|
1333
|
+
if (code !== 0) fallback();
|
|
1334
|
+
});
|
|
1335
|
+
return;
|
|
1336
|
+
}
|
|
1337
|
+
try {
|
|
1338
|
+
process.kill(-child.pid, name);
|
|
1339
|
+
} catch {
|
|
1340
|
+
child.kill(name);
|
|
1341
|
+
}
|
|
1342
|
+
};
|
|
1343
|
+
let timedOut = false;
|
|
1344
|
+
let forceKill;
|
|
1345
|
+
let abandon;
|
|
1346
|
+
let timeout;
|
|
1347
|
+
const exitCode = await new Promise((resolveExit, reject) => {
|
|
1348
|
+
timeout = setTimeout(() => {
|
|
1349
|
+
timedOut = true;
|
|
1350
|
+
signal("SIGTERM");
|
|
1351
|
+
forceKill = setTimeout(() => signal("SIGKILL"), killGraceMs);
|
|
1352
|
+
abandon = setTimeout(() => reject(new LighthouseError({ message: `Lighthouse timed out after ${timeoutMs}ms and did not exit after SIGKILL` })), killGraceMs * 2);
|
|
1353
|
+
}, timeoutMs);
|
|
1354
|
+
child.once("error", reject);
|
|
1355
|
+
child.once("exit", (code) => resolveExit(code ?? 1));
|
|
1356
|
+
}).finally(() => {
|
|
1357
|
+
if (timeout !== void 0) clearTimeout(timeout);
|
|
1358
|
+
if (forceKill !== void 0) clearTimeout(forceKill);
|
|
1359
|
+
if (abandon !== void 0) clearTimeout(abandon);
|
|
1360
|
+
});
|
|
1361
|
+
if (timedOut) throw new LighthouseError({ message: `Lighthouse timed out after ${timeoutMs}ms` });
|
|
1362
|
+
if (exitCode !== 0) throw new LighthouseError({ message: `Lighthouse exited with ${exitCode}${stderr.trim() ? `: ${stderr.trim()}` : ""}` });
|
|
1363
|
+
};
|
|
1364
|
+
const runLighthouseInDirectory = async (options, directory) => {
|
|
1365
|
+
const version = await lighthouseVersion();
|
|
1366
|
+
const base = {
|
|
1367
|
+
requestedUrl: options.url.href,
|
|
1368
|
+
finalUrl: null,
|
|
1369
|
+
formFactor: options.formFactor,
|
|
1370
|
+
run: options.run,
|
|
1371
|
+
lighthouseVersion: version,
|
|
1372
|
+
userAgent: null,
|
|
1373
|
+
fetchTime: null,
|
|
1374
|
+
categories: {},
|
|
1375
|
+
metrics: {},
|
|
1376
|
+
opportunities: [],
|
|
1377
|
+
warnings: []
|
|
1378
|
+
};
|
|
1379
|
+
let proxy;
|
|
1380
|
+
try {
|
|
1381
|
+
const { cli } = resolveLighthouse();
|
|
1382
|
+
const output = join(directory, "result.json");
|
|
1383
|
+
proxy = await startAuditProxy({
|
|
1384
|
+
allowPrivate: options.allowPrivate,
|
|
1385
|
+
timeoutMs: options.timeoutMs ?? 18e4
|
|
1386
|
+
});
|
|
1387
|
+
const args = [
|
|
1388
|
+
options.url.href,
|
|
1389
|
+
"--output=json",
|
|
1390
|
+
`--output-path=${output}`,
|
|
1391
|
+
"--quiet",
|
|
1392
|
+
"--max-wait-for-load=45000",
|
|
1393
|
+
`--chrome-flags=${lighthouseChromeFlags(proxy.url)}`
|
|
1394
|
+
];
|
|
1395
|
+
if (options.formFactor === "desktop") args.push("--preset=desktop");
|
|
1396
|
+
await runLighthouseProcess(cli, args, options.timeoutMs ?? 18e4);
|
|
1397
|
+
return parseLighthouseResult(JSON.parse(await readFile(output, "utf8")), options.url.href, options.formFactor, options.run);
|
|
1398
|
+
} catch (error) {
|
|
1399
|
+
return {
|
|
1400
|
+
...base,
|
|
1401
|
+
error: errorMessage$1(error)
|
|
1402
|
+
};
|
|
1403
|
+
} finally {
|
|
1404
|
+
await proxy?.close();
|
|
1405
|
+
}
|
|
1406
|
+
};
|
|
1407
|
+
const runLighthouse = async (options) => {
|
|
1408
|
+
let directory;
|
|
1409
|
+
try {
|
|
1410
|
+
directory = await mkdtemp(join(tmpdir(), "tanstack-seo-lighthouse-"));
|
|
1411
|
+
return await runLighthouseInDirectory(options, directory);
|
|
1412
|
+
} finally {
|
|
1413
|
+
if (directory !== void 0) await rm(directory, {
|
|
1414
|
+
recursive: true,
|
|
1415
|
+
force: true
|
|
1416
|
+
});
|
|
1417
|
+
}
|
|
1418
|
+
};
|
|
1419
|
+
Effect.fn("SeoAudit.runLighthouse")(function* (options) {
|
|
1420
|
+
const directory = yield* Effect.acquireRelease(Effect.tryPromise({
|
|
1421
|
+
try: () => mkdtemp(join(tmpdir(), "tanstack-seo-lighthouse-")),
|
|
1422
|
+
catch: (cause) => new LighthouseError({
|
|
1423
|
+
message: errorMessage$1(cause),
|
|
1424
|
+
cause
|
|
1425
|
+
})
|
|
1426
|
+
}), (path) => Effect.promise(() => rm(path, {
|
|
1427
|
+
recursive: true,
|
|
1428
|
+
force: true
|
|
1429
|
+
})));
|
|
1430
|
+
return yield* Effect.tryPromise({
|
|
1431
|
+
try: () => runLighthouseInDirectory(options, directory),
|
|
1432
|
+
catch: (cause) => new LighthouseError({
|
|
1433
|
+
message: errorMessage$1(cause),
|
|
1434
|
+
cause
|
|
1435
|
+
})
|
|
1436
|
+
});
|
|
1437
|
+
});
|
|
1438
|
+
const makeLighthouseScanner = (options, dependencies = {}) => ({
|
|
1439
|
+
id: "lighthouse",
|
|
1440
|
+
description: "Lighthouse performance and accessibility measurements",
|
|
1441
|
+
scan: (input) => Effect.fn("SeoAudit.scanLighthouse")(function* () {
|
|
1442
|
+
const target = new URL(input.target.url);
|
|
1443
|
+
const requestedFormFactors = input.options.formFactors === void 0 || input.options.formFactors.length === 0 ? ["mobile"] : input.options.formFactors;
|
|
1444
|
+
const requestedRuns = Math.max(1, Math.floor(input.options.runs ?? 1));
|
|
1445
|
+
const runs = [...new Set(requestedFormFactors)].flatMap((formFactor) => Array.from({ length: requestedRuns }, (_, index) => ({
|
|
1446
|
+
formFactor,
|
|
1447
|
+
run: index + 1
|
|
1448
|
+
})));
|
|
1449
|
+
const observations = yield* Effect.forEach(runs, ({ formFactor, run }) => Effect.tryPromise({
|
|
1450
|
+
try: () => (dependencies.run ?? runLighthouse)({
|
|
1451
|
+
...options,
|
|
1452
|
+
url: target,
|
|
1453
|
+
formFactor,
|
|
1454
|
+
run
|
|
1455
|
+
}),
|
|
1456
|
+
catch: (cause) => new ScannerFailure({
|
|
1457
|
+
scanner: "lighthouse",
|
|
1458
|
+
target: target.href,
|
|
1459
|
+
message: errorMessage$1(cause),
|
|
1460
|
+
cause
|
|
1461
|
+
})
|
|
1462
|
+
}), { concurrency: 1 });
|
|
1463
|
+
const errors = observations.flatMap((observation) => observation.error === null ? [] : [observation.error]);
|
|
1464
|
+
if (errors.length === observations.length) return yield* new ScannerFailure({
|
|
1465
|
+
scanner: "lighthouse",
|
|
1466
|
+
target: target.href,
|
|
1467
|
+
message: errors[0] ?? "Every Lighthouse run failed."
|
|
1468
|
+
});
|
|
1469
|
+
return { evidence: observations };
|
|
1470
|
+
})()
|
|
1471
|
+
});
|
|
1472
|
+
//#endregion
|
|
1473
|
+
//#region src/audit/scanners/hosted.ts
|
|
1474
|
+
var HostedScannerError = class extends Data$1.TaggedError("HostedScannerError") {};
|
|
1475
|
+
const errorMessage = (error) => error instanceof Error ? error.message : String(error);
|
|
1476
|
+
const userAgent = "TanStackSeoAudit/1.0";
|
|
1477
|
+
const IsitagentreadyCheck = Schema$1.Struct({
|
|
1478
|
+
status: Schema$1.String,
|
|
1479
|
+
message: Schema$1.optionalKey(Schema$1.String)
|
|
1480
|
+
});
|
|
1481
|
+
const IsitagentreadyPayload = Schema$1.Struct({
|
|
1482
|
+
scannedAt: Schema$1.optionalKey(Schema$1.String),
|
|
1483
|
+
scanned_at: Schema$1.optionalKey(Schema$1.String),
|
|
1484
|
+
level: Schema$1.optionalKey(Schema$1.Finite),
|
|
1485
|
+
levelName: Schema$1.optionalKey(Schema$1.String),
|
|
1486
|
+
level_name: Schema$1.optionalKey(Schema$1.String),
|
|
1487
|
+
checks: Schema$1.optionalKey(Schema$1.Record(Schema$1.String, Schema$1.Record(Schema$1.String, IsitagentreadyCheck))),
|
|
1488
|
+
nextLevel: Schema$1.optionalKey(Schema$1.Struct({
|
|
1489
|
+
name: Schema$1.optionalKey(Schema$1.String),
|
|
1490
|
+
requirements: Schema$1.optionalKey(Schema$1.Array(Schema$1.Struct({
|
|
1491
|
+
check: Schema$1.optionalKey(Schema$1.String),
|
|
1492
|
+
description: Schema$1.optionalKey(Schema$1.String),
|
|
1493
|
+
prompt: Schema$1.optionalKey(Schema$1.String)
|
|
1494
|
+
})))
|
|
1495
|
+
})),
|
|
1496
|
+
next_level: Schema$1.optionalKey(Schema$1.Struct({
|
|
1497
|
+
name: Schema$1.optionalKey(Schema$1.String),
|
|
1498
|
+
requirements: Schema$1.optionalKey(Schema$1.Array(Schema$1.Struct({
|
|
1499
|
+
check: Schema$1.optionalKey(Schema$1.String),
|
|
1500
|
+
description: Schema$1.optionalKey(Schema$1.String),
|
|
1501
|
+
prompt: Schema$1.optionalKey(Schema$1.String)
|
|
1502
|
+
})))
|
|
1503
|
+
}))
|
|
1504
|
+
});
|
|
1505
|
+
const IsAgenticPayload = Schema$1.Struct({
|
|
1506
|
+
score: Schema$1.optionalKey(Schema$1.Finite),
|
|
1507
|
+
score_label: Schema$1.optionalKey(Schema$1.String),
|
|
1508
|
+
report_url: Schema$1.optionalKey(Schema$1.String),
|
|
1509
|
+
scanned_at: Schema$1.optionalKey(Schema$1.String),
|
|
1510
|
+
score_breakdown: Schema$1.optionalKey(Schema$1.Struct({
|
|
1511
|
+
essential: Schema$1.optionalKey(Schema$1.Struct({
|
|
1512
|
+
earned: Schema$1.Finite,
|
|
1513
|
+
available: Schema$1.Finite
|
|
1514
|
+
})),
|
|
1515
|
+
recommended: Schema$1.optionalKey(Schema$1.Struct({
|
|
1516
|
+
earned: Schema$1.Finite,
|
|
1517
|
+
available: Schema$1.Finite
|
|
1518
|
+
}))
|
|
1519
|
+
})),
|
|
1520
|
+
issues: Schema$1.optionalKey(Schema$1.Array(Schema$1.Struct({
|
|
1521
|
+
result: Schema$1.String,
|
|
1522
|
+
tier: Schema$1.optionalKey(Schema$1.String),
|
|
1523
|
+
name: Schema$1.String,
|
|
1524
|
+
details: Schema$1.optionalKey(Schema$1.String),
|
|
1525
|
+
recommendation: Schema$1.optionalKey(Schema$1.String)
|
|
1526
|
+
})))
|
|
1527
|
+
});
|
|
1528
|
+
const SseEvent = Schema$1.Struct({ type: Schema$1.String });
|
|
1529
|
+
const IsitagentreadyJson = Schema$1.fromJsonString(IsitagentreadyPayload);
|
|
1530
|
+
const IsAgenticJson = Schema$1.fromJsonString(IsAgenticPayload);
|
|
1531
|
+
const SseEventJson = Schema$1.fromJsonString(SseEvent);
|
|
1532
|
+
const decodeIsitagentready = Schema$1.decodeUnknownSync(IsitagentreadyPayload);
|
|
1533
|
+
const decodeIsAgentic = Schema$1.decodeUnknownSync(IsAgenticPayload);
|
|
1534
|
+
const decodeIsitagentreadyJson = Schema$1.decodeUnknownEffect(IsitagentreadyJson);
|
|
1535
|
+
const decodeIsAgenticJson = Schema$1.decodeUnknownEffect(IsAgenticJson);
|
|
1536
|
+
const decodeSseEventJson = Schema$1.decodeUnknownOption(SseEventJson);
|
|
1537
|
+
const enabledChecks = [
|
|
1538
|
+
"robotsTxt",
|
|
1539
|
+
"sitemap",
|
|
1540
|
+
"linkHeaders",
|
|
1541
|
+
"dnsAid",
|
|
1542
|
+
"markdownNegotiation",
|
|
1543
|
+
"robotsTxtAiRules",
|
|
1544
|
+
"contentSignals",
|
|
1545
|
+
"webBotAuth",
|
|
1546
|
+
"apiCatalog",
|
|
1547
|
+
"oauthDiscovery",
|
|
1548
|
+
"oauthProtectedResource",
|
|
1549
|
+
"authMd",
|
|
1550
|
+
"mcpServerCard",
|
|
1551
|
+
"agentSkills",
|
|
1552
|
+
"webMcp",
|
|
1553
|
+
"ard",
|
|
1554
|
+
"x402",
|
|
1555
|
+
"mpp",
|
|
1556
|
+
"ucp",
|
|
1557
|
+
"acp"
|
|
1558
|
+
];
|
|
1559
|
+
const encodeRequest = Schema$1.encodeSync(Schema$1.fromJsonString(Schema$1.Struct({
|
|
1560
|
+
url: Schema$1.String,
|
|
1561
|
+
enabledChecks: Schema$1.Array(Schema$1.Literals(enabledChecks))
|
|
1562
|
+
})));
|
|
1563
|
+
const parseSseDataFrames = (buffer) => {
|
|
1564
|
+
let rest = buffer.replaceAll("\r\n", "\n");
|
|
1565
|
+
const events = [];
|
|
1566
|
+
let boundary = rest.indexOf("\n\n");
|
|
1567
|
+
while (boundary !== -1) {
|
|
1568
|
+
const frame = rest.slice(0, boundary);
|
|
1569
|
+
rest = rest.slice(boundary + 2);
|
|
1570
|
+
const data = frame.split("\n").filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trimStart()).join("\n");
|
|
1571
|
+
if (data.length > 0) {
|
|
1572
|
+
const event = Option.getOrNull(decodeSseEventJson(data));
|
|
1573
|
+
if (event !== null) events.push(event);
|
|
1574
|
+
}
|
|
1575
|
+
boundary = rest.indexOf("\n\n");
|
|
1576
|
+
}
|
|
1577
|
+
return {
|
|
1578
|
+
events,
|
|
1579
|
+
rest
|
|
1580
|
+
};
|
|
1581
|
+
};
|
|
1582
|
+
const pinnedResponse = async (request$2) => {
|
|
1583
|
+
const addresses = await resolveTargetUrl(request$2.url, request$2.allowPrivate);
|
|
1584
|
+
const transport = request$2.url.protocol === "https:" ? request$1 : request;
|
|
1585
|
+
const hostname = networkHostname(request$2.url.hostname);
|
|
1586
|
+
const headers = {
|
|
1587
|
+
...request$2.headers,
|
|
1588
|
+
host: request$2.url.host,
|
|
1589
|
+
"user-agent": userAgent
|
|
1590
|
+
};
|
|
1591
|
+
if (request$2.body !== void 0) headers["content-length"] = Buffer.byteLength(request$2.body);
|
|
1592
|
+
return new Promise((resolve, reject) => {
|
|
1593
|
+
const outgoing = transport({
|
|
1594
|
+
hostname,
|
|
1595
|
+
port: request$2.url.port || void 0,
|
|
1596
|
+
path: `${request$2.url.pathname}${request$2.url.search}`,
|
|
1597
|
+
method: request$2.method,
|
|
1598
|
+
headers,
|
|
1599
|
+
agent: false,
|
|
1600
|
+
lookup: pinnedLookup(addresses),
|
|
1601
|
+
signal: AbortSignal.timeout(request$2.timeoutMs),
|
|
1602
|
+
...request$2.url.protocol === "https:" ? {
|
|
1603
|
+
servername: isIP(hostname) === 0 ? hostname : void 0,
|
|
1604
|
+
checkServerIdentity: (_host, certificate) => checkServerIdentity(hostname, certificate)
|
|
1605
|
+
} : void 0
|
|
1606
|
+
});
|
|
1607
|
+
outgoing.once("response", resolve);
|
|
1608
|
+
outgoing.once("error", reject);
|
|
1609
|
+
outgoing.end(request$2.body);
|
|
1610
|
+
});
|
|
1611
|
+
};
|
|
1612
|
+
const readBody = async (response, maxBodyBytes) => {
|
|
1613
|
+
const chunks = [];
|
|
1614
|
+
let length = 0;
|
|
1615
|
+
for await (const chunk of response) {
|
|
1616
|
+
const value = new Uint8Array(chunk);
|
|
1617
|
+
if (length + value.byteLength > maxBodyBytes) {
|
|
1618
|
+
response.destroy();
|
|
1619
|
+
throw new Error("Hosted scanner response exceeded byte limit");
|
|
1620
|
+
}
|
|
1621
|
+
chunks.push(value);
|
|
1622
|
+
length += value.byteLength;
|
|
1623
|
+
}
|
|
1624
|
+
const bytes = new Uint8Array(length);
|
|
1625
|
+
let offset = 0;
|
|
1626
|
+
for (const chunk of chunks) {
|
|
1627
|
+
bytes.set(chunk, offset);
|
|
1628
|
+
offset += chunk.byteLength;
|
|
1629
|
+
}
|
|
1630
|
+
const body = new TextDecoder().decode(bytes);
|
|
1631
|
+
if (body.trim().length === 0) throw new Error("Hosted scanner returned an empty body");
|
|
1632
|
+
return body;
|
|
1633
|
+
};
|
|
1634
|
+
const consumeStream = async (response, maxBodyBytes) => {
|
|
1635
|
+
let buffer = "";
|
|
1636
|
+
let length = 0;
|
|
1637
|
+
let complete = false;
|
|
1638
|
+
const decoder = new TextDecoder();
|
|
1639
|
+
try {
|
|
1640
|
+
for await (const chunk of response) {
|
|
1641
|
+
const value = new Uint8Array(chunk);
|
|
1642
|
+
length += value.byteLength;
|
|
1643
|
+
if (length > maxBodyBytes) throw new Error("Hosted scanner stream exceeded byte limit");
|
|
1644
|
+
buffer += decoder.decode(value, { stream: true });
|
|
1645
|
+
const parsed = parseSseDataFrames(buffer);
|
|
1646
|
+
buffer = parsed.rest;
|
|
1647
|
+
if (parsed.events.some((event) => event.type === "error")) throw new Error("is-agentic scan failed");
|
|
1648
|
+
if (parsed.events.some((event) => event.type === "scan_complete" || event.type === "scan_archived")) {
|
|
1649
|
+
complete = true;
|
|
1650
|
+
response.destroy();
|
|
1651
|
+
break;
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1654
|
+
} catch (cause) {
|
|
1655
|
+
if (complete) return;
|
|
1656
|
+
throw cause;
|
|
1657
|
+
}
|
|
1658
|
+
buffer += decoder.decode();
|
|
1659
|
+
if (!complete) throw new Error("is-agentic scan ended before a report was available");
|
|
1660
|
+
};
|
|
1661
|
+
const originHeader = (origin, path) => {
|
|
1662
|
+
const base = origin.replace(/\/$/, "");
|
|
1663
|
+
return {
|
|
1664
|
+
origin: base,
|
|
1665
|
+
referer: `${base}${path}`
|
|
1666
|
+
};
|
|
1667
|
+
};
|
|
1668
|
+
const complete = (parsed, requestedUrl, startedAt) => ({
|
|
1669
|
+
...parsed,
|
|
1670
|
+
requestedUrl,
|
|
1671
|
+
totalMs: Math.round((performance.now() - startedAt) * 10) / 10,
|
|
1672
|
+
error: null
|
|
1673
|
+
});
|
|
1674
|
+
const failed = (scanner, target, requestedUrl, startedAt, error) => ({
|
|
1675
|
+
scanner,
|
|
1676
|
+
target: target.href,
|
|
1677
|
+
requestedUrl,
|
|
1678
|
+
reportUrl: null,
|
|
1679
|
+
scannedAt: null,
|
|
1680
|
+
score: null,
|
|
1681
|
+
scoreLabel: null,
|
|
1682
|
+
level: null,
|
|
1683
|
+
levelName: null,
|
|
1684
|
+
summary: null,
|
|
1685
|
+
checks: [],
|
|
1686
|
+
findings: [],
|
|
1687
|
+
totalMs: Math.round((performance.now() - startedAt) * 10) / 10,
|
|
1688
|
+
error: errorMessage(error)
|
|
1689
|
+
});
|
|
1690
|
+
const parseIsitagentready = (payload, target, origin) => {
|
|
1691
|
+
const record = decodeIsitagentready(payload);
|
|
1692
|
+
const checks = Object.entries(record.checks ?? {}).flatMap(([category, group]) => Object.entries(group).map(([id, check]) => ({
|
|
1693
|
+
id,
|
|
1694
|
+
category,
|
|
1695
|
+
status: check.status,
|
|
1696
|
+
message: check.message ?? null
|
|
1697
|
+
})));
|
|
1698
|
+
const passed = checks.filter((check) => check.status === "pass").length;
|
|
1699
|
+
const failedCount = checks.filter((check) => check.status === "fail").length;
|
|
1700
|
+
const level = record.level ?? null;
|
|
1701
|
+
const levelName = record.levelName ?? record.level_name ?? null;
|
|
1702
|
+
const next = record.nextLevel ?? record.next_level;
|
|
1703
|
+
const findings = (next?.requirements ?? []).flatMap((requirement) => {
|
|
1704
|
+
const name = requirement.check ?? requirement.description;
|
|
1705
|
+
return name === void 0 ? [] : [{
|
|
1706
|
+
result: "missing",
|
|
1707
|
+
tier: next?.name ?? null,
|
|
1708
|
+
name,
|
|
1709
|
+
details: requirement.description ?? null,
|
|
1710
|
+
recommendation: requirement.prompt ?? null
|
|
1711
|
+
}];
|
|
1712
|
+
});
|
|
1713
|
+
return {
|
|
1714
|
+
scanner: "isitagentready",
|
|
1715
|
+
target: target.href,
|
|
1716
|
+
reportUrl: `${origin.replace(/\/$/, "")}/${target.hostname}`,
|
|
1717
|
+
scannedAt: record.scannedAt ?? record.scanned_at ?? null,
|
|
1718
|
+
score: null,
|
|
1719
|
+
scoreLabel: null,
|
|
1720
|
+
level,
|
|
1721
|
+
levelName,
|
|
1722
|
+
summary: level === null && levelName === null ? `${passed} pass, ${failedCount} fail` : `Level ${level ?? "?"} ${levelName ?? ""}`.trim() + `, ${passed} pass, ${failedCount} fail`,
|
|
1723
|
+
checks,
|
|
1724
|
+
findings
|
|
1725
|
+
};
|
|
1726
|
+
};
|
|
1727
|
+
const parseIsAgentic = (payload, target) => {
|
|
1728
|
+
const record = decodeIsAgentic(payload);
|
|
1729
|
+
const findings = (record.issues ?? []).map((issue) => ({
|
|
1730
|
+
result: issue.result,
|
|
1731
|
+
tier: issue.tier ?? null,
|
|
1732
|
+
name: issue.name,
|
|
1733
|
+
details: issue.details ?? null,
|
|
1734
|
+
recommendation: issue.recommendation ?? null
|
|
1735
|
+
}));
|
|
1736
|
+
const failedCount = findings.filter((finding) => finding.result === "failed").length;
|
|
1737
|
+
const partial = findings.filter((finding) => finding.result === "partial").length;
|
|
1738
|
+
const bucket = (label, value) => value === void 0 ? null : `${label} ${value.earned}/${value.available}`;
|
|
1739
|
+
return {
|
|
1740
|
+
scanner: "is-agentic",
|
|
1741
|
+
target: target.href,
|
|
1742
|
+
reportUrl: record.report_url ?? null,
|
|
1743
|
+
scannedAt: record.scanned_at ?? null,
|
|
1744
|
+
score: record.score ?? null,
|
|
1745
|
+
scoreLabel: record.score_label ?? null,
|
|
1746
|
+
level: null,
|
|
1747
|
+
levelName: null,
|
|
1748
|
+
summary: [
|
|
1749
|
+
record.score_label,
|
|
1750
|
+
bucket("Essential", record.score_breakdown?.essential),
|
|
1751
|
+
bucket("Recommended", record.score_breakdown?.recommended),
|
|
1752
|
+
`${failedCount} failed, ${partial} partial`
|
|
1753
|
+
].filter((part) => part !== null && part !== void 0 && part.length > 0).join(", "),
|
|
1754
|
+
checks: [],
|
|
1755
|
+
findings
|
|
1756
|
+
};
|
|
1757
|
+
};
|
|
1758
|
+
const scanIsAgentic = Effect.fn("SeoAudit.scanHostedIsAgentic")(function* (target, options) {
|
|
1759
|
+
const requested = new URL("/api/scan/stream", options.origins.isAgentic);
|
|
1760
|
+
requested.searchParams.set("target", target.href);
|
|
1761
|
+
requested.searchParams.set("force", "1");
|
|
1762
|
+
const headers = originHeader(options.origins.isAgentic, `/scan/${target.hostname}`);
|
|
1763
|
+
const response = yield* Effect.tryPromise({
|
|
1764
|
+
try: () => pinnedResponse({
|
|
1765
|
+
method: "GET",
|
|
1766
|
+
url: requested,
|
|
1767
|
+
headers: {
|
|
1768
|
+
accept: "text/event-stream",
|
|
1769
|
+
"cache-control": "no-cache",
|
|
1770
|
+
pragma: "no-cache",
|
|
1771
|
+
...headers
|
|
1772
|
+
},
|
|
1773
|
+
allowPrivate: options.allowPrivate,
|
|
1774
|
+
timeoutMs: options.timeoutMs
|
|
1775
|
+
}),
|
|
1776
|
+
catch: (cause) => new HostedScannerError({
|
|
1777
|
+
message: errorMessage(cause),
|
|
1778
|
+
cause
|
|
1779
|
+
})
|
|
1780
|
+
});
|
|
1781
|
+
const status = response.statusCode ?? 0;
|
|
1782
|
+
if (status < 200 || status >= 300) {
|
|
1783
|
+
const retry = response.headers["retry-after"];
|
|
1784
|
+
response.destroy();
|
|
1785
|
+
return yield* new HostedScannerError({ message: `is-agentic.com scan stream returned HTTP ${status}${typeof retry === "string" ? `, retry after ${retry}s` : ""}` });
|
|
1786
|
+
}
|
|
1787
|
+
yield* Effect.tryPromise({
|
|
1788
|
+
try: () => consumeStream(response, options.maxBodyBytes),
|
|
1789
|
+
catch: (cause) => new HostedScannerError({
|
|
1790
|
+
message: errorMessage(cause),
|
|
1791
|
+
cause
|
|
1792
|
+
})
|
|
1793
|
+
});
|
|
1794
|
+
const report = new URL("/api/v1/report", options.origins.isAgentic);
|
|
1795
|
+
report.searchParams.set("url", target.href);
|
|
1796
|
+
let lastError = new HostedScannerError({ message: "is-agentic report is not available yet" });
|
|
1797
|
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
|
1798
|
+
const reportResponse = yield* Effect.tryPromise({
|
|
1799
|
+
try: () => pinnedResponse({
|
|
1800
|
+
method: "GET",
|
|
1801
|
+
url: report,
|
|
1802
|
+
headers: { accept: "application/json" },
|
|
1803
|
+
allowPrivate: options.allowPrivate,
|
|
1804
|
+
timeoutMs: options.timeoutMs
|
|
1805
|
+
}),
|
|
1806
|
+
catch: (cause) => new HostedScannerError({
|
|
1807
|
+
message: errorMessage(cause),
|
|
1808
|
+
cause
|
|
1809
|
+
})
|
|
1810
|
+
});
|
|
1811
|
+
const reportStatus = reportResponse.statusCode ?? 0;
|
|
1812
|
+
const body = yield* Effect.tryPromise({
|
|
1813
|
+
try: () => readBody(reportResponse, options.maxBodyBytes),
|
|
1814
|
+
catch: (cause) => new HostedScannerError({
|
|
1815
|
+
message: errorMessage(cause),
|
|
1816
|
+
cause
|
|
1817
|
+
})
|
|
1818
|
+
});
|
|
1819
|
+
if (reportStatus >= 200 && reportStatus < 300) {
|
|
1820
|
+
const payload = yield* decodeIsAgenticJson(body).pipe(Effect.mapError((cause) => new HostedScannerError({
|
|
1821
|
+
message: errorMessage(cause),
|
|
1822
|
+
cause
|
|
1823
|
+
})));
|
|
1824
|
+
return {
|
|
1825
|
+
requested,
|
|
1826
|
+
parsed: parseIsAgentic(payload, target)
|
|
1827
|
+
};
|
|
1828
|
+
}
|
|
1829
|
+
lastError = new HostedScannerError({ message: `is-agentic report returned HTTP ${reportStatus}` });
|
|
1830
|
+
yield* Effect.sleep(`${250 * (attempt + 1)} millis`);
|
|
1831
|
+
}
|
|
1832
|
+
return yield* lastError;
|
|
1833
|
+
});
|
|
1834
|
+
const scanIsitagentready = Effect.fn("SeoAudit.scanHostedIsitagentready")(function* (target, options) {
|
|
1835
|
+
const requested = new URL("/api/scan", options.origins.isitagentready);
|
|
1836
|
+
const headers = originHeader(options.origins.isitagentready, `/${target.hostname}`);
|
|
1837
|
+
const response = yield* Effect.tryPromise({
|
|
1838
|
+
try: () => pinnedResponse({
|
|
1839
|
+
method: "POST",
|
|
1840
|
+
url: requested,
|
|
1841
|
+
headers: {
|
|
1842
|
+
accept: "application/json",
|
|
1843
|
+
"content-type": "application/json",
|
|
1844
|
+
...headers
|
|
1845
|
+
},
|
|
1846
|
+
body: encodeRequest({
|
|
1847
|
+
url: target.href,
|
|
1848
|
+
enabledChecks: [...enabledChecks]
|
|
1849
|
+
}),
|
|
1850
|
+
allowPrivate: options.allowPrivate,
|
|
1851
|
+
timeoutMs: options.timeoutMs
|
|
1852
|
+
}),
|
|
1853
|
+
catch: (cause) => new HostedScannerError({
|
|
1854
|
+
message: errorMessage(cause),
|
|
1855
|
+
cause
|
|
1856
|
+
})
|
|
1857
|
+
});
|
|
1858
|
+
const status = response.statusCode ?? 0;
|
|
1859
|
+
if (status < 200 || status >= 300) {
|
|
1860
|
+
response.destroy();
|
|
1861
|
+
return yield* new HostedScannerError({ message: `isitagentready.com returned HTTP ${status}` });
|
|
1862
|
+
}
|
|
1863
|
+
const body = yield* Effect.tryPromise({
|
|
1864
|
+
try: () => readBody(response, options.maxBodyBytes),
|
|
1865
|
+
catch: (cause) => new HostedScannerError({
|
|
1866
|
+
message: errorMessage(cause),
|
|
1867
|
+
cause
|
|
1868
|
+
})
|
|
1869
|
+
});
|
|
1870
|
+
const payload = yield* decodeIsitagentreadyJson(body).pipe(Effect.mapError((cause) => new HostedScannerError({
|
|
1871
|
+
message: errorMessage(cause),
|
|
1872
|
+
cause
|
|
1873
|
+
})));
|
|
1874
|
+
return {
|
|
1875
|
+
requested,
|
|
1876
|
+
parsed: parseIsitagentready(payload, target, options.origins.isitagentready)
|
|
1877
|
+
};
|
|
1878
|
+
});
|
|
1879
|
+
/** Opt-in hosted scanners. Results are attributed failures, never fatal audit failures. */
|
|
1880
|
+
const scanHosted = async (target, options) => {
|
|
1881
|
+
const output = [];
|
|
1882
|
+
const readyStartedAt = performance.now();
|
|
1883
|
+
try {
|
|
1884
|
+
const result = await Effect.runPromise(scanIsitagentready(target, options));
|
|
1885
|
+
output.push(complete(result.parsed, result.requested.href, readyStartedAt));
|
|
1886
|
+
} catch (error) {
|
|
1887
|
+
output.push(failed("isitagentready", target, new URL("/api/scan", options.origins.isitagentready).href, readyStartedAt, error));
|
|
1888
|
+
}
|
|
1889
|
+
const agenticStartedAt = performance.now();
|
|
1890
|
+
try {
|
|
1891
|
+
const result = await Effect.runPromise(scanIsAgentic(target, options));
|
|
1892
|
+
output.push(complete(result.parsed, result.requested.href, agenticStartedAt));
|
|
1893
|
+
} catch (error) {
|
|
1894
|
+
output.push(failed("is-agentic", target, new URL("/api/scan/stream", options.origins.isAgentic).href, agenticStartedAt, error));
|
|
1895
|
+
}
|
|
1896
|
+
return output;
|
|
1897
|
+
};
|
|
1898
|
+
const makeHostedScanner = (options) => ({
|
|
1899
|
+
id: "hosted",
|
|
1900
|
+
description: "Opt-in external agent-readiness reports",
|
|
1901
|
+
scan: (input) => Effect.fn("SeoAudit.scanHosted")(function* () {
|
|
1902
|
+
const target = new URL(input.target.url);
|
|
1903
|
+
if (options.allowPrivateTargets !== true) yield* Effect.tryPromise({
|
|
1904
|
+
try: () => resolveTargetUrl(target, false),
|
|
1905
|
+
catch: (cause) => new ScannerFailure({
|
|
1906
|
+
scanner: "hosted",
|
|
1907
|
+
target: target.href,
|
|
1908
|
+
message: "Hosted scanners only accept public target URLs.",
|
|
1909
|
+
cause
|
|
1910
|
+
})
|
|
1911
|
+
});
|
|
1912
|
+
const evidence = yield* Effect.tryPromise({
|
|
1913
|
+
try: () => scanHosted(target, options),
|
|
1914
|
+
catch: (cause) => new ScannerFailure({
|
|
1915
|
+
scanner: "hosted",
|
|
1916
|
+
target: target.href,
|
|
1917
|
+
message: errorMessage(cause),
|
|
1918
|
+
cause
|
|
1919
|
+
})
|
|
1920
|
+
});
|
|
1921
|
+
if (evidence.every((report) => report.error !== null)) return yield* new ScannerFailure({
|
|
1922
|
+
scanner: "hosted",
|
|
1923
|
+
target: target.href,
|
|
1924
|
+
message: evidence.map((report) => report.error).join("; ")
|
|
1925
|
+
});
|
|
1926
|
+
return { evidence };
|
|
1927
|
+
})()
|
|
1928
|
+
});
|
|
1929
|
+
//#endregion
|
|
2
1930
|
export { Audit, AuditComparisonResult, AuditDiff, AuditDiffChange, AuditDiffOutcome, AuditDiffSummary, AuditFinding, AuditLayer, AuditLive, AuditPolicy, AuditReport, FindingSeverity, InvalidTarget, ScannerFailure, ScannerRegistryLive, ScannerResult, ScannerStatus, compareAuditReports, defaultAuditOptions, evaluateRules, lighthouseChromeFlags, makeAudit, makeHostedScanner, makeHttpScanner, makeLighthouseScanner, makeScannerRegistry, pageRules, parseLighthouseResult, renderAuditDiff, renderAuditMarkdown, scanner, writeAuditFiles };
|
|
1931
|
+
|
|
1932
|
+
//# sourceMappingURL=audit.js.map
|