pagegraph 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1929 +0,0 @@
1
- import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises";
2
- import { join } from "node:path";
3
- import * as Context from "effect/Context";
4
- import * as Data$1 from "effect/Data";
5
- import * as Effect$1 from "effect/Effect";
6
- import * as Layer from "effect/Layer";
7
- import { Data, Schema } from "effect";
8
- import { createHash } from "node:crypto";
9
- import { createServer, request } from "node:http";
10
- import { request as request$1 } from "node:https";
11
- import { BlockList, connect, isIP } from "node:net";
12
- import { checkServerIdentity } from "node:tls";
13
- import { lookup } from "node:dns/promises";
14
- import { spawn } from "node:child_process";
15
- import { createRequire } from "node:module";
16
- import { tmpdir } from "node:os";
17
- import * as Option from "effect/Option";
18
- import * as Schema$1 from "effect/Schema";
19
- //#region src/audit/rules.ts
20
- const record = (value) => typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
21
- const stringAt = (value, ...keys) => {
22
- let current = value;
23
- for (const key of keys) current = record(current)?.[key];
24
- return typeof current === "string" ? current : void 0;
25
- };
26
- const firstString = (value, paths) => {
27
- for (const path of paths) {
28
- const found = stringAt(value, ...path);
29
- if (found !== void 0) return found;
30
- }
31
- };
32
- const pageEvidence = (value) => {
33
- if (!Array.isArray(value)) return value;
34
- return value.find((entry) => stringAt(entry, "kind") === "page-html") ?? value.find((entry) => stringAt(entry, "kind") === "page-head") ?? value[0];
35
- };
36
- const finding = (input, rule, message, fix, observed, severity = "structural") => ({
37
- scanner: "rules",
38
- target: input.target.url,
39
- severity,
40
- rule,
41
- message,
42
- fix,
43
- ...observed === void 0 ? {} : { observed }
44
- });
45
- /** Generic page-quality rules. They accept the normalized evidence shape used
46
- * by HTTP and hosted scanners, while remaining pure and dependency-free. */
47
- const pageRules = [
48
- {
49
- id: "http-status",
50
- evaluate: (observation, input) => {
51
- const status = record(pageEvidence(observation.evidence))?.status;
52
- return typeof status === "number" && status >= 400 ? [finding(input, "http-status", `Fetch returned HTTP ${status}.`, "Return a successful HTTP status.", status)] : [];
53
- }
54
- },
55
- {
56
- id: "title-length",
57
- evaluate: (observation, input) => {
58
- const evidence = pageEvidence(observation.evidence);
59
- const title = firstString(evidence, [
60
- ["title"],
61
- ["document", "title"],
62
- ["head", "title"]
63
- ]);
64
- const policy = input.options.policy;
65
- const min = policy?.titleMinLength ?? 10;
66
- const max = policy?.titleMaxLength ?? 70;
67
- if (title === void 0 || title.trim() === "") return [finding(input, "missing-title", "Page is missing a usable title.", "Add a descriptive <title>.")];
68
- if (title.length < min || title.length > max) return [finding(input, "title-length", `Title length ${title.length} is outside ${min}-${max} characters.`, `Keep the title between ${min} and ${max} characters.`, title, "editorial")];
69
- return [];
70
- }
71
- },
72
- {
73
- id: "description-length",
74
- evaluate: (observation, input) => {
75
- const evidence = pageEvidence(observation.evidence);
76
- const description = firstString(evidence, [
77
- ["description"],
78
- ["metaDescription"],
79
- ["document", "description"],
80
- ["head", "description"]
81
- ]);
82
- const policy = input.options.policy;
83
- const min = policy?.descriptionMinLength ?? 50;
84
- const max = policy?.descriptionMaxLength ?? 160;
85
- if (description === void 0 || description.trim() === "") return [finding(input, "missing-description", "Page is missing a meta description.", "Add a concise description that summarizes the page.")];
86
- if (description.length < min || description.length > max) return [finding(input, "description-length", `Meta description length ${description.length} is outside ${min}-${max} characters.`, `Keep the meta description between ${min} and ${max} characters.`, description, "editorial")];
87
- return [];
88
- }
89
- },
90
- {
91
- id: "canonical",
92
- evaluate: (observation, input) => {
93
- if (input.options.policy?.requireCanonical === false) return [];
94
- const canonical = firstString(pageEvidence(observation.evidence), [
95
- ["canonical"],
96
- ["canonicalUrl"],
97
- ["document", "canonical"],
98
- ["document", "canonicalUrl"],
99
- ["head", "canonical"]
100
- ]);
101
- if (canonical === void 0) return [finding(input, "missing-canonical", "Page is missing a canonical URL.", "Add a canonical link that identifies the preferred URL.")];
102
- try {
103
- const observed = new URL(canonical, input.target.url);
104
- const expected = new URL(firstString(pageEvidence(observation.evidence), [["finalUrl"]]) ?? input.target.url);
105
- observed.hash = "";
106
- expected.hash = "";
107
- return observed.href === expected.href ? [] : [finding(input, "canonical-mismatch", `Canonical points to ${observed.href} instead of the audited URL.`, "Use a self-referencing canonical unless this page is intentionally consolidated.", observed.href)];
108
- } catch {
109
- return [finding(input, "invalid-canonical", "Page canonical is not a valid URL.", "Use an absolute or resolvable canonical URL.", canonical)];
110
- }
111
- }
112
- }
113
- ];
114
- const evaluateRules = (rules, observation, input) => rules.flatMap((rule) => rule.evaluate(observation, input));
115
- /** Default policy for a web-page audit. */
116
- const defaultAuditOptions = {
117
- concurrency: 4,
118
- policy: {
119
- titleMinLength: 10,
120
- titleMaxLength: 70,
121
- descriptionMinLength: 50,
122
- descriptionMaxLength: 160,
123
- requireCanonical: true
124
- }
125
- };
126
- //#endregion
127
- //#region src/audit/scanner-registry.ts
128
- let ScannerRegistry;
129
- (function(_ScannerRegistry) {
130
- class Service extends Context.Service()("pagegraph/ScannerRegistry") {}
131
- _ScannerRegistry.Service = Service;
132
- })(ScannerRegistry || (ScannerRegistry = {}));
133
- const makeScannerRegistry = (scanners) => {
134
- const values = [...scanners];
135
- const byId = new Map(values.map((value) => [value.id, value]));
136
- return {
137
- all: () => values,
138
- get: (id) => byId.get(id)
139
- };
140
- };
141
- const ScannerRegistryLive = (scanners) => Layer.succeed(ScannerRegistry.Service, makeScannerRegistry(scanners));
142
- //#endregion
143
- //#region src/audit/audit.ts
144
- var InvalidTarget = class extends Data$1.TaggedError("InvalidTarget") {};
145
- let Audit;
146
- (function(_Audit) {
147
- class Service extends Context.Service()("pagegraph/Audit") {}
148
- _Audit.Service = Service;
149
- })(Audit || (Audit = {}));
150
- const validateTarget = (target) => Effect$1.try({
151
- try: () => {
152
- const url = new URL(target);
153
- if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("only http and https URLs are supported");
154
- if (url.username || url.password) throw new Error("credentials in URLs are not supported");
155
- return { url: url.toString() };
156
- },
157
- catch: (cause) => new InvalidTarget({
158
- target,
159
- message: cause instanceof Error ? cause.message : "invalid URL"
160
- })
161
- });
162
- const runOne = Effect$1.fn("Audit.runScanner")(function* (scanner, target, options) {
163
- const input = {
164
- target,
165
- options
166
- };
167
- return yield* scanner.scan(input).pipe(Effect$1.map((observation) => ({
168
- scanner: scanner.id,
169
- target: target.url,
170
- status: "ok",
171
- ...observation.evidence === void 0 ? {} : { evidence: observation.evidence },
172
- findings: [...observation.findings ?? [], ...evaluateRules([...scanner.rules ?? [], ...scanner.id === "http" ? pageRules : []], observation, input)]
173
- })), Effect$1.catchTag("ScannerFailure", (error) => Effect$1.succeed({
174
- scanner: scanner.id,
175
- target: target.url,
176
- status: "error",
177
- findings: [],
178
- error: error instanceof Error ? error.message : String(error)
179
- })));
180
- });
181
- const runAudit = Effect$1.fn("Audit.run")(function* (registry, request) {
182
- const targets = yield* Effect$1.forEach(request.targets, validateTarget);
183
- const options = {
184
- ...defaultOptions,
185
- ...request.options ?? {},
186
- policy: {
187
- ...defaultOptions.policy,
188
- ...request.options?.policy ?? {}
189
- }
190
- };
191
- const available = registry.all();
192
- const scanners = request.scanners === void 0 ? available : available.filter((scanner) => request.scanners.includes(scanner.id));
193
- const pairs = targets.flatMap((target) => scanners.map((scanner) => ({
194
- target,
195
- scanner
196
- })));
197
- const results = yield* Effect$1.forEach(pairs, ({ target, scanner }) => runOne(scanner, target, options), { concurrency: Math.max(1, Math.floor(options.concurrency ?? 4)) });
198
- const findings = results.flatMap((result) => result.findings);
199
- const warnings = results.filter((result) => result.status === "error").map((result) => `${result.scanner} failed for ${result.target}: ${result.error ?? "unknown error"}`);
200
- return {
201
- schemaVersion: 1,
202
- generatedAt: yield* Effect$1.clockWith((clock) => Effect$1.map(clock.currentTimeMillis, (millis) => new Date(millis).toISOString())),
203
- targets: targets.map(({ url }) => url),
204
- results,
205
- findings,
206
- warnings
207
- };
208
- });
209
- const defaultOptions = {
210
- concurrency: 4,
211
- policy: {
212
- titleMinLength: 10,
213
- titleMaxLength: 70,
214
- descriptionMinLength: 50,
215
- descriptionMaxLength: 160,
216
- requireCanonical: true
217
- }
218
- };
219
- const makeAudit = (registry) => ({ run: (request) => runAudit(registry, request) });
220
- const AuditLive = Layer.effect(Audit.Service, Effect$1.map(ScannerRegistry.Service, makeAudit));
221
- //#endregion
222
- //#region src/audit/model.ts
223
- const FindingSeverity = Schema.Literals(["structural", "editorial"]);
224
- /** A machine-readable audit finding. Values are intentionally open-ended so
225
- * scanner authors can retain the evidence that led to a finding. */
226
- const AuditFinding = Schema.Struct({
227
- scanner: Schema.String,
228
- target: Schema.String,
229
- severity: FindingSeverity,
230
- rule: Schema.String,
231
- message: Schema.String,
232
- fix: Schema.optionalKey(Schema.String),
233
- observed: Schema.optionalKey(Schema.Unknown),
234
- expected: Schema.optionalKey(Schema.Unknown)
235
- });
236
- const ScannerStatus = Schema.Literals(["ok", "error"]);
237
- /** The result of one scanner against one target. Scanner failures are data in
238
- * a report rather than an un-attributed failure of the entire audit. */
239
- const ScannerResult = Schema.Struct({
240
- scanner: Schema.String,
241
- target: Schema.String,
242
- status: ScannerStatus,
243
- evidence: Schema.optionalKey(Schema.Unknown),
244
- findings: Schema.Array(AuditFinding),
245
- error: Schema.optionalKey(Schema.String)
246
- });
247
- const AuditPolicy = Schema.Struct({
248
- titleMinLength: Schema.optionalKey(Schema.Number),
249
- titleMaxLength: Schema.optionalKey(Schema.Number),
250
- descriptionMinLength: Schema.optionalKey(Schema.Number),
251
- descriptionMaxLength: Schema.optionalKey(Schema.Number),
252
- requireCanonical: Schema.optionalKey(Schema.Boolean)
253
- });
254
- /** JSON/Markdown report contract for `pagegraph audit`. */
255
- const AuditReport = Schema.Struct({
256
- schemaVersion: Schema.Literal(1),
257
- generatedAt: Schema.String,
258
- targets: Schema.Array(Schema.String),
259
- results: Schema.Array(ScannerResult),
260
- findings: Schema.Array(AuditFinding),
261
- warnings: Schema.Array(Schema.String)
262
- });
263
- //#endregion
264
- //#region src/audit/diff.ts
265
- const DiffFindingGroup = Schema.Struct({
266
- target: Schema.String,
267
- scanner: Schema.String,
268
- rule: Schema.String,
269
- severity: FindingSeverity,
270
- findings: Schema.Array(AuditFinding)
271
- });
272
- const FindingAdded = Schema.TaggedStruct("FindingAdded", { group: DiffFindingGroup });
273
- const FindingRemoved = Schema.TaggedStruct("FindingRemoved", { group: DiffFindingGroup });
274
- const FindingChanged = Schema.TaggedStruct("FindingChanged", {
275
- target: Schema.String,
276
- scanner: Schema.String,
277
- rule: Schema.String,
278
- beforeSeverity: FindingSeverity,
279
- afterSeverity: FindingSeverity,
280
- before: Schema.Array(AuditFinding),
281
- after: Schema.Array(AuditFinding)
282
- });
283
- const ScannerDegraded = Schema.TaggedStruct("ScannerDegraded", {
284
- target: Schema.String,
285
- scanner: Schema.String
286
- });
287
- const ScannerRecovered = Schema.TaggedStruct("ScannerRecovered", {
288
- target: Schema.String,
289
- scanner: Schema.String
290
- });
291
- const TargetAdded = Schema.TaggedStruct("TargetAdded", { target: Schema.String });
292
- const TargetRemoved = Schema.TaggedStruct("TargetRemoved", { target: Schema.String });
293
- const ScannerAdded = Schema.TaggedStruct("ScannerAdded", {
294
- target: Schema.String,
295
- scanner: Schema.String,
296
- status: ScannerStatus
297
- });
298
- const ScannerRemoved = Schema.TaggedStruct("ScannerRemoved", {
299
- target: Schema.String,
300
- scanner: Schema.String
301
- });
302
- /** One normalized semantic change between two audit reports. */
303
- const AuditDiffChange = Schema.Union([
304
- FindingAdded,
305
- FindingRemoved,
306
- FindingChanged,
307
- ScannerDegraded,
308
- ScannerRecovered,
309
- TargetAdded,
310
- TargetRemoved,
311
- ScannerAdded,
312
- ScannerRemoved
313
- ]).pipe(Schema.toTaggedUnion("_tag"));
314
- const AuditDiffOutcome = Schema.Literals([
315
- "unchanged",
316
- "changed",
317
- "regressed"
318
- ]);
319
- const AuditDiffSummary = Schema.Struct({
320
- structuralRegressions: Schema.Number,
321
- editorialRegressions: Schema.Number,
322
- scannerRegressions: Schema.Number,
323
- coverageRegressions: Schema.Number,
324
- improvements: Schema.Number,
325
- informationalChanges: Schema.Number
326
- });
327
- const AuditDiffSource = Schema.Struct({
328
- reportSchemaVersion: Schema.Literal(1),
329
- generatedAt: Schema.String,
330
- targets: Schema.Array(Schema.String)
331
- });
332
- /** Versioned machine-readable contract emitted by `pagegraph diff --json`. */
333
- const AuditDiff = Schema.Struct({
334
- kind: Schema.Literal("audit-diff"),
335
- schemaVersion: Schema.Literal(1),
336
- before: AuditDiffSource,
337
- after: AuditDiffSource,
338
- outcome: AuditDiffOutcome,
339
- summary: AuditDiffSummary,
340
- changes: Schema.Array(AuditDiffChange),
341
- ignored: Schema.Tuple([
342
- Schema.Literal("generatedAt"),
343
- Schema.Literal("warnings"),
344
- Schema.Literal("results[].evidence")
345
- ])
346
- });
347
- const AuditComparisonResult = Data.taggedEnum();
348
- const compareStrings = (left, right) => left < right ? -1 : left > right ? 1 : 0;
349
- const canonicalize = (value) => {
350
- if (Array.isArray(value)) return value.map(canonicalize);
351
- if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).sort(([left], [right]) => compareStrings(left, right)).map(([key, entry]) => [key, canonicalize(entry)]));
352
- return value;
353
- };
354
- const canonicalJson = (value) => JSON.stringify(canonicalize(value));
355
- const tupleKey = (...parts) => JSON.stringify(parts);
356
- const severityOf = (findings) => findings.some((finding) => finding.severity === "structural") ? "structural" : "editorial";
357
- const findingsByIdentity = (findings) => {
358
- const groups = /* @__PURE__ */ new Map();
359
- for (const finding of findings) {
360
- const key = tupleKey(finding.target, finding.scanner, finding.rule);
361
- const group = groups.get(key);
362
- if (group) group.push(finding);
363
- else groups.set(key, [finding]);
364
- }
365
- return new Map([...groups.entries()].map(([key, group]) => {
366
- const sorted = group.toSorted((left, right) => compareStrings(canonicalJson(left), canonicalJson(right)));
367
- const first = sorted[0];
368
- return [key, {
369
- target: first.target,
370
- scanner: first.scanner,
371
- rule: first.rule,
372
- severity: severityOf(sorted),
373
- findings: sorted
374
- }];
375
- }));
376
- };
377
- const validateReport = (label, report) => {
378
- const issues = [];
379
- const targets = /* @__PURE__ */ new Set();
380
- for (const target of report.targets) {
381
- if (targets.has(target)) issues.push(`${label}: duplicate target ${target}`);
382
- targets.add(target);
383
- }
384
- const results = /* @__PURE__ */ new Set();
385
- for (const result of report.results) {
386
- if (!targets.has(result.target)) issues.push(`${label}: scanner ${result.scanner} references undeclared target ${result.target}`);
387
- const key = tupleKey(result.target, result.scanner);
388
- if (results.has(key)) issues.push(`${label}: duplicate scanner result ${result.scanner} for ${result.target}`);
389
- results.add(key);
390
- for (const finding of result.findings) if (finding.target !== result.target) issues.push(`${label}: finding ${finding.rule} targets ${finding.target} inside result for ${result.target}`);
391
- }
392
- const flattened = report.results.flatMap((result) => result.findings).map(canonicalJson).sort(compareStrings);
393
- const topLevel = report.findings.map(canonicalJson).sort(compareStrings);
394
- if (canonicalJson(flattened) !== canonicalJson(topLevel)) issues.push(`${label}: top-level findings do not match result findings`);
395
- return issues;
396
- };
397
- const resultMap = (report) => new Map(report.results.map((result) => [tupleKey(result.target, result.scanner), result]));
398
- const changeSortKey = (change) => {
399
- switch (change._tag) {
400
- case "FindingAdded":
401
- case "FindingRemoved": return tupleKey(change.group.target, change.group.scanner, change.group.rule, change._tag);
402
- case "FindingChanged": return tupleKey(change.target, change.scanner, change.rule, change._tag);
403
- case "ScannerAdded":
404
- case "ScannerRemoved":
405
- case "ScannerDegraded":
406
- case "ScannerRecovered": return tupleKey(change.target, change.scanner, "", change._tag);
407
- case "TargetAdded":
408
- case "TargetRemoved": return tupleKey(change.target, "", "", change._tag);
409
- }
410
- };
411
- const summarize = (changes) => {
412
- let structuralRegressions = 0;
413
- let editorialRegressions = 0;
414
- let scannerRegressions = 0;
415
- let coverageRegressions = 0;
416
- let improvements = 0;
417
- let informationalChanges = 0;
418
- for (const change of changes) switch (change._tag) {
419
- case "FindingAdded":
420
- if (change.group.severity === "structural") structuralRegressions++;
421
- else editorialRegressions++;
422
- break;
423
- case "FindingRemoved":
424
- case "ScannerRecovered":
425
- improvements++;
426
- break;
427
- case "FindingChanged":
428
- if (change.beforeSeverity === "editorial" && change.afterSeverity === "structural") structuralRegressions++;
429
- else if (change.beforeSeverity === "structural" && change.afterSeverity === "editorial") improvements++;
430
- else informationalChanges++;
431
- break;
432
- case "ScannerDegraded":
433
- scannerRegressions++;
434
- break;
435
- case "TargetRemoved":
436
- case "ScannerRemoved":
437
- coverageRegressions++;
438
- break;
439
- case "TargetAdded":
440
- informationalChanges++;
441
- break;
442
- case "ScannerAdded": if (change.status === "error") scannerRegressions++;
443
- else informationalChanges++;
444
- }
445
- return {
446
- structuralRegressions,
447
- editorialRegressions,
448
- scannerRegressions,
449
- coverageRegressions,
450
- improvements,
451
- informationalChanges
452
- };
453
- };
454
- const outcomeOf = (summary, changes) => {
455
- if (summary.structuralRegressions > 0 || summary.scannerRegressions > 0 || summary.coverageRegressions > 0) return "regressed";
456
- return changes.length > 0 ? "changed" : "unchanged";
457
- };
458
- /** Compare two decoded audit reports while ignoring non-semantic scanner evidence. */
459
- const compareAuditReports = (before, after) => {
460
- const issues = [...validateReport("before", before), ...validateReport("after", after)];
461
- if (issues.length > 0) return AuditComparisonResult.InvalidReport({ issues });
462
- const changes = [];
463
- const beforeTargets = new Set(before.targets);
464
- const afterTargets = new Set(after.targets);
465
- for (const target of beforeTargets) if (!afterTargets.has(target)) changes.push({
466
- _tag: "TargetRemoved",
467
- target
468
- });
469
- for (const target of afterTargets) if (!beforeTargets.has(target)) changes.push({
470
- _tag: "TargetAdded",
471
- target
472
- });
473
- const beforeResults = resultMap(before);
474
- const afterResults = resultMap(after);
475
- for (const [key, result] of beforeResults) {
476
- const next = afterResults.get(key);
477
- if (next === void 0) changes.push({
478
- _tag: "ScannerRemoved",
479
- target: result.target,
480
- scanner: result.scanner
481
- });
482
- else if (result.status === "ok" && next.status === "error") changes.push({
483
- _tag: "ScannerDegraded",
484
- target: result.target,
485
- scanner: result.scanner
486
- });
487
- else if (result.status === "error" && next.status === "ok") changes.push({
488
- _tag: "ScannerRecovered",
489
- target: result.target,
490
- scanner: result.scanner
491
- });
492
- }
493
- for (const [key, result] of afterResults) if (!beforeResults.has(key)) changes.push({
494
- _tag: "ScannerAdded",
495
- target: result.target,
496
- scanner: result.scanner,
497
- status: result.status
498
- });
499
- const beforeFindings = findingsByIdentity(before.findings);
500
- const afterFindings = findingsByIdentity(after.findings);
501
- for (const [key, group] of beforeFindings) {
502
- const next = afterFindings.get(key);
503
- if (next === void 0) changes.push({
504
- _tag: "FindingRemoved",
505
- group
506
- });
507
- else if (canonicalJson(group.findings) !== canonicalJson(next.findings)) changes.push({
508
- _tag: "FindingChanged",
509
- target: group.target,
510
- scanner: group.scanner,
511
- rule: group.rule,
512
- beforeSeverity: group.severity,
513
- afterSeverity: next.severity,
514
- before: group.findings,
515
- after: next.findings
516
- });
517
- }
518
- for (const [key, group] of afterFindings) if (!beforeFindings.has(key)) changes.push({
519
- _tag: "FindingAdded",
520
- group
521
- });
522
- changes.sort((left, right) => compareStrings(changeSortKey(left), changeSortKey(right)));
523
- const summary = summarize(changes);
524
- const targets = (report) => report.targets.toSorted(compareStrings);
525
- return AuditComparisonResult.Success({ diff: {
526
- kind: "audit-diff",
527
- schemaVersion: 1,
528
- before: {
529
- reportSchemaVersion: 1,
530
- generatedAt: before.generatedAt,
531
- targets: targets(before)
532
- },
533
- after: {
534
- reportSchemaVersion: 1,
535
- generatedAt: after.generatedAt,
536
- targets: targets(after)
537
- },
538
- outcome: outcomeOf(summary, changes),
539
- summary,
540
- changes,
541
- ignored: [
542
- "generatedAt",
543
- "warnings",
544
- "results[].evidence"
545
- ]
546
- } });
547
- };
548
- //#endregion
549
- //#region src/audit/layers.ts
550
- /** Compose the default audit services with an explicit scanner set. */
551
- const AuditLayer = (scanners) => AuditLive.pipe(Layer.provide(ScannerRegistryLive(scanners)));
552
- //#endregion
553
- //#region src/audit/scanner.ts
554
- /** A typed, attributed failure from a scanner. The audit service catches this
555
- * at the scanner boundary and preserves it in the report. */
556
- var ScannerFailure = class extends Data$1.TaggedError("ScannerFailure") {};
557
- const scanner = (definition) => definition;
558
- //#endregion
559
- //#region src/audit/render.ts
560
- const cell = (value) => String(value ?? "—").replaceAll("|", "\\|").replaceAll("\n", " ");
561
- const renderAuditMarkdown = (report) => {
562
- const structural = report.findings.filter((finding) => finding.severity === "structural");
563
- const editorial = report.findings.filter((finding) => finding.severity === "editorial");
564
- const lines = [
565
- "# SEO audit",
566
- "",
567
- `Generated: ${report.generatedAt}`,
568
- `Targets: ${report.targets.length} · Structural: ${structural.length} · Editorial: ${editorial.length}`,
569
- "",
570
- "## Scanner results",
571
- "",
572
- "| Scanner | Target | Status | Findings |",
573
- "| --- | --- | --- | ---: |",
574
- ...report.results.map((result) => `| ${cell(result.scanner)} | ${cell(result.target)} | ${cell(result.status)} | ${result.findings.length} |`)
575
- ];
576
- if (report.findings.length > 0) lines.push("", "## Findings", "", "| Severity | Rule | Target | Finding |", "| --- | --- | --- | --- |", ...report.findings.map((finding) => `| ${cell(finding.severity)} | ${cell(finding.rule)} | ${cell(finding.target)} | ${cell(finding.message)} |`));
577
- if (report.warnings.length > 0) lines.push("", "## Warnings", "", ...report.warnings.map((warning) => `- ${warning}`));
578
- return `${lines.join("\n")}\n`;
579
- };
580
- const renderDiffChange = (change) => {
581
- switch (change._tag) {
582
- case "FindingAdded": return `+ [${change.group.severity}] ${change.group.rule} · ${change.group.target} (${change.group.scanner})`;
583
- case "FindingRemoved": return `- [${change.group.severity}] ${change.group.rule} · ${change.group.target} (${change.group.scanner})`;
584
- case "FindingChanged": return `~ [${change.beforeSeverity} → ${change.afterSeverity}] ${change.rule} · ${change.target} (${change.scanner})`;
585
- case "ScannerDegraded": return `✗ scanner ${change.scanner} degraded · ${change.target}`;
586
- case "ScannerRecovered": return `✓ scanner ${change.scanner} recovered · ${change.target}`;
587
- case "TargetAdded": return `+ target coverage · ${change.target}`;
588
- case "TargetRemoved": return `- target coverage · ${change.target}`;
589
- case "ScannerAdded": return `+ scanner coverage ${change.scanner} (${change.status}) · ${change.target}`;
590
- case "ScannerRemoved": return `- scanner coverage ${change.scanner} · ${change.target}`;
591
- }
592
- };
593
- const diffSection = (title, changes) => changes.length === 0 ? [] : [
594
- "",
595
- `## ${title}`,
596
- "",
597
- ...changes.map(renderDiffChange)
598
- ];
599
- /** Human-readable rendering of the semantic comparison contract. */
600
- const renderAuditDiff = (diff) => {
601
- const regressions = [];
602
- const editorial = [];
603
- const improvements = [];
604
- const informational = [];
605
- for (const change of diff.changes) switch (change._tag) {
606
- case "FindingAdded":
607
- if (change.group.severity === "structural") regressions.push(change);
608
- else editorial.push(change);
609
- break;
610
- case "FindingChanged":
611
- if (change.beforeSeverity === "editorial" && change.afterSeverity === "structural") regressions.push(change);
612
- else if (change.beforeSeverity === "structural" && change.afterSeverity === "editorial") improvements.push(change);
613
- else informational.push(change);
614
- break;
615
- case "ScannerDegraded":
616
- case "TargetRemoved":
617
- case "ScannerRemoved":
618
- regressions.push(change);
619
- break;
620
- case "FindingRemoved":
621
- case "ScannerRecovered":
622
- improvements.push(change);
623
- break;
624
- case "TargetAdded":
625
- informational.push(change);
626
- break;
627
- case "ScannerAdded": if (change.status === "error") regressions.push(change);
628
- else informational.push(change);
629
- }
630
- const lines = [
631
- "# SEO audit diff",
632
- "",
633
- `${diff.outcome === "regressed" ? "✗" : diff.outcome === "changed" ? "!" : "✓"} Outcome: ${diff.outcome}`,
634
- `Before: ${diff.before.generatedAt} · ${diff.before.targets.length} target(s)`,
635
- `After: ${diff.after.generatedAt} · ${diff.after.targets.length} target(s)`
636
- ];
637
- if (diff.changes.length === 0) lines.push("", "No semantic changes. Volatile scanner evidence was ignored.");
638
- else lines.push(...diffSection("Regressions", regressions), ...diffSection("Editorial changes", editorial), ...diffSection("Improvements", improvements), ...diffSection("Other changes", informational));
639
- lines.push("", `Ignored: ${diff.ignored.join(", ")}`, "", `Summary: ${diff.summary.structuralRegressions} structural · ${diff.summary.editorialRegressions} editorial · ${diff.summary.scannerRegressions} scanner · ${diff.summary.coverageRegressions} coverage regression(s)`);
640
- return `${lines.join("\n")}\n`;
641
- };
642
- const atomicWrite = async (path, contents) => {
643
- const temporary = `${path}.${process.pid}.${crypto.randomUUID()}.tmp`;
644
- await writeFile(temporary, contents, "utf8");
645
- await rename(temporary, path);
646
- };
647
- const writeAuditFiles = async (report, outputDirectory) => {
648
- await mkdir(outputDirectory, { recursive: true });
649
- const stamp = report.generatedAt.replaceAll(":", "-");
650
- const json = join(outputDirectory, `${stamp}.json`);
651
- const markdown = join(outputDirectory, `${stamp}.md`);
652
- await Promise.all([atomicWrite(json, `${JSON.stringify(report, null, 2)}\n`), atomicWrite(markdown, renderAuditMarkdown(report))]);
653
- return {
654
- json,
655
- markdown
656
- };
657
- };
658
- //#endregion
659
- //#region src/audit/network.ts
660
- const blockedIpv4Addresses = new BlockList();
661
- blockedIpv4Addresses.addAddress("168.63.129.16", "ipv4");
662
- for (const [network, prefix] of [
663
- ["0.0.0.0", 8],
664
- ["10.0.0.0", 8],
665
- ["100.64.0.0", 10],
666
- ["127.0.0.0", 8],
667
- ["169.254.0.0", 16],
668
- ["172.16.0.0", 12],
669
- ["192.0.0.0", 24],
670
- ["192.0.2.0", 24],
671
- ["192.88.99.0", 24],
672
- ["192.168.0.0", 16],
673
- ["198.18.0.0", 15],
674
- ["198.51.100.0", 24],
675
- ["203.0.113.0", 24],
676
- ["224.0.0.0", 4],
677
- ["240.0.0.0", 4]
678
- ]) blockedIpv4Addresses.addSubnet(network, prefix, "ipv4");
679
- const ipv4Groups = (address) => {
680
- const octets = address.split(".").map(Number);
681
- if (octets.length !== 4 || octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255)) throw new Error(`Invalid embedded IPv4 address: ${address}`);
682
- return [octets[0] << 8 | octets[1], octets[2] << 8 | octets[3]];
683
- };
684
- const ipv6Value = (address) => {
685
- const [head = "", tail, extra] = address.toLowerCase().split("::");
686
- if (extra !== void 0) throw new Error(`Invalid IPv6 address: ${address}`);
687
- const parseGroups = (side) => side === "" ? [] : side.split(":").flatMap((group) => group.includes(".") ? ipv4Groups(group) : [Number.parseInt(group, 16)]);
688
- const left = parseGroups(head);
689
- const right = parseGroups(tail ?? "");
690
- if (left.some((group) => !Number.isInteger(group) || group < 0 || group > 65535)) throw new Error(`Invalid IPv6 address: ${address}`);
691
- const missing = 8 - left.length - right.length;
692
- if (tail === void 0 && missing !== 0 || missing < 0) throw new Error(`Invalid IPv6 address: ${address}`);
693
- return [
694
- ...left,
695
- ...Array(missing).fill(0),
696
- ...right
697
- ].reduce((value, group) => value << 16n | BigInt(group), 0n);
698
- };
699
- const blockedIpv6Ranges = [
700
- ["::", 128],
701
- ["::1", 128],
702
- ["::", 96],
703
- ["::ffff:0:0", 96],
704
- ["::ffff:0:0:0", 96],
705
- ["64:ff9b::", 96],
706
- ["64:ff9b:1::", 48],
707
- ["100::", 64],
708
- ["2001::", 23],
709
- ["2001:db8::", 32],
710
- ["2002::", 16],
711
- ["3fff::", 20],
712
- ["5f00::", 16],
713
- ["fc00::", 7],
714
- ["fe80::", 10],
715
- ["ff00::", 8]
716
- ];
717
- const isBlockedIpv6 = (address) => {
718
- try {
719
- const value = ipv6Value(address);
720
- return blockedIpv6Ranges.some(([network, prefix]) => {
721
- const shift = BigInt(128 - prefix);
722
- return value >> shift === ipv6Value(network) >> shift;
723
- });
724
- } catch {
725
- return true;
726
- }
727
- };
728
- const networkHostname = (hostname) => hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
729
- const isPrivateAddress = (address) => {
730
- const family = isIP(address);
731
- if (family === 4) return blockedIpv4Addresses.check(address, "ipv4");
732
- if (family === 6) return isBlockedIpv6(address);
733
- return true;
734
- };
735
- const resolveHostname = async (hostname) => {
736
- const normalized = networkHostname(hostname);
737
- const literalFamily = isIP(normalized);
738
- if (literalFamily === 4 || literalFamily === 6) return [{
739
- address: normalized,
740
- family: literalFamily
741
- }];
742
- return (await lookup(normalized, {
743
- all: true,
744
- verbatim: true
745
- })).flatMap(({ address, family }) => family === 4 || family === 6 ? [{
746
- address,
747
- family
748
- }] : []);
749
- };
750
- /** Resolve every DNS answer before connecting. A host is rejected if any answer is private. */
751
- const resolveTargetUrl = async (url, allowPrivate, resolve = resolveHostname) => {
752
- if (url.protocol !== "https:" && url.protocol !== "http:") throw new Error(`Unsupported URL protocol: ${url.protocol}`);
753
- if (url.username || url.password) throw new Error("Target URLs must not contain credentials");
754
- const hostname = networkHostname(url.hostname).toLowerCase();
755
- if (!allowPrivate && (hostname === "localhost" || hostname.endsWith(".local"))) throw new Error(`Private target is not allowed: ${hostname}`);
756
- const addresses = await resolve(hostname);
757
- if (addresses.length === 0 || addresses.some(({ address, family }) => isIP(address) !== family) || !allowPrivate && addresses.some(({ address }) => isPrivateAddress(address))) throw new Error(`Private target is not allowed: ${hostname}`);
758
- return addresses;
759
- };
760
- /** A lookup callback pinned to the already validated DNS answers. */
761
- const pinnedLookup = (addresses) => {
762
- const [first] = addresses;
763
- if (first === void 0) throw new Error("Cannot create a pinned lookup without an address");
764
- return (_hostname, options, callback) => {
765
- if (options.all) {
766
- callback(null, addresses.map(({ address, family }) => ({
767
- address,
768
- family
769
- })));
770
- return;
771
- }
772
- callback(null, first.address, first.family);
773
- };
774
- };
775
- //#endregion
776
- //#region src/audit/scanners/http.ts
777
- var HttpProbeError = class extends Data$1.TaggedError("HttpProbeError") {};
778
- const errorMessage$2 = (error) => error instanceof Error ? error.message : String(error);
779
- const discoveryPaths = [
780
- {
781
- kind: "robots",
782
- path: "/robots.txt",
783
- accept: "text/plain, */*;q=0.1"
784
- },
785
- {
786
- kind: "sitemap",
787
- path: "/sitemap.xml",
788
- accept: "application/xml, text/xml;q=0.9, */*;q=0.1"
789
- },
790
- {
791
- kind: "llms",
792
- path: "/llms.txt",
793
- accept: "text/plain, text/markdown;q=0.9, */*;q=0.1"
794
- }
795
- ];
796
- const selectedHeaders = [
797
- "cache-control",
798
- "cf-cache-status",
799
- "content-language",
800
- "content-length",
801
- "content-signal",
802
- "content-type",
803
- "link",
804
- "server",
805
- "vary",
806
- "x-robots-tag"
807
- ];
808
- const buildProbeRequests = (targets, extraDiscovery = []) => {
809
- const requests = [];
810
- const seenDiscoveryOrigins = /* @__PURE__ */ new Set();
811
- for (const target of targets) {
812
- requests.push({
813
- kind: "page-head",
814
- method: "HEAD",
815
- accept: "text/html, */*;q=0.1",
816
- url: target
817
- }, {
818
- kind: "page-html",
819
- method: "GET",
820
- accept: "text/html, */*;q=0.1",
821
- url: target
822
- }, {
823
- kind: "page-markdown",
824
- method: "GET",
825
- accept: "text/markdown, text/plain;q=0.8, */*;q=0.1",
826
- url: target
827
- });
828
- if (seenDiscoveryOrigins.has(target.origin)) continue;
829
- seenDiscoveryOrigins.add(target.origin);
830
- for (const discovery of [...discoveryPaths, ...extraDiscovery]) requests.push({
831
- kind: discovery.kind,
832
- method: "GET",
833
- accept: discovery.accept,
834
- url: new URL(discovery.path, target.origin)
835
- });
836
- }
837
- return requests;
838
- };
839
- const responseHeaders = (headers) => {
840
- const values = {};
841
- for (const header of selectedHeaders) {
842
- const value = headers[header];
843
- if (value !== void 0) values[header] = Array.isArray(value) ? value.join(", ") : value;
844
- }
845
- return values;
846
- };
847
- const attribute = (tag, name) => {
848
- const match = new RegExp(`${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i").exec(tag);
849
- return match?.[1] ?? match?.[2] ?? match?.[3] ?? null;
850
- };
851
- const metaContent = (html, key) => {
852
- for (const match of html.matchAll(/<meta\b[^>]*>/gi)) {
853
- const tag = match[0];
854
- if (attribute(tag, "name")?.toLowerCase() === key) return attribute(tag, "content");
855
- }
856
- return null;
857
- };
858
- const canonicalUrl = (html) => {
859
- for (const match of html.matchAll(/<link\b[^>]*>/gi)) {
860
- const tag = match[0];
861
- if ((attribute(tag, "rel")?.toLowerCase().split(/\s+/) ?? []).includes("canonical")) return attribute(tag, "href");
862
- }
863
- return null;
864
- };
865
- const countMatches = (value, pattern) => [...value.matchAll(pattern)].length;
866
- const extractDocumentSignals = (body, contentType, requestedUrl) => {
867
- const isHtml = contentType.includes("text/html");
868
- const isMarkdown = contentType.includes("text/markdown");
869
- const looksJson = contentType.includes("json") || requestedUrl.pathname.endsWith(".json");
870
- const titleMatch = isHtml ? /<title\b[^>]*>([\s\S]*?)<\/title>/i.exec(body) : null;
871
- let jsonValid = null;
872
- if (looksJson && body.trim().length > 0) try {
873
- JSON.parse(body);
874
- jsonValid = true;
875
- } catch {
876
- jsonValid = false;
877
- }
878
- return {
879
- title: titleMatch?.[1]?.replace(/\s+/g, " ").trim() ?? null,
880
- description: isHtml ? metaContent(body, "description") : null,
881
- canonicalUrl: isHtml ? canonicalUrl(body) : null,
882
- robots: isHtml ? metaContent(body, "robots") : null,
883
- jsonLdCount: isHtml ? countMatches(body, /<script\b[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>/gi) : 0,
884
- modulePreloadCount: isHtml ? countMatches(body, /<link\b[^>]*rel\s*=\s*["'][^"']*modulepreload[^"']*["'][^>]*>/gi) : 0,
885
- scriptCount: isHtml ? countMatches(body, /<script\b/gi) : 0,
886
- stylesheetCount: isHtml ? countMatches(body, /<link\b[^>]*rel\s*=\s*["'][^"']*stylesheet[^"']*["'][^>]*>/gi) : 0,
887
- wordCount: isMarkdown ? body.trim().split(/\s+/).filter(Boolean).length : null,
888
- jsonValid
889
- };
890
- };
891
- const readBoundedBody = async (response, maxBodyBytes) => {
892
- const chunks = [];
893
- let length = 0;
894
- let truncated = false;
895
- for await (const chunk of response) {
896
- const value = new Uint8Array(chunk);
897
- const remaining = maxBodyBytes - length;
898
- if (value.byteLength > remaining) {
899
- if (remaining > 0) chunks.push(value.slice(0, remaining));
900
- length += Math.max(remaining, 0);
901
- truncated = true;
902
- response.destroy();
903
- break;
904
- }
905
- chunks.push(value);
906
- length += value.byteLength;
907
- }
908
- const bytes = new Uint8Array(length);
909
- let offset = 0;
910
- for (const chunk of chunks) {
911
- bytes.set(chunk, offset);
912
- offset += chunk.byteLength;
913
- }
914
- return {
915
- bytes,
916
- truncated
917
- };
918
- };
919
- const bodyExcerpt = (body, contentType) => {
920
- if (contentType.includes("text/html") || body.length === 0) return null;
921
- return [...body].filter((character) => {
922
- const code = character.charCodeAt(0);
923
- return code === 9 || code === 10 || code === 13 || code >= 32;
924
- }).join("").slice(0, 600);
925
- };
926
- const requestPinned = async (url, request$4, options) => {
927
- const addresses = await resolveTargetUrl(url, options.allowPrivate, options.resolve);
928
- const transport = url.protocol === "https:" ? request$1 : request;
929
- const hostname = networkHostname(url.hostname);
930
- return new Promise((resolve, reject) => {
931
- const outgoing = transport({
932
- hostname,
933
- port: url.port || void 0,
934
- path: `${url.pathname}${url.search}`,
935
- method: request$4.method,
936
- headers: {
937
- accept: request$4.accept,
938
- host: url.host,
939
- "user-agent": "pagegraph/0.4 (+https://tanstack.com)"
940
- },
941
- agent: false,
942
- lookup: pinnedLookup(addresses),
943
- signal: AbortSignal.timeout(options.timeoutMs),
944
- ...url.protocol === "https:" ? {
945
- servername: isIP(hostname) === 0 ? hostname : void 0,
946
- checkServerIdentity: (_host, certificate) => checkServerIdentity(hostname, certificate)
947
- } : void 0
948
- });
949
- outgoing.once("response", resolve);
950
- outgoing.once("error", reject);
951
- outgoing.end();
952
- });
953
- };
954
- const probeHttp = async (request, options) => {
955
- const startedAt = performance.now();
956
- const redirects = [];
957
- let headersAt = null;
958
- try {
959
- let currentUrl = request.url;
960
- let response = null;
961
- for (let redirectCount = 0; redirectCount <= 10; redirectCount += 1) {
962
- response = await requestPinned(currentUrl, request, options);
963
- headersAt = performance.now();
964
- const location = response.headers.location;
965
- const status = response.statusCode ?? 0;
966
- if (status < 300 || status >= 400 || location === void 0) break;
967
- if (redirectCount === 10) throw new Error("Redirect limit exceeded");
968
- response.destroy();
969
- currentUrl = new URL(location, currentUrl);
970
- redirects.push(currentUrl.href);
971
- }
972
- if (response === null) throw new Error("Request did not produce a response");
973
- const bounded = request.method === "HEAD" ? {
974
- bytes: /* @__PURE__ */ new Uint8Array(),
975
- truncated: false
976
- } : await readBoundedBody(response, options.maxBodyBytes);
977
- const contentType = typeof response.headers["content-type"] === "string" ? response.headers["content-type"] : "";
978
- const body = new TextDecoder().decode(bounded.bytes);
979
- return {
980
- kind: request.kind,
981
- method: request.method,
982
- accept: request.accept,
983
- requestedUrl: request.url.href,
984
- finalUrl: currentUrl.href,
985
- redirects,
986
- status: response.statusCode ?? null,
987
- ok: response.statusCode !== void 0 && response.statusCode >= 200 && response.statusCode < 300,
988
- headersMs: headersAt === null ? null : Math.round((headersAt - startedAt) * 10) / 10,
989
- totalMs: Math.round((performance.now() - startedAt) * 10) / 10,
990
- responseHeaders: responseHeaders(response.headers),
991
- capturedBodyBytes: bounded.bytes.byteLength,
992
- capturedBodySha256: bounded.bytes.byteLength === 0 ? null : createHash("sha256").update(bounded.bytes).digest("hex"),
993
- bodyTruncated: bounded.truncated,
994
- bodyExcerpt: bodyExcerpt(body, contentType),
995
- document: body.length === 0 ? null : extractDocumentSignals(body, contentType, currentUrl),
996
- error: null
997
- };
998
- } catch (error) {
999
- return {
1000
- kind: request.kind,
1001
- method: request.method,
1002
- accept: request.accept,
1003
- requestedUrl: request.url.href,
1004
- finalUrl: null,
1005
- redirects,
1006
- status: null,
1007
- ok: false,
1008
- headersMs: headersAt === null ? null : Math.round((headersAt - startedAt) * 10) / 10,
1009
- totalMs: Math.round((performance.now() - startedAt) * 10) / 10,
1010
- responseHeaders: {},
1011
- capturedBodyBytes: 0,
1012
- capturedBodySha256: null,
1013
- bodyTruncated: false,
1014
- bodyExcerpt: null,
1015
- document: null,
1016
- error: errorMessage$2(error)
1017
- };
1018
- }
1019
- };
1020
- const probeAll = async (requests, options) => {
1021
- const results = Array.from({ length: requests.length });
1022
- const pending = requests.map((request, index) => ({
1023
- index,
1024
- request
1025
- }));
1026
- const workers = Array.from({ length: Math.min(4, Math.max(1, pending.length)) }, async () => {
1027
- while (pending.length > 0) {
1028
- const item = pending.shift();
1029
- if (item !== void 0) results[item.index] = await probeHttp(item.request, options);
1030
- }
1031
- });
1032
- await Promise.all(workers);
1033
- return results.map((result, index) => {
1034
- if (result === void 0) throw new Error(`Probe ${index} did not produce a result`);
1035
- return result;
1036
- });
1037
- };
1038
- Effect$1.fn("SeoAudit.probeHttp")(function* (request, options) {
1039
- return yield* Effect$1.tryPromise({
1040
- try: () => probeHttp(request, options),
1041
- catch: (cause) => new HttpProbeError({
1042
- message: errorMessage$2(cause),
1043
- cause
1044
- })
1045
- });
1046
- });
1047
- const makeHttpScanner = (options) => ({
1048
- id: "http",
1049
- description: "HTTP metadata, discovery, and content-negotiation probes",
1050
- scan: (input) => Effect$1.fn("SeoAudit.scanHttp")(function* () {
1051
- const target = new URL(input.target.url);
1052
- const evidence = yield* Effect$1.tryPromise({
1053
- try: () => probeAll(buildProbeRequests([target], options.discovery), options),
1054
- catch: (cause) => new ScannerFailure({
1055
- scanner: "http",
1056
- target: target.href,
1057
- message: errorMessage$2(cause),
1058
- cause
1059
- })
1060
- });
1061
- const acquisitionErrors = evidence.flatMap((probe) => probe.status === null && probe.error !== null ? [probe.error] : []);
1062
- if (acquisitionErrors.length === evidence.length) return yield* new ScannerFailure({
1063
- scanner: "http",
1064
- target: target.href,
1065
- message: `Every HTTP probe failed: ${[...new Set(acquisitionErrors)].join("; ")}`
1066
- });
1067
- return { evidence };
1068
- })()
1069
- });
1070
- //#endregion
1071
- //#region src/audit/proxy.ts
1072
- const proxyRequestHeaders = (headers, target) => {
1073
- const forwarded = {
1074
- ...headers,
1075
- host: target.host
1076
- };
1077
- delete forwarded["proxy-authorization"];
1078
- delete forwarded["proxy-connection"];
1079
- return forwarded;
1080
- };
1081
- const connectAuthority = (authority) => {
1082
- const target = new URL(`http://${authority}`);
1083
- if (target.username || target.password || target.pathname !== "/" || target.search || target.hash) throw new Error("Invalid proxy CONNECT authority");
1084
- return target;
1085
- };
1086
- const connectPinned = (target, addresses, port, timeoutMs, track) => new Promise((resolve, reject) => {
1087
- const socket = connect({
1088
- host: networkHostname(target.hostname),
1089
- port,
1090
- autoSelectFamily: true,
1091
- lookup: pinnedLookup(addresses)
1092
- });
1093
- track(socket);
1094
- socket.setTimeout(timeoutMs, () => socket.destroy(/* @__PURE__ */ new Error("Proxy connection timed out")));
1095
- socket.once("connect", () => resolve(socket));
1096
- socket.once("error", reject);
1097
- });
1098
- /**
1099
- * Start a loopback-only validating proxy for browser scanners. Every request,
1100
- * including CONNECT and every redirect hop, is DNS-pinned and revalidated.
1101
- */
1102
- const startAuditProxy = async (options) => {
1103
- const sockets = /* @__PURE__ */ new Set();
1104
- const timeoutMs = options.timeoutMs ?? 45e3;
1105
- let closed = false;
1106
- let closePromise;
1107
- const track = (socket) => {
1108
- sockets.add(socket);
1109
- socket.once("close", () => sockets.delete(socket));
1110
- };
1111
- const server = createServer(async (request$3, response) => {
1112
- try {
1113
- if (request$3.url === void 0) throw new Error("Proxy request URL is missing");
1114
- const target = new URL(request$3.url);
1115
- if (target.protocol !== "http:") throw new Error("HTTPS proxy requests must use CONNECT");
1116
- const addresses = await resolveTargetUrl(target, options.allowPrivate, options.resolve);
1117
- if (closed || request$3.destroyed) throw new Error("Audit proxy is closed");
1118
- const upstream = request({
1119
- hostname: networkHostname(target.hostname),
1120
- port: target.port || void 0,
1121
- path: `${target.pathname}${target.search}`,
1122
- method: request$3.method,
1123
- headers: proxyRequestHeaders(request$3.headers, target),
1124
- agent: false,
1125
- lookup: pinnedLookup(addresses)
1126
- });
1127
- upstream.once("socket", track);
1128
- upstream.setTimeout(timeoutMs, () => upstream.destroy(/* @__PURE__ */ new Error("Proxy request timed out")));
1129
- upstream.once("response", (upstreamResponse) => {
1130
- response.writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers);
1131
- upstreamResponse.pipe(response);
1132
- });
1133
- upstream.once("error", (error) => {
1134
- if (!response.headersSent) response.writeHead(502, { "content-type": "text/plain" });
1135
- response.end(error.message);
1136
- });
1137
- request$3.pipe(upstream);
1138
- } catch (error) {
1139
- response.writeHead(403, { "content-type": "text/plain" });
1140
- response.end(error instanceof Error ? error.message : String(error));
1141
- }
1142
- });
1143
- server.on("connection", track);
1144
- server.on("connect", async (request, client, head) => {
1145
- try {
1146
- if (request.url === void 0) throw new Error("Proxy CONNECT authority is missing");
1147
- const target = connectAuthority(request.url);
1148
- const port = target.port === "" ? 443 : Number(target.port);
1149
- if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("Invalid proxy CONNECT port");
1150
- const addresses = await resolveTargetUrl(target, options.allowPrivate, options.resolve);
1151
- if (closed || client.destroyed) throw new Error("Audit proxy is closed");
1152
- const upstream = await connectPinned(target, addresses, port, timeoutMs, track);
1153
- client.write("HTTP/1.1 200 Connection Established\r\n\r\n");
1154
- if (head.byteLength > 0) upstream.write(head);
1155
- upstream.pipe(client);
1156
- client.pipe(upstream);
1157
- } catch (error) {
1158
- client.end(`HTTP/1.1 403 Forbidden\r\nConnection: close\r\nContent-Type: text/plain\r\n\r\n${error instanceof Error ? error.message : String(error)}`);
1159
- }
1160
- });
1161
- await new Promise((resolve, reject) => {
1162
- server.once("error", reject);
1163
- server.listen(0, "127.0.0.1", resolve);
1164
- });
1165
- const address = server.address();
1166
- if (address === null || typeof address === "string") throw new Error("Audit proxy did not bind");
1167
- return {
1168
- url: `http://127.0.0.1:${address.port}`,
1169
- close: () => {
1170
- closePromise ??= new Promise((resolve, reject) => {
1171
- closed = true;
1172
- for (const socket of sockets) socket.destroy();
1173
- server.close((error) => error ? reject(error) : resolve());
1174
- });
1175
- return closePromise;
1176
- }
1177
- };
1178
- };
1179
- //#endregion
1180
- //#region src/audit/scanners/lighthouse.ts
1181
- const require = createRequire(import.meta.url);
1182
- const metricIds = [
1183
- "first-contentful-paint",
1184
- "largest-contentful-paint",
1185
- "total-blocking-time",
1186
- "cumulative-layout-shift",
1187
- "speed-index"
1188
- ];
1189
- const UnknownRecord = Schema$1.Record(Schema$1.String, Schema$1.Unknown);
1190
- const decodeUnknownRecord = Schema$1.decodeUnknownOption(UnknownRecord);
1191
- const asRecord = (value) => Option.getOrNull(decodeUnknownRecord(value));
1192
- const asString = (value) => typeof value === "string" ? value : null;
1193
- const asNumber = (value) => typeof value === "number" && Number.isFinite(value) ? value : null;
1194
- var LighthouseError = class extends Data$1.TaggedError("LighthouseError") {};
1195
- const errorMessage$1 = (error) => error instanceof Error ? error.message : String(error);
1196
- const resolveLighthouse = () => {
1197
- try {
1198
- return {
1199
- packageJson: require.resolve("lighthouse/package.json"),
1200
- cli: require.resolve("lighthouse/cli/index.js")
1201
- };
1202
- } catch (cause) {
1203
- throw new LighthouseError({
1204
- message: "Lighthouse is not installed. Install it as an optional dependency to enable browser audits.",
1205
- cause
1206
- });
1207
- }
1208
- };
1209
- const parseCategories = (value) => {
1210
- const categories = {};
1211
- for (const [id, candidate] of Object.entries(asRecord(value) ?? {})) {
1212
- const category = asRecord(candidate);
1213
- if (category !== null) categories[id] = {
1214
- id,
1215
- title: asString(category.title) ?? id,
1216
- score: asNumber(category.score)
1217
- };
1218
- }
1219
- return categories;
1220
- };
1221
- const parseMetrics = (auditsValue) => {
1222
- const metrics = {};
1223
- const audits = asRecord(auditsValue) ?? {};
1224
- for (const id of metricIds) {
1225
- const audit = asRecord(audits[id]);
1226
- if (audit !== null) metrics[id] = {
1227
- id,
1228
- title: asString(audit.title) ?? id,
1229
- numericValue: asNumber(audit.numericValue),
1230
- numericUnit: asString(audit.numericUnit),
1231
- displayValue: asString(audit.displayValue),
1232
- score: asNumber(audit.score)
1233
- };
1234
- }
1235
- return metrics;
1236
- };
1237
- const parseOpportunities = (auditsValue) => {
1238
- const opportunities = [];
1239
- for (const [id, candidate] of Object.entries(asRecord(auditsValue) ?? {})) {
1240
- const audit = asRecord(candidate);
1241
- const details = asRecord(audit?.details);
1242
- if (audit === null || details?.type !== "opportunity") continue;
1243
- const savingsMs = asNumber(details.overallSavingsMs);
1244
- const savingsBytes = asNumber(details.overallSavingsBytes);
1245
- if ((savingsMs ?? 0) <= 0 && (savingsBytes ?? 0) <= 0) continue;
1246
- opportunities.push({
1247
- id,
1248
- title: asString(audit.title) ?? id,
1249
- displayValue: asString(audit.displayValue),
1250
- savingsMs,
1251
- savingsBytes
1252
- });
1253
- }
1254
- return opportunities.sort((left, right) => (right.savingsMs ?? 0) - (left.savingsMs ?? 0) || (right.savingsBytes ?? 0) - (left.savingsBytes ?? 0));
1255
- };
1256
- const parseWarnings = (value) => Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
1257
- const parseLighthouseResult = (value, requestedUrl, formFactor, run) => {
1258
- const result = asRecord(value);
1259
- if (result === null) throw new LighthouseError({ message: "Lighthouse returned a non-object result" });
1260
- const environment = asRecord(result.environment);
1261
- return {
1262
- requestedUrl,
1263
- finalUrl: asString(result.finalDisplayedUrl) ?? asString(result.finalUrl),
1264
- formFactor,
1265
- run,
1266
- lighthouseVersion: asString(result.lighthouseVersion),
1267
- userAgent: asString(environment?.hostUserAgent),
1268
- fetchTime: asString(result.fetchTime),
1269
- categories: parseCategories(result.categories),
1270
- metrics: parseMetrics(result.audits),
1271
- opportunities: parseOpportunities(result.audits),
1272
- warnings: parseWarnings(result.runWarnings),
1273
- error: null
1274
- };
1275
- };
1276
- const lighthouseVersion = async () => {
1277
- try {
1278
- const { packageJson } = resolveLighthouse();
1279
- const value = JSON.parse(await readFile(packageJson, "utf8"));
1280
- return asString(asRecord(value)?.version);
1281
- } catch {
1282
- return null;
1283
- }
1284
- };
1285
- const lighthouseChromeFlags = (proxyUrl) => [
1286
- "--headless=new",
1287
- `--proxy-server=${proxyUrl}`,
1288
- "--proxy-bypass-list=<-loopback>",
1289
- "--disable-quic",
1290
- "--force-webrtc-ip-handling-policy=disable_non_proxied_udp"
1291
- ].join(" ");
1292
- const windowsProcessTreeKill = (pid) => ({
1293
- command: "taskkill",
1294
- arguments: [
1295
- "/pid",
1296
- String(pid),
1297
- "/T",
1298
- "/F"
1299
- ]
1300
- });
1301
- const runLighthouseProcess = async (cli, arguments_, timeoutMs, killGraceMs = 1e3) => {
1302
- const child = spawn(process.execPath, [cli, ...arguments_], {
1303
- detached: process.platform !== "win32",
1304
- stdio: [
1305
- "ignore",
1306
- "ignore",
1307
- "pipe"
1308
- ]
1309
- });
1310
- let stderr = "";
1311
- child.stderr.setEncoding("utf8");
1312
- child.stderr.on("data", (chunk) => {
1313
- if (stderr.length < 16e3) stderr += chunk;
1314
- });
1315
- const signal = (name) => {
1316
- if (child.pid === void 0) return;
1317
- if (process.platform === "win32") {
1318
- const taskkill = windowsProcessTreeKill(child.pid);
1319
- const killer = spawn(taskkill.command, taskkill.arguments, {
1320
- stdio: "ignore",
1321
- windowsHide: true
1322
- });
1323
- const fallback = () => {
1324
- try {
1325
- child.kill("SIGKILL");
1326
- } catch {}
1327
- };
1328
- killer.once("error", fallback);
1329
- killer.once("exit", (code) => {
1330
- if (code !== 0) fallback();
1331
- });
1332
- return;
1333
- }
1334
- try {
1335
- process.kill(-child.pid, name);
1336
- } catch {
1337
- child.kill(name);
1338
- }
1339
- };
1340
- let timedOut = false;
1341
- let forceKill;
1342
- let abandon;
1343
- let timeout;
1344
- const exitCode = await new Promise((resolveExit, reject) => {
1345
- timeout = setTimeout(() => {
1346
- timedOut = true;
1347
- signal("SIGTERM");
1348
- forceKill = setTimeout(() => signal("SIGKILL"), killGraceMs);
1349
- abandon = setTimeout(() => reject(new LighthouseError({ message: `Lighthouse timed out after ${timeoutMs}ms and did not exit after SIGKILL` })), killGraceMs * 2);
1350
- }, timeoutMs);
1351
- child.once("error", reject);
1352
- child.once("exit", (code) => resolveExit(code ?? 1));
1353
- }).finally(() => {
1354
- if (timeout !== void 0) clearTimeout(timeout);
1355
- if (forceKill !== void 0) clearTimeout(forceKill);
1356
- if (abandon !== void 0) clearTimeout(abandon);
1357
- });
1358
- if (timedOut) throw new LighthouseError({ message: `Lighthouse timed out after ${timeoutMs}ms` });
1359
- if (exitCode !== 0) throw new LighthouseError({ message: `Lighthouse exited with ${exitCode}${stderr.trim() ? `: ${stderr.trim()}` : ""}` });
1360
- };
1361
- const runLighthouseInDirectory = async (options, directory) => {
1362
- const version = await lighthouseVersion();
1363
- const base = {
1364
- requestedUrl: options.url.href,
1365
- finalUrl: null,
1366
- formFactor: options.formFactor,
1367
- run: options.run,
1368
- lighthouseVersion: version,
1369
- userAgent: null,
1370
- fetchTime: null,
1371
- categories: {},
1372
- metrics: {},
1373
- opportunities: [],
1374
- warnings: []
1375
- };
1376
- let proxy;
1377
- try {
1378
- const { cli } = resolveLighthouse();
1379
- const output = join(directory, "result.json");
1380
- proxy = await startAuditProxy({
1381
- allowPrivate: options.allowPrivate,
1382
- timeoutMs: options.timeoutMs ?? 18e4
1383
- });
1384
- const args = [
1385
- options.url.href,
1386
- "--output=json",
1387
- `--output-path=${output}`,
1388
- "--quiet",
1389
- "--max-wait-for-load=45000",
1390
- `--chrome-flags=${lighthouseChromeFlags(proxy.url)}`
1391
- ];
1392
- if (options.formFactor === "desktop") args.push("--preset=desktop");
1393
- await runLighthouseProcess(cli, args, options.timeoutMs ?? 18e4);
1394
- return parseLighthouseResult(JSON.parse(await readFile(output, "utf8")), options.url.href, options.formFactor, options.run);
1395
- } catch (error) {
1396
- return {
1397
- ...base,
1398
- error: errorMessage$1(error)
1399
- };
1400
- } finally {
1401
- await proxy?.close();
1402
- }
1403
- };
1404
- const runLighthouse = async (options) => {
1405
- let directory;
1406
- try {
1407
- directory = await mkdtemp(join(tmpdir(), "tanstack-seo-lighthouse-"));
1408
- return await runLighthouseInDirectory(options, directory);
1409
- } finally {
1410
- if (directory !== void 0) await rm(directory, {
1411
- recursive: true,
1412
- force: true
1413
- });
1414
- }
1415
- };
1416
- Effect$1.fn("SeoAudit.runLighthouse")(function* (options) {
1417
- const directory = yield* Effect$1.acquireRelease(Effect$1.tryPromise({
1418
- try: () => mkdtemp(join(tmpdir(), "tanstack-seo-lighthouse-")),
1419
- catch: (cause) => new LighthouseError({
1420
- message: errorMessage$1(cause),
1421
- cause
1422
- })
1423
- }), (path) => Effect$1.promise(() => rm(path, {
1424
- recursive: true,
1425
- force: true
1426
- })));
1427
- return yield* Effect$1.tryPromise({
1428
- try: () => runLighthouseInDirectory(options, directory),
1429
- catch: (cause) => new LighthouseError({
1430
- message: errorMessage$1(cause),
1431
- cause
1432
- })
1433
- });
1434
- });
1435
- const makeLighthouseScanner = (options, dependencies = {}) => ({
1436
- id: "lighthouse",
1437
- description: "Lighthouse performance and accessibility measurements",
1438
- scan: (input) => Effect$1.fn("SeoAudit.scanLighthouse")(function* () {
1439
- const target = new URL(input.target.url);
1440
- const requestedFormFactors = input.options.formFactors === void 0 || input.options.formFactors.length === 0 ? ["mobile"] : input.options.formFactors;
1441
- const requestedRuns = Math.max(1, Math.floor(input.options.runs ?? 1));
1442
- const runs = [...new Set(requestedFormFactors)].flatMap((formFactor) => Array.from({ length: requestedRuns }, (_, index) => ({
1443
- formFactor,
1444
- run: index + 1
1445
- })));
1446
- const observations = yield* Effect$1.forEach(runs, ({ formFactor, run }) => Effect$1.tryPromise({
1447
- try: () => (dependencies.run ?? runLighthouse)({
1448
- ...options,
1449
- url: target,
1450
- formFactor,
1451
- run
1452
- }),
1453
- catch: (cause) => new ScannerFailure({
1454
- scanner: "lighthouse",
1455
- target: target.href,
1456
- message: errorMessage$1(cause),
1457
- cause
1458
- })
1459
- }), { concurrency: 1 });
1460
- const errors = observations.flatMap((observation) => observation.error === null ? [] : [observation.error]);
1461
- if (errors.length === observations.length) return yield* new ScannerFailure({
1462
- scanner: "lighthouse",
1463
- target: target.href,
1464
- message: errors[0] ?? "Every Lighthouse run failed."
1465
- });
1466
- return { evidence: observations };
1467
- })()
1468
- });
1469
- //#endregion
1470
- //#region src/audit/scanners/hosted.ts
1471
- var HostedScannerError = class extends Data$1.TaggedError("HostedScannerError") {};
1472
- const errorMessage = (error) => error instanceof Error ? error.message : String(error);
1473
- const userAgent = "TanStackSeoAudit/1.0";
1474
- const IsitagentreadyCheck = Schema$1.Struct({
1475
- status: Schema$1.String,
1476
- message: Schema$1.optionalKey(Schema$1.String)
1477
- });
1478
- const IsitagentreadyPayload = Schema$1.Struct({
1479
- scannedAt: Schema$1.optionalKey(Schema$1.String),
1480
- scanned_at: Schema$1.optionalKey(Schema$1.String),
1481
- level: Schema$1.optionalKey(Schema$1.Finite),
1482
- levelName: Schema$1.optionalKey(Schema$1.String),
1483
- level_name: Schema$1.optionalKey(Schema$1.String),
1484
- checks: Schema$1.optionalKey(Schema$1.Record(Schema$1.String, Schema$1.Record(Schema$1.String, IsitagentreadyCheck))),
1485
- nextLevel: Schema$1.optionalKey(Schema$1.Struct({
1486
- name: Schema$1.optionalKey(Schema$1.String),
1487
- requirements: Schema$1.optionalKey(Schema$1.Array(Schema$1.Struct({
1488
- check: Schema$1.optionalKey(Schema$1.String),
1489
- description: Schema$1.optionalKey(Schema$1.String),
1490
- prompt: Schema$1.optionalKey(Schema$1.String)
1491
- })))
1492
- })),
1493
- next_level: Schema$1.optionalKey(Schema$1.Struct({
1494
- name: Schema$1.optionalKey(Schema$1.String),
1495
- requirements: Schema$1.optionalKey(Schema$1.Array(Schema$1.Struct({
1496
- check: Schema$1.optionalKey(Schema$1.String),
1497
- description: Schema$1.optionalKey(Schema$1.String),
1498
- prompt: Schema$1.optionalKey(Schema$1.String)
1499
- })))
1500
- }))
1501
- });
1502
- const IsAgenticPayload = Schema$1.Struct({
1503
- score: Schema$1.optionalKey(Schema$1.Finite),
1504
- score_label: Schema$1.optionalKey(Schema$1.String),
1505
- report_url: Schema$1.optionalKey(Schema$1.String),
1506
- scanned_at: Schema$1.optionalKey(Schema$1.String),
1507
- score_breakdown: Schema$1.optionalKey(Schema$1.Struct({
1508
- essential: Schema$1.optionalKey(Schema$1.Struct({
1509
- earned: Schema$1.Finite,
1510
- available: Schema$1.Finite
1511
- })),
1512
- recommended: Schema$1.optionalKey(Schema$1.Struct({
1513
- earned: Schema$1.Finite,
1514
- available: Schema$1.Finite
1515
- }))
1516
- })),
1517
- issues: Schema$1.optionalKey(Schema$1.Array(Schema$1.Struct({
1518
- result: Schema$1.String,
1519
- tier: Schema$1.optionalKey(Schema$1.String),
1520
- name: Schema$1.String,
1521
- details: Schema$1.optionalKey(Schema$1.String),
1522
- recommendation: Schema$1.optionalKey(Schema$1.String)
1523
- })))
1524
- });
1525
- const SseEvent = Schema$1.Struct({ type: Schema$1.String });
1526
- const IsitagentreadyJson = Schema$1.fromJsonString(IsitagentreadyPayload);
1527
- const IsAgenticJson = Schema$1.fromJsonString(IsAgenticPayload);
1528
- const SseEventJson = Schema$1.fromJsonString(SseEvent);
1529
- const decodeIsitagentready = Schema$1.decodeUnknownSync(IsitagentreadyPayload);
1530
- const decodeIsAgentic = Schema$1.decodeUnknownSync(IsAgenticPayload);
1531
- const decodeIsitagentreadyJson = Schema$1.decodeUnknownEffect(IsitagentreadyJson);
1532
- const decodeIsAgenticJson = Schema$1.decodeUnknownEffect(IsAgenticJson);
1533
- const decodeSseEventJson = Schema$1.decodeUnknownOption(SseEventJson);
1534
- const enabledChecks = [
1535
- "robotsTxt",
1536
- "sitemap",
1537
- "linkHeaders",
1538
- "dnsAid",
1539
- "markdownNegotiation",
1540
- "robotsTxtAiRules",
1541
- "contentSignals",
1542
- "webBotAuth",
1543
- "apiCatalog",
1544
- "oauthDiscovery",
1545
- "oauthProtectedResource",
1546
- "authMd",
1547
- "mcpServerCard",
1548
- "agentSkills",
1549
- "webMcp",
1550
- "ard",
1551
- "x402",
1552
- "mpp",
1553
- "ucp",
1554
- "acp"
1555
- ];
1556
- const encodeRequest = Schema$1.encodeSync(Schema$1.fromJsonString(Schema$1.Struct({
1557
- url: Schema$1.String,
1558
- enabledChecks: Schema$1.Array(Schema$1.Literals(enabledChecks))
1559
- })));
1560
- const parseSseDataFrames = (buffer) => {
1561
- let rest = buffer.replaceAll("\r\n", "\n");
1562
- const events = [];
1563
- let boundary = rest.indexOf("\n\n");
1564
- while (boundary !== -1) {
1565
- const frame = rest.slice(0, boundary);
1566
- rest = rest.slice(boundary + 2);
1567
- const data = frame.split("\n").filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trimStart()).join("\n");
1568
- if (data.length > 0) {
1569
- const event = Option.getOrNull(decodeSseEventJson(data));
1570
- if (event !== null) events.push(event);
1571
- }
1572
- boundary = rest.indexOf("\n\n");
1573
- }
1574
- return {
1575
- events,
1576
- rest
1577
- };
1578
- };
1579
- const pinnedResponse = async (request$2) => {
1580
- const addresses = await resolveTargetUrl(request$2.url, request$2.allowPrivate);
1581
- const transport = request$2.url.protocol === "https:" ? request$1 : request;
1582
- const hostname = networkHostname(request$2.url.hostname);
1583
- const headers = {
1584
- ...request$2.headers,
1585
- host: request$2.url.host,
1586
- "user-agent": userAgent
1587
- };
1588
- if (request$2.body !== void 0) headers["content-length"] = Buffer.byteLength(request$2.body);
1589
- return new Promise((resolve, reject) => {
1590
- const outgoing = transport({
1591
- hostname,
1592
- port: request$2.url.port || void 0,
1593
- path: `${request$2.url.pathname}${request$2.url.search}`,
1594
- method: request$2.method,
1595
- headers,
1596
- agent: false,
1597
- lookup: pinnedLookup(addresses),
1598
- signal: AbortSignal.timeout(request$2.timeoutMs),
1599
- ...request$2.url.protocol === "https:" ? {
1600
- servername: isIP(hostname) === 0 ? hostname : void 0,
1601
- checkServerIdentity: (_host, certificate) => checkServerIdentity(hostname, certificate)
1602
- } : void 0
1603
- });
1604
- outgoing.once("response", resolve);
1605
- outgoing.once("error", reject);
1606
- outgoing.end(request$2.body);
1607
- });
1608
- };
1609
- const readBody = async (response, maxBodyBytes) => {
1610
- const chunks = [];
1611
- let length = 0;
1612
- for await (const chunk of response) {
1613
- const value = new Uint8Array(chunk);
1614
- if (length + value.byteLength > maxBodyBytes) {
1615
- response.destroy();
1616
- throw new Error("Hosted scanner response exceeded byte limit");
1617
- }
1618
- chunks.push(value);
1619
- length += value.byteLength;
1620
- }
1621
- const bytes = new Uint8Array(length);
1622
- let offset = 0;
1623
- for (const chunk of chunks) {
1624
- bytes.set(chunk, offset);
1625
- offset += chunk.byteLength;
1626
- }
1627
- const body = new TextDecoder().decode(bytes);
1628
- if (body.trim().length === 0) throw new Error("Hosted scanner returned an empty body");
1629
- return body;
1630
- };
1631
- const consumeStream = async (response, maxBodyBytes) => {
1632
- let buffer = "";
1633
- let length = 0;
1634
- let complete = false;
1635
- const decoder = new TextDecoder();
1636
- try {
1637
- for await (const chunk of response) {
1638
- const value = new Uint8Array(chunk);
1639
- length += value.byteLength;
1640
- if (length > maxBodyBytes) throw new Error("Hosted scanner stream exceeded byte limit");
1641
- buffer += decoder.decode(value, { stream: true });
1642
- const parsed = parseSseDataFrames(buffer);
1643
- buffer = parsed.rest;
1644
- if (parsed.events.some((event) => event.type === "error")) throw new Error("is-agentic scan failed");
1645
- if (parsed.events.some((event) => event.type === "scan_complete" || event.type === "scan_archived")) {
1646
- complete = true;
1647
- response.destroy();
1648
- break;
1649
- }
1650
- }
1651
- } catch (cause) {
1652
- if (complete) return;
1653
- throw cause;
1654
- }
1655
- buffer += decoder.decode();
1656
- if (!complete) throw new Error("is-agentic scan ended before a report was available");
1657
- };
1658
- const originHeader = (origin, path) => {
1659
- const base = origin.replace(/\/$/, "");
1660
- return {
1661
- origin: base,
1662
- referer: `${base}${path}`
1663
- };
1664
- };
1665
- const complete = (parsed, requestedUrl, startedAt) => ({
1666
- ...parsed,
1667
- requestedUrl,
1668
- totalMs: Math.round((performance.now() - startedAt) * 10) / 10,
1669
- error: null
1670
- });
1671
- const failed = (scanner, target, requestedUrl, startedAt, error) => ({
1672
- scanner,
1673
- target: target.href,
1674
- requestedUrl,
1675
- reportUrl: null,
1676
- scannedAt: null,
1677
- score: null,
1678
- scoreLabel: null,
1679
- level: null,
1680
- levelName: null,
1681
- summary: null,
1682
- checks: [],
1683
- findings: [],
1684
- totalMs: Math.round((performance.now() - startedAt) * 10) / 10,
1685
- error: errorMessage(error)
1686
- });
1687
- const parseIsitagentready = (payload, target, origin) => {
1688
- const record = decodeIsitagentready(payload);
1689
- const checks = Object.entries(record.checks ?? {}).flatMap(([category, group]) => Object.entries(group).map(([id, check]) => ({
1690
- id,
1691
- category,
1692
- status: check.status,
1693
- message: check.message ?? null
1694
- })));
1695
- const passed = checks.filter((check) => check.status === "pass").length;
1696
- const failedCount = checks.filter((check) => check.status === "fail").length;
1697
- const level = record.level ?? null;
1698
- const levelName = record.levelName ?? record.level_name ?? null;
1699
- const next = record.nextLevel ?? record.next_level;
1700
- const findings = (next?.requirements ?? []).flatMap((requirement) => {
1701
- const name = requirement.check ?? requirement.description;
1702
- return name === void 0 ? [] : [{
1703
- result: "missing",
1704
- tier: next?.name ?? null,
1705
- name,
1706
- details: requirement.description ?? null,
1707
- recommendation: requirement.prompt ?? null
1708
- }];
1709
- });
1710
- return {
1711
- scanner: "isitagentready",
1712
- target: target.href,
1713
- reportUrl: `${origin.replace(/\/$/, "")}/${target.hostname}`,
1714
- scannedAt: record.scannedAt ?? record.scanned_at ?? null,
1715
- score: null,
1716
- scoreLabel: null,
1717
- level,
1718
- levelName,
1719
- summary: level === null && levelName === null ? `${passed} pass, ${failedCount} fail` : `Level ${level ?? "?"} ${levelName ?? ""}`.trim() + `, ${passed} pass, ${failedCount} fail`,
1720
- checks,
1721
- findings
1722
- };
1723
- };
1724
- const parseIsAgentic = (payload, target) => {
1725
- const record = decodeIsAgentic(payload);
1726
- const findings = (record.issues ?? []).map((issue) => ({
1727
- result: issue.result,
1728
- tier: issue.tier ?? null,
1729
- name: issue.name,
1730
- details: issue.details ?? null,
1731
- recommendation: issue.recommendation ?? null
1732
- }));
1733
- const failedCount = findings.filter((finding) => finding.result === "failed").length;
1734
- const partial = findings.filter((finding) => finding.result === "partial").length;
1735
- const bucket = (label, value) => value === void 0 ? null : `${label} ${value.earned}/${value.available}`;
1736
- return {
1737
- scanner: "is-agentic",
1738
- target: target.href,
1739
- reportUrl: record.report_url ?? null,
1740
- scannedAt: record.scanned_at ?? null,
1741
- score: record.score ?? null,
1742
- scoreLabel: record.score_label ?? null,
1743
- level: null,
1744
- levelName: null,
1745
- summary: [
1746
- record.score_label,
1747
- bucket("Essential", record.score_breakdown?.essential),
1748
- bucket("Recommended", record.score_breakdown?.recommended),
1749
- `${failedCount} failed, ${partial} partial`
1750
- ].filter((part) => part !== null && part !== void 0 && part.length > 0).join(", "),
1751
- checks: [],
1752
- findings
1753
- };
1754
- };
1755
- const scanIsAgentic = Effect$1.fn("SeoAudit.scanHostedIsAgentic")(function* (target, options) {
1756
- const requested = new URL("/api/scan/stream", options.origins.isAgentic);
1757
- requested.searchParams.set("target", target.href);
1758
- requested.searchParams.set("force", "1");
1759
- const headers = originHeader(options.origins.isAgentic, `/scan/${target.hostname}`);
1760
- const response = yield* Effect$1.tryPromise({
1761
- try: () => pinnedResponse({
1762
- method: "GET",
1763
- url: requested,
1764
- headers: {
1765
- accept: "text/event-stream",
1766
- "cache-control": "no-cache",
1767
- pragma: "no-cache",
1768
- ...headers
1769
- },
1770
- allowPrivate: options.allowPrivate,
1771
- timeoutMs: options.timeoutMs
1772
- }),
1773
- catch: (cause) => new HostedScannerError({
1774
- message: errorMessage(cause),
1775
- cause
1776
- })
1777
- });
1778
- const status = response.statusCode ?? 0;
1779
- if (status < 200 || status >= 300) {
1780
- const retry = response.headers["retry-after"];
1781
- response.destroy();
1782
- return yield* new HostedScannerError({ message: `is-agentic.com scan stream returned HTTP ${status}${typeof retry === "string" ? `, retry after ${retry}s` : ""}` });
1783
- }
1784
- yield* Effect$1.tryPromise({
1785
- try: () => consumeStream(response, options.maxBodyBytes),
1786
- catch: (cause) => new HostedScannerError({
1787
- message: errorMessage(cause),
1788
- cause
1789
- })
1790
- });
1791
- const report = new URL("/api/v1/report", options.origins.isAgentic);
1792
- report.searchParams.set("url", target.href);
1793
- let lastError = new HostedScannerError({ message: "is-agentic report is not available yet" });
1794
- for (let attempt = 0; attempt < 5; attempt += 1) {
1795
- const reportResponse = yield* Effect$1.tryPromise({
1796
- try: () => pinnedResponse({
1797
- method: "GET",
1798
- url: report,
1799
- headers: { accept: "application/json" },
1800
- allowPrivate: options.allowPrivate,
1801
- timeoutMs: options.timeoutMs
1802
- }),
1803
- catch: (cause) => new HostedScannerError({
1804
- message: errorMessage(cause),
1805
- cause
1806
- })
1807
- });
1808
- const reportStatus = reportResponse.statusCode ?? 0;
1809
- const body = yield* Effect$1.tryPromise({
1810
- try: () => readBody(reportResponse, options.maxBodyBytes),
1811
- catch: (cause) => new HostedScannerError({
1812
- message: errorMessage(cause),
1813
- cause
1814
- })
1815
- });
1816
- if (reportStatus >= 200 && reportStatus < 300) {
1817
- const payload = yield* decodeIsAgenticJson(body).pipe(Effect$1.mapError((cause) => new HostedScannerError({
1818
- message: errorMessage(cause),
1819
- cause
1820
- })));
1821
- return {
1822
- requested,
1823
- parsed: parseIsAgentic(payload, target)
1824
- };
1825
- }
1826
- lastError = new HostedScannerError({ message: `is-agentic report returned HTTP ${reportStatus}` });
1827
- yield* Effect$1.sleep(`${250 * (attempt + 1)} millis`);
1828
- }
1829
- return yield* lastError;
1830
- });
1831
- const scanIsitagentready = Effect$1.fn("SeoAudit.scanHostedIsitagentready")(function* (target, options) {
1832
- const requested = new URL("/api/scan", options.origins.isitagentready);
1833
- const headers = originHeader(options.origins.isitagentready, `/${target.hostname}`);
1834
- const response = yield* Effect$1.tryPromise({
1835
- try: () => pinnedResponse({
1836
- method: "POST",
1837
- url: requested,
1838
- headers: {
1839
- accept: "application/json",
1840
- "content-type": "application/json",
1841
- ...headers
1842
- },
1843
- body: encodeRequest({
1844
- url: target.href,
1845
- enabledChecks: [...enabledChecks]
1846
- }),
1847
- allowPrivate: options.allowPrivate,
1848
- timeoutMs: options.timeoutMs
1849
- }),
1850
- catch: (cause) => new HostedScannerError({
1851
- message: errorMessage(cause),
1852
- cause
1853
- })
1854
- });
1855
- const status = response.statusCode ?? 0;
1856
- if (status < 200 || status >= 300) {
1857
- response.destroy();
1858
- return yield* new HostedScannerError({ message: `isitagentready.com returned HTTP ${status}` });
1859
- }
1860
- const body = yield* Effect$1.tryPromise({
1861
- try: () => readBody(response, options.maxBodyBytes),
1862
- catch: (cause) => new HostedScannerError({
1863
- message: errorMessage(cause),
1864
- cause
1865
- })
1866
- });
1867
- const payload = yield* decodeIsitagentreadyJson(body).pipe(Effect$1.mapError((cause) => new HostedScannerError({
1868
- message: errorMessage(cause),
1869
- cause
1870
- })));
1871
- return {
1872
- requested,
1873
- parsed: parseIsitagentready(payload, target, options.origins.isitagentready)
1874
- };
1875
- });
1876
- /** Opt-in hosted scanners. Results are attributed failures, never fatal audit failures. */
1877
- const scanHosted = async (target, options) => {
1878
- const output = [];
1879
- const readyStartedAt = performance.now();
1880
- try {
1881
- const result = await Effect$1.runPromise(scanIsitagentready(target, options));
1882
- output.push(complete(result.parsed, result.requested.href, readyStartedAt));
1883
- } catch (error) {
1884
- output.push(failed("isitagentready", target, new URL("/api/scan", options.origins.isitagentready).href, readyStartedAt, error));
1885
- }
1886
- const agenticStartedAt = performance.now();
1887
- try {
1888
- const result = await Effect$1.runPromise(scanIsAgentic(target, options));
1889
- output.push(complete(result.parsed, result.requested.href, agenticStartedAt));
1890
- } catch (error) {
1891
- output.push(failed("is-agentic", target, new URL("/api/scan/stream", options.origins.isAgentic).href, agenticStartedAt, error));
1892
- }
1893
- return output;
1894
- };
1895
- const makeHostedScanner = (options) => ({
1896
- id: "hosted",
1897
- description: "Opt-in external agent-readiness reports",
1898
- scan: (input) => Effect$1.fn("SeoAudit.scanHosted")(function* () {
1899
- const target = new URL(input.target.url);
1900
- if (options.allowPrivateTargets !== true) yield* Effect$1.tryPromise({
1901
- try: () => resolveTargetUrl(target, false),
1902
- catch: (cause) => new ScannerFailure({
1903
- scanner: "hosted",
1904
- target: target.href,
1905
- message: "Hosted scanners only accept public target URLs.",
1906
- cause
1907
- })
1908
- });
1909
- const evidence = yield* Effect$1.tryPromise({
1910
- try: () => scanHosted(target, options),
1911
- catch: (cause) => new ScannerFailure({
1912
- scanner: "hosted",
1913
- target: target.href,
1914
- message: errorMessage(cause),
1915
- cause
1916
- })
1917
- });
1918
- if (evidence.every((report) => report.error !== null)) return yield* new ScannerFailure({
1919
- scanner: "hosted",
1920
- target: target.href,
1921
- message: evidence.map((report) => report.error).join("; ")
1922
- });
1923
- return { evidence };
1924
- })()
1925
- });
1926
- //#endregion
1927
- export { defaultAuditOptions as A, ScannerStatus as C, makeAudit as D, InvalidTarget as E, pageRules as M, ScannerRegistryLive as O, ScannerResult as S, AuditLive as T, compareAuditReports as _, makeHttpScanner as a, AuditReport as b, writeAuditFiles as c, AuditLayer as d, AuditComparisonResult as f, AuditDiffSummary as g, AuditDiffOutcome as h, parseLighthouseResult as i, evaluateRules as j, makeScannerRegistry as k, ScannerFailure as l, AuditDiffChange as m, lighthouseChromeFlags as n, renderAuditDiff as o, AuditDiff as p, makeLighthouseScanner as r, renderAuditMarkdown as s, makeHostedScanner as t, scanner as u, AuditFinding as v, Audit as w, FindingSeverity as x, AuditPolicy as y };
1928
-
1929
- //# sourceMappingURL=audit-B96V1x3q.js.map