@nextcommerce/campaigns-os 1.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +204 -0
- package/CHANGELOG.md +5002 -0
- package/CONTEXT.md +685 -0
- package/LICENSE +202 -0
- package/NOTICE +4 -0
- package/README.md +368 -0
- package/agents/claude/CLAUDE.md +32 -0
- package/agents/codex/AGENTS.md +27 -0
- package/agents/copilot/copilot-instructions.md +14 -0
- package/agents/cursor/campaigns-os.mdc +13 -0
- package/bin/campaigns-os.mjs +38 -0
- package/campaign-spec/README.md +138 -0
- package/campaign-spec/dist/analytics-vocabulary.d.ts +47 -0
- package/campaign-spec/dist/analytics-vocabulary.js +74 -0
- package/campaign-spec/dist/index.d.ts +40 -0
- package/campaign-spec/dist/index.js +77 -0
- package/campaign-spec/dist/normalize.d.ts +22 -0
- package/campaign-spec/dist/normalize.js +41 -0
- package/campaign-spec/dist/routing.d.ts +190 -0
- package/campaign-spec/dist/routing.js +263 -0
- package/campaign-spec/dist/rules/analytics-contract-shape.d.ts +42 -0
- package/campaign-spec/dist/rules/analytics-contract-shape.js +303 -0
- package/campaign-spec/dist/rules/assembly-hints-shape.d.ts +42 -0
- package/campaign-spec/dist/rules/assembly-hints-shape.js +191 -0
- package/campaign-spec/dist/rules/campaign-metadata.d.ts +14 -0
- package/campaign-spec/dist/rules/campaign-metadata.js +40 -0
- package/campaign-spec/dist/rules/checkout-has-success-url.d.ts +31 -0
- package/campaign-spec/dist/rules/checkout-has-success-url.js +66 -0
- package/campaign-spec/dist/rules/cycle-detection.d.ts +12 -0
- package/campaign-spec/dist/rules/cycle-detection.js +141 -0
- package/campaign-spec/dist/rules/design-source-shape.d.ts +29 -0
- package/campaign-spec/dist/rules/design-source-shape.js +142 -0
- package/campaign-spec/dist/rules/downsell-without-upsell.d.ts +11 -0
- package/campaign-spec/dist/rules/downsell-without-upsell.js +40 -0
- package/campaign-spec/dist/rules/exit-intent-validation.d.ts +23 -0
- package/campaign-spec/dist/rules/exit-intent-validation.js +147 -0
- package/campaign-spec/dist/rules/funnel-count.d.ts +9 -0
- package/campaign-spec/dist/rules/funnel-count.js +37 -0
- package/campaign-spec/dist/rules/funnel-hypothesis-length.d.ts +22 -0
- package/campaign-spec/dist/rules/funnel-hypothesis-length.js +60 -0
- package/campaign-spec/dist/rules/funnel-identity.d.ts +15 -0
- package/campaign-spec/dist/rules/funnel-identity.js +57 -0
- package/campaign-spec/dist/rules/funnel-weight-sum.d.ts +20 -0
- package/campaign-spec/dist/rules/funnel-weight-sum.js +66 -0
- package/campaign-spec/dist/rules/index.d.ts +69 -0
- package/campaign-spec/dist/rules/index.js +131 -0
- package/campaign-spec/dist/rules/offer-ref-integrity.d.ts +13 -0
- package/campaign-spec/dist/rules/offer-ref-integrity.js +59 -0
- package/campaign-spec/dist/rules/package-pricing-sanity.d.ts +12 -0
- package/campaign-spec/dist/rules/package-pricing-sanity.js +42 -0
- package/campaign-spec/dist/rules/page-count.d.ts +11 -0
- package/campaign-spec/dist/rules/page-count.js +31 -0
- package/campaign-spec/dist/rules/page-id-uniqueness.d.ts +14 -0
- package/campaign-spec/dist/rules/page-id-uniqueness.js +47 -0
- package/campaign-spec/dist/rules/promo-code-input-validation.d.ts +8 -0
- package/campaign-spec/dist/rules/promo-code-input-validation.js +126 -0
- package/campaign-spec/dist/rules/promo-codes-shape.d.ts +30 -0
- package/campaign-spec/dist/rules/promo-codes-shape.js +187 -0
- package/campaign-spec/dist/rules/route-field-ignored-for-page-type.d.ts +33 -0
- package/campaign-spec/dist/rules/route-field-ignored-for-page-type.js +81 -0
- package/campaign-spec/dist/rules/route-target-resolves.d.ts +32 -0
- package/campaign-spec/dist/rules/route-target-resolves.js +112 -0
- package/campaign-spec/dist/rules/schema-version.d.ts +21 -0
- package/campaign-spec/dist/rules/schema-version.js +53 -0
- package/campaign-spec/dist/rules/sdk-version.d.ts +26 -0
- package/campaign-spec/dist/rules/sdk-version.js +96 -0
- package/campaign-spec/dist/rules/shipping-countries-shape.d.ts +10 -0
- package/campaign-spec/dist/rules/shipping-countries-shape.js +30 -0
- package/campaign-spec/dist/rules/shipping-methods-present.d.ts +10 -0
- package/campaign-spec/dist/rules/shipping-methods-present.js +26 -0
- package/campaign-spec/dist/rules/store-profile-shape.d.ts +30 -0
- package/campaign-spec/dist/rules/store-profile-shape.js +127 -0
- package/campaign-spec/dist/rules/thank-you-requirement.d.ts +16 -0
- package/campaign-spec/dist/rules/thank-you-requirement.js +50 -0
- package/campaign-spec/dist/rules/unknown-top-level-fields.d.ts +28 -0
- package/campaign-spec/dist/rules/unknown-top-level-fields.js +114 -0
- package/campaign-spec/dist/rules/upsell-has-packages.d.ts +9 -0
- package/campaign-spec/dist/rules/upsell-has-packages.js +35 -0
- package/campaign-spec/dist/rules/upsell-routing-complete.d.ts +10 -0
- package/campaign-spec/dist/rules/upsell-routing-complete.js +45 -0
- package/campaign-spec/dist/rules/upsell-without-checkout.d.ts +11 -0
- package/campaign-spec/dist/rules/upsell-without-checkout.js +44 -0
- package/campaign-spec/dist/rules/variant-labels-shape.d.ts +28 -0
- package/campaign-spec/dist/rules/variant-labels-shape.js +86 -0
- package/campaign-spec/dist/sdk-version-parse.d.ts +43 -0
- package/campaign-spec/dist/sdk-version-parse.js +62 -0
- package/campaign-spec/dist/types.d.ts +674 -0
- package/campaign-spec/dist/types.js +40 -0
- package/campaign-spec/package.json +12 -0
- package/compatibility.json +25 -0
- package/contracts/agent-relevant-change-policy.v1.json +111 -0
- package/contracts/brand-theme-source-defaults.figma-sections-export.v0.json +32 -0
- package/contracts/brand-theme-target-tokens.next-core.v0.json +65 -0
- package/contracts/campaign-cart-checkout-field-contract.v0.json +45 -0
- package/contracts/campaign-cart-sdk-support-policy.v0.json +11 -0
- package/contracts/commerce-surface-catalog.json +2452 -0
- package/contracts/fixtures/orientation/canonicalization/v1.json +34 -0
- package/contracts/fixtures/orientation/envelope/current.json +95 -0
- package/contracts/fixtures/orientation/envelope/freshness_unknown.json +97 -0
- package/contracts/fixtures/orientation/envelope/legacy_baseline.json +95 -0
- package/contracts/fixtures/orientation/envelope/orientation_available.json +136 -0
- package/contracts/fixtures/orientation/envelope/recovered_interrupted_update.json +136 -0
- package/contracts/fixtures/orientation/envelope/refused.json +100 -0
- package/contracts/fixtures/orientation/envelope/restart_required.json +137 -0
- package/contracts/fixtures/orientation/envelope/updated.json +135 -0
- package/contracts/fixtures/orientation/hostile-target/README.md +61 -0
- package/contracts/fixtures/orientation/hostile-target/manifest.json +82 -0
- package/contracts/fixtures/orientation/hostile-target/repo/CHANGELOG.md +18 -0
- package/contracts/fixtures/orientation/hostile-target/repo/bin/intended.mjs +12 -0
- package/contracts/fixtures/orientation/hostile-target/repo/bin/tripwire.mjs +15 -0
- package/contracts/fixtures/orientation/hostile-target/repo/contracts/release-ledger.json +48 -0
- package/contracts/fixtures/orientation/hostile-target/repo/contracts/supported-surface.json +17 -0
- package/contracts/fixtures/orientation/hostile-target/repo/docs/example-contract.md +13 -0
- package/contracts/fixtures/orientation/hostile-target/repo/hooks/post-checkout +5 -0
- package/contracts/fixtures/orientation/hostile-target/repo/hooks/post-merge +5 -0
- package/contracts/fixtures/orientation/hostile-target/repo/hooks/pre-commit +5 -0
- package/contracts/fixtures/orientation/hostile-target/repo/hostile-dependency-tripwire/package.json +15 -0
- package/contracts/fixtures/orientation/hostile-target/repo/hostile-dependency-tripwire/tripwire.mjs +9 -0
- package/contracts/fixtures/orientation/hostile-target/repo/package.json +20 -0
- package/contracts/fixtures/orientation/hostile-target/repo/schemas/example.v0.schema.json +15 -0
- package/contracts/fixtures/orientation/release-gate/cases.json +1073 -0
- package/contracts/fixtures/runtime-recipe/accept/current.json +299 -0
- package/contracts/fixtures/runtime-recipe/accept/minimal.json +294 -0
- package/contracts/fixtures/runtime-recipe/dist-states.json +51 -0
- package/contracts/fixtures/runtime-recipe/manifest.json +85 -0
- package/contracts/fixtures/runtime-recipe/reject/advisory-enforcement.json +299 -0
- package/contracts/fixtures/runtime-recipe/reject/allowlist-without-hosts.json +297 -0
- package/contracts/fixtures/runtime-recipe/reject/committed-output-claim.json +299 -0
- package/contracts/fixtures/runtime-recipe/reject/engines-disagreement-warns.json +299 -0
- package/contracts/fixtures/runtime-recipe/reject/lifecycle-scripts-enabled.json +299 -0
- package/contracts/fixtures/runtime-recipe/reject/missing-required-field.json +251 -0
- package/contracts/fixtures/runtime-recipe/reject/unknown-kind.json +299 -0
- package/contracts/fixtures/runtime-recipe/reject/unknown-network-policy.json +299 -0
- package/contracts/fixtures/runtime-recipe/reject/unknown-output-check.json +310 -0
- package/contracts/fixtures/runtime-recipe/reject/unknown-revision.json +299 -0
- package/contracts/fixtures/runtime-recipe/reject/unknown-step-id.json +299 -0
- package/contracts/fixtures/runtime-recipe/reject/unperformable-check-skipped.json +299 -0
- package/contracts/fixtures/runtime-recipe/reject/unpinned-lockfile.json +299 -0
- package/contracts/fixtures/sidecar-bundle/production-shaped/.campaign-runtime/assembly-report.json +180 -0
- package/contracts/fixtures/sidecar-bundle/production-shaped/.campaign-runtime/build-context.json +115 -0
- package/contracts/fixtures/sidecar-bundle/production-shaped/.campaign-runtime/doctor-output.json +29 -0
- package/contracts/fixtures/sidecar-bundle/production-shaped/.campaign-runtime/qa-verdict.json +27 -0
- package/contracts/fixtures/sidecar-bundle/production-shaped/campaign-runtime.build.json +141 -0
- package/contracts/migration-sidecar-bundle.v0.json +149 -0
- package/contracts/orientation-limits.v1.json +41 -0
- package/contracts/orientation-reason-codes.v1.json +196 -0
- package/contracts/private-template-sources.json +8 -0
- package/contracts/release-ledger.json +4598 -0
- package/contracts/reserved-skill-names.json +13 -0
- package/contracts/runtime-recipe.campaigns-os-node-v1.json +299 -0
- package/contracts/supported-surface.json +180 -0
- package/contracts/template-brand-contract.apollo-mv-single-step.v0.json +27 -0
- package/contracts/template-brand-contract.apollo.v0.json +27 -0
- package/contracts/template-brand-contract.demeter.v0.json +27 -0
- package/contracts/template-brand-contract.olympus-mv-single-step.v0.json +27 -0
- package/contracts/template-brand-contract.olympus-mv-two-step.v0.json +28 -0
- package/contracts/template-brand-contract.olympus.v0.json +27 -0
- package/contracts/template-brand-contract.shared-commerce.v0.json +190 -0
- package/contracts/template-brand-contract.shop-single-step.v0.json +27 -0
- package/contracts/template-brand-contract.shop-three-step.v0.json +29 -0
- package/contracts/template-slot-manifest.apollo-mv-single-step.v0.json +14 -0
- package/contracts/template-slot-manifest.apollo.v0.json +12 -0
- package/contracts/template-slot-manifest.demeter.v0.json +30 -0
- package/contracts/template-slot-manifest.olympus-mv-single-step.v0.json +14 -0
- package/contracts/template-slot-manifest.olympus-mv-two-step.v0.json +15 -0
- package/contracts/template-slot-manifest.olympus.v0.json +12 -0
- package/contracts/template-slot-manifest.shared-content-core.v0.json +4254 -0
- package/contracts/template-slot-manifest.shop-single-step.v0.json +47 -0
- package/contracts/template-slot-manifest.shop-three-step.v0.json +33 -0
- package/docs/brand-theme-bridge.md +159 -0
- package/docs/build-packet.md +1300 -0
- package/docs/campaign-build-brief.md +145 -0
- package/docs/campaign-standardization-report.md +329 -0
- package/docs/campaigns-os-build-flow.md +117 -0
- package/docs/design-source-package.md +784 -0
- package/docs/legacy-migration.md +58 -0
- package/docs/migration-sidecar-bundle.md +139 -0
- package/docs/orientation-contract-reference.md +1220 -0
- package/docs/polish-evidence.md +502 -0
- package/docs/qa-and-test-orders.md +1691 -0
- package/docs/release-ledger-authoring-guide.md +274 -0
- package/docs/runtime-readiness.md +211 -0
- package/docs/supported-surface.md +83 -0
- package/docs/versioning.md +55 -0
- package/docs/workflow-findings-sidecar.md +588 -0
- package/package.json +135 -0
- package/prompts/first-build.md +27 -0
- package/prompts/friction-log.md +30 -0
- package/schemas/campaign-build-brief.v1.schema.json +149 -0
- package/schemas/campaign-design-source-package.v0.schema.json +697 -0
- package/schemas/campaign-runtime-assembly-report.v0.schema.json +390 -0
- package/schemas/campaign-runtime-build-context.v0.schema.json +339 -0
- package/schemas/campaign-runtime-build-packet.v0.schema.json +452 -0
- package/schemas/campaign-spec.v4.schema.json +582 -0
- package/schemas/campaigns-os-doctor-output.v0.schema.json +43 -0
- package/schemas/campaigns-os-legacy-migration-inventory.v0.schema.json +112 -0
- package/schemas/campaigns-os-legacy-provisioning-plan.v0.schema.json +38 -0
- package/schemas/campaigns-os-legacy-provisioning-receipt.v0.schema.json +57 -0
- package/schemas/campaigns-os-qa-verdict-sidecar.v0.schema.json +150 -0
- package/schemas/campaigns-os-qa-verdict.v0.schema.json +438 -0
- package/schemas/campaigns-os-release-ledger.v1.schema.json +162 -0
- package/schemas/campaigns-os-run-record.v0.schema.json +343 -0
- package/schemas/campaigns-os-runtime-recipe.v1.schema.json +313 -0
- package/schemas/campaigns-os-sidecar-bundle-conformance.v0.schema.json +78 -0
- package/schemas/campaigns-os-tooling-orientation.v1.schema.json +381 -0
- package/schemas/campaigns-os-workflow-finding.v0.schema.json +134 -0
- package/schemas/source-html-manifest.v0.schema.json +252 -0
- package/skills/next-campaigns-build/SKILL.md +73 -0
- package/skills/next-campaigns-os/SKILL.md +101 -0
- package/skills/next-campaigns-os/references/session-intake.md +160 -0
- package/skills/next-campaigns-os-setup/SKILL.md +22 -0
- package/skills/next-campaigns-polish/SKILL.md +145 -0
- package/skills/next-campaigns-qa/SKILL.md +92 -0
- package/skills.json +56 -0
- package/skills.sh +64 -0
- package/src/adapter-decision-contract.mjs +333 -0
- package/src/brand-theme.mjs +1151 -0
- package/src/browser-launch.mjs +79 -0
- package/src/build-brief.mjs +781 -0
- package/src/built-site-scope.mjs +312 -0
- package/src/campaign-ecosystem.mjs +734 -0
- package/src/campaign-identity.mjs +405 -0
- package/src/campaign-workspace.mjs +127 -0
- package/src/checkpoint-waiver.mjs +302 -0
- package/src/cli.mjs +13442 -0
- package/src/commercial-journey.mjs +1119 -0
- package/src/commercial-parity.mjs +965 -0
- package/src/consent.mjs +347 -0
- package/src/content-residue.mjs +322 -0
- package/src/deadline.mjs +81 -0
- package/src/design-source-package.mjs +2604 -0
- package/src/deviation.mjs +107 -0
- package/src/doctor-check-registry.mjs +49 -0
- package/src/doctor-sidecar.mjs +106 -0
- package/src/finding-cause.mjs +557 -0
- package/src/findings.mjs +326 -0
- package/src/fs-identity.mjs +68 -0
- package/src/gate-actions.mjs +105 -0
- package/src/html-scan.mjs +53 -0
- package/src/install-mode.mjs +272 -0
- package/src/legacy-migration.d.ts +128 -0
- package/src/legacy-migration.mjs +510 -0
- package/src/lifecycle.mjs +338 -0
- package/src/local-proof.mjs +401 -0
- package/src/map-pin-writeback.mjs +210 -0
- package/src/orchestration-stage-contract.mjs +81 -0
- package/src/package-install-fixture.mjs +33 -0
- package/src/page-kit-build-summary.mjs +175 -0
- package/src/page-kit-campaign-config.mjs +57 -0
- package/src/page-kit-sdk-version.mjs +392 -0
- package/src/page-kit-store-profile.mjs +369 -0
- package/src/page-kit-sync.mjs +162 -0
- package/src/polish-browser.mjs +867 -0
- package/src/polish-capture.mjs +1094 -0
- package/src/polish-deadline.mjs +51 -0
- package/src/polish-gate.mjs +739 -0
- package/src/polish-node.mjs +639 -0
- package/src/polish-page-load.mjs +1100 -0
- package/src/private-template-source.mjs +237 -0
- package/src/proof-policy.mjs +82 -0
- package/src/qa-analytics-correctness.mjs +307 -0
- package/src/qa-analytics-errors.mjs +38 -0
- package/src/qa-analytics-parity.mjs +699 -0
- package/src/qa-binding-evidence.mjs +140 -0
- package/src/qa-browser.mjs +6608 -0
- package/src/qa-cart-entry.mjs +406 -0
- package/src/qa-commercial-parity.mjs +641 -0
- package/src/qa-node.mjs +3620 -0
- package/src/qa-order-bump.mjs +381 -0
- package/src/qa-parity-capture.mjs +428 -0
- package/src/qa-parity-fixture.mjs +359 -0
- package/src/qa-publish.mjs +362 -0
- package/src/qa-purchase-data-layer.mjs +263 -0
- package/src/qa-route-probe.mjs +272 -0
- package/src/qa-sidecar.mjs +188 -0
- package/src/qa-test-order-topology.mjs +207 -0
- package/src/qa-url-privacy.mjs +13 -0
- package/src/qa-verdict-discovery.mjs +192 -0
- package/src/qa-verdict-publish.mjs +105 -0
- package/src/qa-verdict.mjs +287 -0
- package/src/remit.mjs +388 -0
- package/src/repo-scan.mjs +83 -0
- package/src/route-identity.mjs +133 -0
- package/src/run-record-closeout.mjs +229 -0
- package/src/run-record.mjs +839 -0
- package/src/run-session.mjs +226 -0
- package/src/runtime-state-ignore.mjs +113 -0
- package/src/sdk-attribute-index.mjs +212 -0
- package/src/sdk-markup.mjs +358 -0
- package/src/sdk-meta-tags.mjs +52 -0
- package/src/shell-token.mjs +7 -0
- package/src/sidecar-bundle.mjs +399 -0
- package/src/source-asset-crawl.mjs +469 -0
- package/src/source-html-intake.mjs +627 -0
- package/src/source-html-manifest.mjs +276 -0
- package/src/source-prep.mjs +284 -0
- package/src/spec-derive-store.mjs +431 -0
- package/src/spec-derive.mjs +514 -0
- package/src/spec-fetch.mjs +66 -0
- package/src/spec-hash.mjs +48 -0
- package/src/spec-identity.mjs +27 -0
- package/src/stage-ledger.mjs +509 -0
- package/src/standardization-report.mjs +1297 -0
- package/src/template-brand-contract.mjs +473 -0
- package/src/template-freshness.mjs +196 -0
- package/src/template-reference.mjs +81 -0
- package/src/template-slot-manifest.mjs +150 -0
- package/src/text-safety.mjs +62 -0
- package/src/theme-gate.mjs +185 -0
- package/src/upsell-selector-scope.mjs +299 -0
|
@@ -0,0 +1,1094 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
function normalizeString(value) {
|
|
4
|
+
return typeof value === "string" && value.trim() ? value.trim() : null;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
function normalizedToken(value) {
|
|
8
|
+
return normalizeString(value)?.toLocaleLowerCase("en-US") || "";
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function isPlainObject(value) {
|
|
12
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export const POLISH_ROUTE_CAPTURE_SCHEMA_VERSION = "campaigns-os-polish-route-capture/v0";
|
|
16
|
+
export const POLISH_CAPTURE_PRODUCER = "campaigns-os polish capture";
|
|
17
|
+
export const POLISH_CAPTURE_INTEGRITY_SCHEMA_VERSION = "campaigns-os-polish-route-capture-integrity/v0";
|
|
18
|
+
export const POLISH_CAPTURE_INTEGRITY_ALGORITHM = "sha256";
|
|
19
|
+
export const MAX_PAGE_LOAD_RESOURCE_LEDGER_ENTRIES = 2_048;
|
|
20
|
+
export const MAX_PAGE_LOAD_RESPONSE_RECORDS = 4_096;
|
|
21
|
+
export const MAX_PAGE_LOAD_MEDIA_ELEMENTS = 512;
|
|
22
|
+
export const MAX_PAGE_LOAD_MEDIA_SOURCES_PER_ELEMENT = 32;
|
|
23
|
+
export const MAX_PAGE_LOAD_MEDIA_ANCESTORS = 64;
|
|
24
|
+
export const MAX_POLISH_CAPTURE_URL_LENGTH = 8_192;
|
|
25
|
+
export const POLISH_RESOURCE_TYPES = Object.freeze([
|
|
26
|
+
"cspviolationreport",
|
|
27
|
+
"document",
|
|
28
|
+
"eventsource",
|
|
29
|
+
"fetch",
|
|
30
|
+
"font",
|
|
31
|
+
"image",
|
|
32
|
+
"manifest",
|
|
33
|
+
"media",
|
|
34
|
+
"other",
|
|
35
|
+
"ping",
|
|
36
|
+
"prefetch",
|
|
37
|
+
"preflight",
|
|
38
|
+
"script",
|
|
39
|
+
"signedexchange",
|
|
40
|
+
"stylesheet",
|
|
41
|
+
"texttrack",
|
|
42
|
+
"unknown",
|
|
43
|
+
"websocket",
|
|
44
|
+
"xhr",
|
|
45
|
+
]);
|
|
46
|
+
export const POLISH_PRELOAD_ATTRIBUTES = Object.freeze([
|
|
47
|
+
"auto",
|
|
48
|
+
"empty",
|
|
49
|
+
"metadata",
|
|
50
|
+
"missing",
|
|
51
|
+
"none",
|
|
52
|
+
"other",
|
|
53
|
+
]);
|
|
54
|
+
|
|
55
|
+
const KNOWN_RESOURCE_TYPES = new Set(POLISH_RESOURCE_TYPES.filter((value) => value !== "unknown"));
|
|
56
|
+
// Beacon-class resource roles: one-shot programmatic requests whose failure
|
|
57
|
+
// says nothing about what the page renders. This is a deliberate allowlist.
|
|
58
|
+
// A failed cross-origin request in one of these roles is a warning; a failed
|
|
59
|
+
// request in any other role — document, script, stylesheet, image, font,
|
|
60
|
+
// media, and also the roles that are not beacons even though nothing renders
|
|
61
|
+
// from them (a caption track, a manifest, an event stream, a CSP report
|
|
62
|
+
// endpoint, a prefetch hint, a signed exchange, a websocket) — blocks
|
|
63
|
+
// whatever its origin, as does an unknown or ambiguous type. Widening this
|
|
64
|
+
// list is an operator-visible trade-off, not a tidy-up.
|
|
65
|
+
export const POLISH_BEACON_RESOURCE_TYPES = Object.freeze([
|
|
66
|
+
"fetch",
|
|
67
|
+
"other",
|
|
68
|
+
"ping",
|
|
69
|
+
"preflight",
|
|
70
|
+
"xhr",
|
|
71
|
+
]);
|
|
72
|
+
const BEACON_RESOURCE_TYPES = new Set(POLISH_BEACON_RESOURCE_TYPES);
|
|
73
|
+
// Problem codes that are recorded on the capture but do not make it
|
|
74
|
+
// incomplete: they carry evidence the operator should see without turning
|
|
75
|
+
// measured-complete evidence into an unwaivable block.
|
|
76
|
+
export const POLISH_CAPTURE_WARNING_PROBLEM_CODES = Object.freeze([
|
|
77
|
+
"cross_origin_request_failed",
|
|
78
|
+
]);
|
|
79
|
+
const WARNING_PROBLEM_CODES = new Set(POLISH_CAPTURE_WARNING_PROBLEM_CODES);
|
|
80
|
+
|
|
81
|
+
export function polishCaptureMeasurementStatus(problems) {
|
|
82
|
+
return (Array.isArray(problems) ? problems : []).some((problem) => !WARNING_PROBLEM_CODES.has(problem?.code))
|
|
83
|
+
? "incomplete"
|
|
84
|
+
: "complete";
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// A failed request is attributed to a class by origin and role. Failed
|
|
88
|
+
// requests grouped under one ledger entry share both, so the class can be
|
|
89
|
+
// recomputed from the entry alone (polish-page-load.mjs does exactly that).
|
|
90
|
+
export function failedRequestProblemCode({ crossOrigin, resourceType }) {
|
|
91
|
+
return crossOrigin && BEACON_RESOURCE_TYPES.has(resourceType)
|
|
92
|
+
? "cross_origin_request_failed"
|
|
93
|
+
: "dependency_request_failed";
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// The signature of a production build served over plain HTTP: the document
|
|
97
|
+
// came from an http: origin and a dependency-class request to a DIFFERENT
|
|
98
|
+
// http: origin failed. A protocol-relative vendor loader (`//host/...`, which
|
|
99
|
+
// the starter templates emit outside the development environment) resolves to
|
|
100
|
+
// http://host/... off an http://localhost document and fails there, voiding
|
|
101
|
+
// the capture. Read off the finished ledger, so the recorded checkpoint can
|
|
102
|
+
// name the right action (rebuild in local proof mode) without re-reading the
|
|
103
|
+
// browser. Returns the failing ledger entries, empty when the signature is
|
|
104
|
+
// absent — including for an https: capture, where a failed http: dependency
|
|
105
|
+
// is a page defect rather than an environment mismatch.
|
|
106
|
+
export function plainHttpDependencyFailures(capture) {
|
|
107
|
+
const captureOrigin = capture?.document_response?.capture_origin;
|
|
108
|
+
if (typeof captureOrigin !== "string" || !captureOrigin.startsWith("http://")) return [];
|
|
109
|
+
const entries = Array.isArray(capture?.resource_ledger?.entries) ? capture.resource_ledger.entries : [];
|
|
110
|
+
return entries.filter((resource) => (
|
|
111
|
+
(resource?.failed_request_count || 0) > 0
|
|
112
|
+
&& (resource?.cross_origin_request_count || 0) > 0
|
|
113
|
+
&& typeof resource?.url === "string"
|
|
114
|
+
&& resource.url.startsWith("http://")
|
|
115
|
+
&& failedRequestProblemCode({ crossOrigin: true, resourceType: resource.resource_type }) === "dependency_request_failed"
|
|
116
|
+
));
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function resolvedCaptureUrl(value, { baseUrl } = {}) {
|
|
120
|
+
if (value === "[url-too-long]" || baseUrl === "[url-too-long]") {
|
|
121
|
+
return { status: "too_long", canonical: null, projected: "[url-too-long]", origin: null };
|
|
122
|
+
}
|
|
123
|
+
if ((typeof value === "string" && value.length > MAX_POLISH_CAPTURE_URL_LENGTH)
|
|
124
|
+
|| (typeof baseUrl === "string" && baseUrl.length > MAX_POLISH_CAPTURE_URL_LENGTH)) {
|
|
125
|
+
return { status: "too_long", canonical: null, projected: "[url-too-long]", origin: null };
|
|
126
|
+
}
|
|
127
|
+
const text = normalizeString(value);
|
|
128
|
+
if (!text) return { status: "empty", canonical: null, projected: null, origin: null };
|
|
129
|
+
try {
|
|
130
|
+
const url = baseUrl ? new URL(text, baseUrl) : new URL(text);
|
|
131
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
132
|
+
return { status: "non_http", canonical: null, projected: "[non-http-url]", origin: null };
|
|
133
|
+
}
|
|
134
|
+
url.hash = "";
|
|
135
|
+
url.username = "";
|
|
136
|
+
url.password = "";
|
|
137
|
+
return {
|
|
138
|
+
status: "http",
|
|
139
|
+
canonical: url.href,
|
|
140
|
+
projected: `${url.origin}${url.pathname}`,
|
|
141
|
+
origin: url.origin,
|
|
142
|
+
};
|
|
143
|
+
} catch {
|
|
144
|
+
return { status: "malformed", canonical: null, projected: "[malformed-url]", origin: null };
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export function redactCaptureUrl(value, { baseUrl } = {}) {
|
|
149
|
+
return resolvedCaptureUrl(value, { baseUrl }).projected;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// The one "HTTP(S) origin or null" parser. Every reader that wants a capture
|
|
153
|
+
// URL's origin — the producer, the validator, the text renderer, the browser
|
|
154
|
+
// adapter and the parity capture's credential guard — calls this, so the
|
|
155
|
+
// scheme rule and the length cap exist once. Never throws and never echoes
|
|
156
|
+
// the value: a URL carrying credentials or query data must not reach a log.
|
|
157
|
+
export function captureOrigin(value) {
|
|
158
|
+
if (typeof value !== "string" || value.length > MAX_POLISH_CAPTURE_URL_LENGTH) return null;
|
|
159
|
+
try {
|
|
160
|
+
const url = new URL(value);
|
|
161
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url.origin : null;
|
|
162
|
+
} catch {
|
|
163
|
+
return null;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function resourceId(canonicalUrl) {
|
|
168
|
+
return `sha256:${createHash("sha256").update(canonicalUrl).digest("hex")}`;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Key-sorted deep copy, so two captures that state the same facts serialize
|
|
172
|
+
// to the same bytes. No cycle guard, deliberately: values reach this module
|
|
173
|
+
// from fresh literals or from JSON.parse, and neither can carry a cycle.
|
|
174
|
+
function canonicalize(value) {
|
|
175
|
+
if (Array.isArray(value)) return value.map(canonicalize);
|
|
176
|
+
if (value && typeof value === "object") {
|
|
177
|
+
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]));
|
|
178
|
+
}
|
|
179
|
+
return value;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export function canonicalJson(value) {
|
|
183
|
+
return JSON.stringify(canonicalize(value));
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function integrityFingerprint(value) {
|
|
187
|
+
return `sha256:${createHash("sha256").update(canonicalJson(value)).digest("hex")}`;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function captureAssociationProjection(capture) {
|
|
191
|
+
return {
|
|
192
|
+
resources: (Array.isArray(capture?.resource_ledger?.entries) ? capture.resource_ledger.entries : []).map((resource) => ({
|
|
193
|
+
resource_id: resource?.resource_id,
|
|
194
|
+
match_resource_ids: Array.isArray(resource?.match_resource_ids) ? resource.match_resource_ids : [],
|
|
195
|
+
})),
|
|
196
|
+
media: (Array.isArray(capture?.media) ? capture.media : []).map((media) => ({
|
|
197
|
+
element_index: media?.element_index,
|
|
198
|
+
source_references: (Array.isArray(media?.source_references) ? media.source_references : []).map((reference) => ({
|
|
199
|
+
source_kind: reference?.source_kind,
|
|
200
|
+
source_index: reference?.source_index,
|
|
201
|
+
resource_id: reference?.resource_id,
|
|
202
|
+
})),
|
|
203
|
+
fetched_resources: (Array.isArray(media?.fetched_resources) ? media.fetched_resources : []).map((resource) => ({
|
|
204
|
+
resource_id: resource?.resource_id,
|
|
205
|
+
matched_source_resource_ids: Array.isArray(resource?.matched_source_resource_ids)
|
|
206
|
+
? resource.matched_source_resource_ids
|
|
207
|
+
: [],
|
|
208
|
+
})),
|
|
209
|
+
})),
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export function buildPolishCaptureIntegrity(captureProjection) {
|
|
214
|
+
const { integrity: ignoredIntegrity, ...projection } = captureProjection || {};
|
|
215
|
+
const associationFingerprint = integrityFingerprint(captureAssociationProjection(projection));
|
|
216
|
+
// This is a versioned tamper-evidence checksum for package-produced artifacts, not a keyed
|
|
217
|
+
// signature. A malicious local editor who rewrites the artifact and recalculates both hashes can
|
|
218
|
+
// forge it; its job is to expose accidental or partial mutation between capture and evaluation.
|
|
219
|
+
return {
|
|
220
|
+
schema_version: POLISH_CAPTURE_INTEGRITY_SCHEMA_VERSION,
|
|
221
|
+
algorithm: POLISH_CAPTURE_INTEGRITY_ALGORITHM,
|
|
222
|
+
association_fingerprint: associationFingerprint,
|
|
223
|
+
projection_fingerprint: integrityFingerprint({
|
|
224
|
+
schema_version: POLISH_CAPTURE_INTEGRITY_SCHEMA_VERSION,
|
|
225
|
+
association_fingerprint: associationFingerprint,
|
|
226
|
+
capture: projection,
|
|
227
|
+
}),
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function resolvedResource(value, { baseUrl } = {}) {
|
|
232
|
+
const resolved = resolvedCaptureUrl(value, { baseUrl });
|
|
233
|
+
return {
|
|
234
|
+
...resolved,
|
|
235
|
+
resource_id: resolved.canonical ? resourceId(resolved.canonical) : null,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function validComputedStyle(style) {
|
|
240
|
+
return isPlainObject(style)
|
|
241
|
+
&& typeof style.display === "string"
|
|
242
|
+
&& style.display.trim() !== ""
|
|
243
|
+
&& typeof style.visibility === "string"
|
|
244
|
+
&& style.visibility.trim() !== "";
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function addProblemCount(problemCounts, code, count = 1) {
|
|
248
|
+
if (!Number.isInteger(count) || count <= 0) return;
|
|
249
|
+
problemCounts.set(code, (problemCounts.get(code) || 0) + count);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function projectedProblems(problemCounts) {
|
|
253
|
+
return [...problemCounts]
|
|
254
|
+
.map(([code, count]) => ({ code, count }))
|
|
255
|
+
.sort((a, b) => a.code.localeCompare(b.code));
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function normalizeResourceType(value) {
|
|
259
|
+
const token = normalizedToken(value);
|
|
260
|
+
return KNOWN_RESOURCE_TYPES.has(token)
|
|
261
|
+
? { value: token, status: "known" }
|
|
262
|
+
: { value: "unknown", status: "unknown" };
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// A ledger entry's resource_type_status is "known" exactly when its type is a
|
|
266
|
+
// named role; "unknown" and "ambiguous" both carry the type "unknown".
|
|
267
|
+
export function resourceTypeKnown(resourceType) {
|
|
268
|
+
return KNOWN_RESOURCE_TYPES.has(resourceType);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// ---------------------------------------------------------------------------
|
|
272
|
+
// The response record: the one shape the browser collector
|
|
273
|
+
// (polish-browser.mjs, createNetworkCollector) emits into `responses[]` and
|
|
274
|
+
// the aggregator below reads. There is exactly one producer, so the
|
|
275
|
+
// aggregator trusts what these constructors build — hop numbering, request
|
|
276
|
+
// identity and sentinel spelling are decided here, once, and never
|
|
277
|
+
// re-validated on the way in. Tests that hand-build records go through the
|
|
278
|
+
// same constructors. The shape is documented in docs/polish-evidence.md.
|
|
279
|
+
//
|
|
280
|
+
// captureProblemRecord(code) { capture_problem: code }
|
|
281
|
+
// singleResponseRecord(id, response) { request_id: id, ...response }
|
|
282
|
+
// redirectChainRecord(id, responses) { request_id: id, redirect_chain:
|
|
283
|
+
// [{ ...response, redirect_hop: 0 }, …] }
|
|
284
|
+
//
|
|
285
|
+
// A response carries `url`, `resource_type`, `status`, `mime_type`,
|
|
286
|
+
// `encoded_data_length` / `declared_data_length`, `canceled`, `source_urls`,
|
|
287
|
+
// the three cache flags below, `from_service_worker` and `failed`; the final
|
|
288
|
+
// main document additionally carries `is_final_main_document` and
|
|
289
|
+
// `document_context_fingerprint`.
|
|
290
|
+
// ---------------------------------------------------------------------------
|
|
291
|
+
|
|
292
|
+
// The cache observations the collector projects from CDP. A response is
|
|
293
|
+
// cache-served when any of them is true; no other spelling is read.
|
|
294
|
+
export const POLISH_RESPONSE_CACHE_FLAGS = Object.freeze([
|
|
295
|
+
"from_disk_cache",
|
|
296
|
+
"from_prefetch_cache",
|
|
297
|
+
"request_served_from_cache",
|
|
298
|
+
]);
|
|
299
|
+
|
|
300
|
+
// In-band sentinels: a record that carries a problem instead of a response.
|
|
301
|
+
// The collector emits `response_record_overflow` when it dropped responses
|
|
302
|
+
// past MAX_PAGE_LOAD_RESPONSE_RECORDS and `document_context_changed` when the
|
|
303
|
+
// main frame or loader changed under the capture.
|
|
304
|
+
export const POLISH_CAPTURE_SENTINEL_PROBLEM_CODES = Object.freeze([
|
|
305
|
+
"document_context_changed",
|
|
306
|
+
"response_record_overflow",
|
|
307
|
+
]);
|
|
308
|
+
const CAPTURE_SENTINEL_PROBLEM_CODES = new Set(POLISH_CAPTURE_SENTINEL_PROBLEM_CODES);
|
|
309
|
+
|
|
310
|
+
export function captureProblemRecord(code) {
|
|
311
|
+
if (!CAPTURE_SENTINEL_PROBLEM_CODES.has(code)) {
|
|
312
|
+
throw new Error(`Unknown polish capture sentinel problem code: ${String(code)}`);
|
|
313
|
+
}
|
|
314
|
+
return { capture_problem: code };
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
export function captureProblemRecordCode(record) {
|
|
318
|
+
return isPlainObject(record) && CAPTURE_SENTINEL_PROBLEM_CODES.has(record.capture_problem)
|
|
319
|
+
? record.capture_problem
|
|
320
|
+
: null;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// The constructors are the one place a response is checked to be a record:
|
|
324
|
+
// a non-object hop is a producer bug, so it throws here rather than being
|
|
325
|
+
// read as an observation downstream.
|
|
326
|
+
function assertResponse(response) {
|
|
327
|
+
if (!isPlainObject(response)) {
|
|
328
|
+
throw new Error("A polish response record is built from a plain response object.");
|
|
329
|
+
}
|
|
330
|
+
return response;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
export function singleResponseRecord(requestId, response) {
|
|
334
|
+
return { request_id: requestId, ...assertResponse(response) };
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
export function redirectChainRecord(requestId, responses) {
|
|
338
|
+
return {
|
|
339
|
+
request_id: requestId,
|
|
340
|
+
redirect_chain: responses.map((response, redirectHop) => ({ ...assertResponse(response), redirect_hop: redirectHop })),
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// The responses a record stands for, in transfer order: a redirect chain's
|
|
345
|
+
// hops, or the single response itself.
|
|
346
|
+
export function responseRecordResponses(record) {
|
|
347
|
+
return isPlainObject(record) && Array.isArray(record.redirect_chain) ? record.redirect_chain : [record];
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
export function responseRecordFromCache(record) {
|
|
351
|
+
return POLISH_RESPONSE_CACHE_FLAGS.some((flag) => record?.[flag] === true);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// Every observed response with the chain it belongs to, bounded to
|
|
355
|
+
// MAX_PAGE_LOAD_RESPONSE_RECORDS; sentinel records become problem counts.
|
|
356
|
+
function prepareResponseRecords(responses, problemCounts) {
|
|
357
|
+
const prepared = [];
|
|
358
|
+
let omittedRecordCount = 0;
|
|
359
|
+
for (const record of (Array.isArray(responses) ? responses : [])) {
|
|
360
|
+
const sentinel = captureProblemRecordCode(record);
|
|
361
|
+
if (sentinel) {
|
|
362
|
+
addProblemCount(problemCounts, sentinel);
|
|
363
|
+
continue;
|
|
364
|
+
}
|
|
365
|
+
const chain = responseRecordResponses(record);
|
|
366
|
+
for (const [index, response] of chain.entries()) {
|
|
367
|
+
if (prepared.length >= MAX_PAGE_LOAD_RESPONSE_RECORDS) {
|
|
368
|
+
omittedRecordCount += chain.length - index;
|
|
369
|
+
break;
|
|
370
|
+
}
|
|
371
|
+
prepared.push({ record: response, chain });
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
if (omittedRecordCount > 0) {
|
|
375
|
+
addProblemCount(problemCounts, "response_record_overflow", omittedRecordCount);
|
|
376
|
+
}
|
|
377
|
+
return prepared;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
// The derivations below are the producer's statements about a resource
|
|
381
|
+
// ledger; the ones the page-load validator recomputes are exported so it
|
|
382
|
+
// reads the same function instead of a second spelling of it. They take
|
|
383
|
+
// ledger entries (or media projections) and nothing else.
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
export function resourceLedgerSort(a, b) {
|
|
387
|
+
return String(a.url).localeCompare(String(b.url)) || String(a.resource_id).localeCompare(String(b.resource_id));
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function largestResourceProjection(resources) {
|
|
391
|
+
const largest = [...resources].sort((a, b) => b.transferred_bytes - a.transferred_bytes || resourceLedgerSort(a, b))[0];
|
|
392
|
+
return largest ? {
|
|
393
|
+
resource_id: largest.resource_id,
|
|
394
|
+
url: largest.url,
|
|
395
|
+
resource_type: largest.resource_type,
|
|
396
|
+
transferred_bytes: largest.transferred_bytes,
|
|
397
|
+
request_count: largest.request_count,
|
|
398
|
+
} : null;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
// The capture's metrics block, summed over the retained ledger entries.
|
|
402
|
+
export function resourceLedgerMetrics(resources) {
|
|
403
|
+
const sum = (field) => resources.reduce((total, resource) => total + resource[field], 0);
|
|
404
|
+
return {
|
|
405
|
+
total_transferred_bytes: sum("transferred_bytes"),
|
|
406
|
+
request_count: sum("request_count"),
|
|
407
|
+
largest_resource: largestResourceProjection(resources),
|
|
408
|
+
cross_origin_request_count: sum("cross_origin_request_count"),
|
|
409
|
+
cache_request_count: sum("cache_request_count"),
|
|
410
|
+
service_worker_request_count: sum("service_worker_request_count"),
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// The problem counts a ledger implies on its own: one per cached, service-
|
|
415
|
+
// worker and unmeasured request, one per ambiguous or unknown-typed entry,
|
|
416
|
+
// and the failed requests attributed by origin and role. Every request under
|
|
417
|
+
// one entry shares the entry's origin relation and resolved type, so the
|
|
418
|
+
// entry decides the class for all of its failed requests.
|
|
419
|
+
export function ledgerProblemCounts(entries) {
|
|
420
|
+
const counts = {
|
|
421
|
+
cache_observed: 0,
|
|
422
|
+
service_worker_observed: 0,
|
|
423
|
+
transfer_size_unavailable: 0,
|
|
424
|
+
resource_type_ambiguous: 0,
|
|
425
|
+
resource_type_unknown: 0,
|
|
426
|
+
cross_origin_request_failed: 0,
|
|
427
|
+
dependency_request_failed: 0,
|
|
428
|
+
};
|
|
429
|
+
for (const resource of entries) {
|
|
430
|
+
counts.cache_observed += resource.cache_request_count;
|
|
431
|
+
counts.service_worker_observed += resource.service_worker_request_count;
|
|
432
|
+
counts.transfer_size_unavailable += resource.unmeasured_request_count;
|
|
433
|
+
if (resource.resource_type_status === "ambiguous") counts.resource_type_ambiguous += 1;
|
|
434
|
+
if (resource.resource_type_status === "unknown") counts.resource_type_unknown += 1;
|
|
435
|
+
if (resource.failed_request_count > 0) {
|
|
436
|
+
counts[failedRequestProblemCode({
|
|
437
|
+
crossOrigin: resource.cross_origin_request_count > 0,
|
|
438
|
+
resourceType: resource.resource_type,
|
|
439
|
+
})] += resource.failed_request_count;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
return counts;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function documentResponseIdentity(value, { baseUrl } = {}) {
|
|
446
|
+
const resolved = resolvedCaptureUrl(value, { baseUrl });
|
|
447
|
+
if (resolved.status !== "http" || !resolved.canonical) return null;
|
|
448
|
+
try {
|
|
449
|
+
const url = new URL(resolved.canonical);
|
|
450
|
+
url.hash = "";
|
|
451
|
+
return url.href;
|
|
452
|
+
} catch {
|
|
453
|
+
return null;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
function normalizedDocumentMimeType(value) {
|
|
458
|
+
const token = typeof value === "string" ? value.split(";", 1)[0].trim().toLowerCase() : "";
|
|
459
|
+
if (token === "text/html") return "html";
|
|
460
|
+
if (token === "application/xhtml+xml") return "xhtml";
|
|
461
|
+
return token ? "other" : "unknown";
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// Whether the document the browser settled on shares the origin the capture
|
|
465
|
+
// was asked for; null on either side is a mismatch.
|
|
466
|
+
export function originMatchesCapture(captureOrigin, finalOrigin) {
|
|
467
|
+
return Boolean(captureOrigin && finalOrigin && captureOrigin === finalOrigin);
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// A final document response is complete only as a 200 HTML/XHTML page on the
|
|
471
|
+
// requested origin; anything else is an error.
|
|
472
|
+
export function documentResponseAcceptable({ http_status, mime_type, origin_matches_capture }) {
|
|
473
|
+
return http_status === 200
|
|
474
|
+
&& (mime_type === "html" || mime_type === "xhtml")
|
|
475
|
+
&& origin_matches_capture === true;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
function assessFinalDocumentResponse(prepared, {
|
|
479
|
+
documentUrl,
|
|
480
|
+
requestedDocumentUrl,
|
|
481
|
+
problemCounts,
|
|
482
|
+
}) {
|
|
483
|
+
const finalIdentity = documentResponseIdentity(documentUrl);
|
|
484
|
+
const projectedUrl = redactCaptureUrl(documentUrl);
|
|
485
|
+
const requestedOrigin = captureOrigin(requestedDocumentUrl);
|
|
486
|
+
const finalOrigin = captureOrigin(documentUrl);
|
|
487
|
+
const empty = (status) => ({
|
|
488
|
+
status,
|
|
489
|
+
url: projectedUrl,
|
|
490
|
+
resource_id: null,
|
|
491
|
+
http_status: null,
|
|
492
|
+
mime_type: "unknown",
|
|
493
|
+
context_fingerprint: null,
|
|
494
|
+
capture_origin: requestedOrigin,
|
|
495
|
+
final_origin: finalOrigin,
|
|
496
|
+
origin_matches_capture: originMatchesCapture(requestedOrigin, finalOrigin),
|
|
497
|
+
});
|
|
498
|
+
if (!finalIdentity) {
|
|
499
|
+
addProblemCount(problemCounts, "document_response_missing");
|
|
500
|
+
return empty("missing");
|
|
501
|
+
}
|
|
502
|
+
const matches = prepared.filter(({ record }) => {
|
|
503
|
+
const resourceType = normalizeResourceType(record?.resource_type);
|
|
504
|
+
return record?.is_final_main_document === true
|
|
505
|
+
&& /^sha256:[a-f0-9]{64}$/.test(record?.document_context_fingerprint || "")
|
|
506
|
+
&& resourceType.value === "document"
|
|
507
|
+
&& resourceType.status === "known"
|
|
508
|
+
&& documentResponseIdentity(record?.url, { baseUrl: documentUrl }) === finalIdentity;
|
|
509
|
+
});
|
|
510
|
+
if (matches.length === 0) {
|
|
511
|
+
addProblemCount(problemCounts, "document_response_missing");
|
|
512
|
+
return empty("missing");
|
|
513
|
+
}
|
|
514
|
+
if (matches.length > 1) {
|
|
515
|
+
addProblemCount(problemCounts, "document_response_ambiguous");
|
|
516
|
+
return empty("ambiguous");
|
|
517
|
+
}
|
|
518
|
+
const record = matches[0].record;
|
|
519
|
+
const resolved = resolvedResource(record.url, { baseUrl: documentUrl });
|
|
520
|
+
const projection = {
|
|
521
|
+
status: "complete",
|
|
522
|
+
url: resolved.projected,
|
|
523
|
+
resource_id: resolved.resource_id,
|
|
524
|
+
http_status: Number.isInteger(record?.status) ? record.status : null,
|
|
525
|
+
mime_type: normalizedDocumentMimeType(record?.mime_type),
|
|
526
|
+
context_fingerprint: record.document_context_fingerprint,
|
|
527
|
+
capture_origin: requestedOrigin,
|
|
528
|
+
final_origin: finalOrigin,
|
|
529
|
+
origin_matches_capture: originMatchesCapture(requestedOrigin, finalOrigin),
|
|
530
|
+
};
|
|
531
|
+
if (record?.failed === true || !documentResponseAcceptable(projection)) {
|
|
532
|
+
addProblemCount(problemCounts, "document_response_error");
|
|
533
|
+
projection.status = "error";
|
|
534
|
+
}
|
|
535
|
+
return projection;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
export function aggregateCdpResponses(responses, {
|
|
539
|
+
documentUrl,
|
|
540
|
+
requestedDocumentUrl,
|
|
541
|
+
requireFinalDocumentResponse = false,
|
|
542
|
+
} = {}) {
|
|
543
|
+
const problemCounts = new Map();
|
|
544
|
+
const documentOrigin = captureOrigin(documentUrl);
|
|
545
|
+
const groups = new Map();
|
|
546
|
+
let unattributedRequestCount = 0;
|
|
547
|
+
const prepared = prepareResponseRecords(responses, problemCounts);
|
|
548
|
+
const documentResponse = requireFinalDocumentResponse
|
|
549
|
+
? assessFinalDocumentResponse(prepared, { documentUrl, requestedDocumentUrl, problemCounts })
|
|
550
|
+
: null;
|
|
551
|
+
|
|
552
|
+
for (const { record, chain } of prepared) {
|
|
553
|
+
const resolved = resolvedResource(record?.url, { baseUrl: documentUrl });
|
|
554
|
+
if (resolved.status === "non_http" && record?.failed !== true) {
|
|
555
|
+
// A data:, blob: or about: response is not a network resource: nothing
|
|
556
|
+
// was transferred, so there is nothing to attribute to the ledger and
|
|
557
|
+
// nothing the operator could repair. It is counted as unattributed
|
|
558
|
+
// (evidence) and is not a problem. Video controls, inline icons and
|
|
559
|
+
// authored data: images produce these on ordinary pages. A non-http
|
|
560
|
+
// load that failed (a revoked blob: URL behind a script or image) is
|
|
561
|
+
// still a dependency the page could not load; it has no ledger entry
|
|
562
|
+
// to carry the failure, so it stays an unresolvable response below.
|
|
563
|
+
unattributedRequestCount += 1;
|
|
564
|
+
continue;
|
|
565
|
+
}
|
|
566
|
+
if (resolved.status !== "http") {
|
|
567
|
+
if (resolved.status === "too_long") addProblemCount(problemCounts, "url_length_overflow");
|
|
568
|
+
addProblemCount(problemCounts, "resource_url_unresolvable");
|
|
569
|
+
unattributedRequestCount += 1;
|
|
570
|
+
continue;
|
|
571
|
+
}
|
|
572
|
+
const transferredBytes = record?.encoded_data_length;
|
|
573
|
+
const canceled = record?.canceled === true;
|
|
574
|
+
const declaredBytes = canceled && Number.isSafeInteger(record?.declared_data_length)
|
|
575
|
+
&& record.declared_data_length >= 0 ? record.declared_data_length : null;
|
|
576
|
+
const resourceType = normalizeResourceType(record?.resource_type);
|
|
577
|
+
const cacheObserved = responseRecordFromCache(record);
|
|
578
|
+
const fromServiceWorker = Boolean(record?.from_service_worker);
|
|
579
|
+
const failed = Boolean(record?.failed);
|
|
580
|
+
const crossOrigin = Boolean(documentOrigin && resolved.origin !== documentOrigin);
|
|
581
|
+
const sourceUrls = record?.source_urls == null ? [] : record.source_urls;
|
|
582
|
+
if (!Array.isArray(sourceUrls)) addProblemCount(problemCounts, "resource_aliases_invalid");
|
|
583
|
+
|
|
584
|
+
const matchResources = [resolved];
|
|
585
|
+
for (const alias of (Array.isArray(sourceUrls) ? sourceUrls : [])) {
|
|
586
|
+
const aliasResource = resolvedResource(alias, { baseUrl: documentUrl });
|
|
587
|
+
if (aliasResource.status !== "http") {
|
|
588
|
+
if (aliasResource.status === "too_long") addProblemCount(problemCounts, "url_length_overflow");
|
|
589
|
+
addProblemCount(problemCounts, "resource_aliases_invalid");
|
|
590
|
+
}
|
|
591
|
+
else matchResources.push(aliasResource);
|
|
592
|
+
}
|
|
593
|
+
for (const hop of chain) {
|
|
594
|
+
const hopResource = resolvedResource(hop?.url, { baseUrl: documentUrl });
|
|
595
|
+
if (hopResource.status === "http") matchResources.push(hopResource);
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
const transferMeasured = Number.isInteger(transferredBytes) && transferredBytes >= 0;
|
|
599
|
+
const sizeAccounted = transferMeasured || declaredBytes !== null;
|
|
600
|
+
|
|
601
|
+
const group = groups.get(resolved.resource_id) || {
|
|
602
|
+
resource_id: resolved.resource_id,
|
|
603
|
+
url: resolved.projected,
|
|
604
|
+
resource_type: resourceType.value,
|
|
605
|
+
resource_type_status: resourceType.status,
|
|
606
|
+
observed_resource_types: new Set(),
|
|
607
|
+
transferred_bytes: 0,
|
|
608
|
+
declared_bytes: 0,
|
|
609
|
+
request_count: 0,
|
|
610
|
+
canceled_request_count: 0,
|
|
611
|
+
declared_request_count: 0,
|
|
612
|
+
unmeasured_request_count: 0,
|
|
613
|
+
failed_request_count: 0,
|
|
614
|
+
statuses: new Set(),
|
|
615
|
+
partial_request_count: 0,
|
|
616
|
+
cross_origin_request_count: 0,
|
|
617
|
+
cache_request_count: 0,
|
|
618
|
+
service_worker_request_count: 0,
|
|
619
|
+
match_resource_ids: new Set([resolved.resource_id]),
|
|
620
|
+
};
|
|
621
|
+
group.observed_resource_types.add(resourceType.value);
|
|
622
|
+
if (resourceType.status === "unknown") group.resource_type_status = "unknown";
|
|
623
|
+
if (transferMeasured) group.transferred_bytes += transferredBytes;
|
|
624
|
+
// A failed request has no transfer size by definition; its failure is
|
|
625
|
+
// attributed once the entry's type is resolved and is not also an
|
|
626
|
+
// unavailable measurement.
|
|
627
|
+
else if (!sizeAccounted && !failed) group.unmeasured_request_count += 1;
|
|
628
|
+
if (canceled) group.canceled_request_count += 1;
|
|
629
|
+
if (declaredBytes !== null) {
|
|
630
|
+
group.declared_bytes = Math.max(group.declared_bytes, declaredBytes);
|
|
631
|
+
group.declared_request_count += 1;
|
|
632
|
+
}
|
|
633
|
+
group.request_count += 1;
|
|
634
|
+
if (failed) group.failed_request_count += 1;
|
|
635
|
+
if (Number.isInteger(record?.status)) group.statuses.add(record.status);
|
|
636
|
+
if (record?.status === 206) group.partial_request_count += 1;
|
|
637
|
+
if (crossOrigin) group.cross_origin_request_count += 1;
|
|
638
|
+
if (cacheObserved) group.cache_request_count += 1;
|
|
639
|
+
if (fromServiceWorker) group.service_worker_request_count += 1;
|
|
640
|
+
for (const match of matchResources) group.match_resource_ids.add(match.resource_id);
|
|
641
|
+
groups.set(resolved.resource_id, group);
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
const allResources = [...groups.values()].map((group) => {
|
|
645
|
+
// CORS preflights share the URL of the request they authorize; observing
|
|
646
|
+
// preflight alongside the real type is expected, not ambiguity. fetch and
|
|
647
|
+
// xhr are likewise one programmatic-call class — which transport API a
|
|
648
|
+
// page used is not a resource-identity conflict. Only explicit preflight
|
|
649
|
+
// records are set aside (the collector classifies older-protocol OPTIONS
|
|
650
|
+
// legs as preflight at the source) and only the fetch-class transports
|
|
651
|
+
// are merged; any pairing outside those, "other" included, remains a
|
|
652
|
+
// genuine identity conflict.
|
|
653
|
+
const substantiveTypes = [...new Set(group.observed_resource_types)].filter((value) => value !== "preflight");
|
|
654
|
+
const canonicalTypes = [...new Set(substantiveTypes.map((value) => (value === "xhr" ? "fetch" : value)))];
|
|
655
|
+
const observedTypes = canonicalTypes.length ? canonicalTypes : [...group.observed_resource_types];
|
|
656
|
+
if (observedTypes.length === 1) {
|
|
657
|
+
// The xhr → fetch mapping exists only to keep the two transports from
|
|
658
|
+
// reading as an identity conflict; a resource observed under a single
|
|
659
|
+
// transport keeps the type it was actually observed with.
|
|
660
|
+
const preferred = substantiveTypes.length === 1 ? substantiveTypes[0] : observedTypes[0];
|
|
661
|
+
if (group.resource_type !== preferred) {
|
|
662
|
+
group.resource_type = preferred;
|
|
663
|
+
group.resource_type_status = KNOWN_RESOURCE_TYPES.has(preferred) ? "known" : "unknown";
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
if (observedTypes.length > 1) {
|
|
667
|
+
group.resource_type = "unknown";
|
|
668
|
+
group.resource_type_status = "ambiguous";
|
|
669
|
+
} else if (group.resource_type_status === "unknown") {
|
|
670
|
+
group.resource_type = "unknown";
|
|
671
|
+
}
|
|
672
|
+
const { observed_resource_types: ignored, statuses, match_resource_ids: matchIds, ...projection } = group;
|
|
673
|
+
return {
|
|
674
|
+
...projection,
|
|
675
|
+
statuses: [...statuses].sort((a, b) => a - b),
|
|
676
|
+
match_resource_ids: [...matchIds].sort(),
|
|
677
|
+
};
|
|
678
|
+
}).sort(resourceLedgerSort);
|
|
679
|
+
// The ledger-implied problems are read off the finished entries (types
|
|
680
|
+
// resolved, failures attributed) so the codes match what the ledger says,
|
|
681
|
+
// which is what the shape rules recompute from.
|
|
682
|
+
for (const [code, count] of Object.entries(ledgerProblemCounts(allResources))) {
|
|
683
|
+
addProblemCount(problemCounts, code, count);
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
const resources = allResources.slice(0, MAX_PAGE_LOAD_RESOURCE_LEDGER_ENTRIES);
|
|
687
|
+
const omittedResources = allResources.slice(MAX_PAGE_LOAD_RESOURCE_LEDGER_ENTRIES);
|
|
688
|
+
if (omittedResources.length) {
|
|
689
|
+
addProblemCount(problemCounts, "resource_ledger_overflow", omittedResources.length);
|
|
690
|
+
}
|
|
691
|
+
const problems = projectedProblems(problemCounts);
|
|
692
|
+
|
|
693
|
+
return {
|
|
694
|
+
measurement_status: polishCaptureMeasurementStatus(problems),
|
|
695
|
+
observed_response_count: prepared.length,
|
|
696
|
+
unattributed_request_count: unattributedRequestCount,
|
|
697
|
+
...resourceLedgerMetrics(resources),
|
|
698
|
+
resources,
|
|
699
|
+
resource_ledger: {
|
|
700
|
+
limit: MAX_PAGE_LOAD_RESOURCE_LEDGER_ENTRIES,
|
|
701
|
+
total_resource_count: allResources.length,
|
|
702
|
+
omitted_resource_count: omittedResources.length,
|
|
703
|
+
omitted_request_count: omittedResources.reduce((sum, resource) => sum + resource.request_count, 0),
|
|
704
|
+
},
|
|
705
|
+
document_response: documentResponse,
|
|
706
|
+
problems,
|
|
707
|
+
};
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
function normalizedPreloadAttribute(value) {
|
|
711
|
+
if (value === null) return "missing";
|
|
712
|
+
if (value === "") return "empty";
|
|
713
|
+
const token = value.toLowerCase();
|
|
714
|
+
return token === "auto" || token === "metadata" || token === "none" ? token : "other";
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
function sourceReference(value, sourceKind, sourceIndex, documentUrl) {
|
|
718
|
+
if (value === null || (typeof value === "string" && value.trim() === "")) return null;
|
|
719
|
+
const resolved = resolvedResource(value, { baseUrl: documentUrl });
|
|
720
|
+
return {
|
|
721
|
+
source_kind: sourceKind,
|
|
722
|
+
source_index: sourceIndex,
|
|
723
|
+
url: resolved.projected,
|
|
724
|
+
resource_id: resolved.resource_id,
|
|
725
|
+
};
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
export function sourceReferenceSort(a, b) {
|
|
729
|
+
const ranks = { current_src: 0, src_attribute: 1, source_src_attribute: 2, observed_source: 3 };
|
|
730
|
+
return (ranks[a.source_kind] ?? 99) - (ranks[b.source_kind] ?? 99)
|
|
731
|
+
|| a.source_index - b.source_index
|
|
732
|
+
|| String(a.resource_id).localeCompare(String(b.resource_id));
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
// Only the two explicit deferral tokens keep a hidden element from fetching
|
|
736
|
+
// eagerly; "auto", an empty or missing attribute and anything else do not.
|
|
737
|
+
export function preloadDefersFetch(preloadAttribute) {
|
|
738
|
+
return preloadAttribute === "none" || preloadAttribute === "metadata";
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
export function normalizeMediaElement(element, { documentUrl, elementIndex = 0 } = {}) {
|
|
742
|
+
const tagName = normalizedToken(element?.tag_name);
|
|
743
|
+
if (tagName !== "video" && tagName !== "audio") {
|
|
744
|
+
throw new Error("Page-load media capture accepts only video or audio elements.");
|
|
745
|
+
}
|
|
746
|
+
if (!Number.isInteger(elementIndex) || elementIndex < 0
|
|
747
|
+
|| !Object.hasOwn(element, "current_src")
|
|
748
|
+
|| (element.current_src !== null && typeof element.current_src !== "string")
|
|
749
|
+
|| !Object.hasOwn(element, "src_attribute")
|
|
750
|
+
|| (element.src_attribute !== null && typeof element.src_attribute !== "string")
|
|
751
|
+
|| !Array.isArray(element.source_src_attributes)
|
|
752
|
+
|| element.source_src_attributes.some((source) => typeof source !== "string")
|
|
753
|
+
|| (Object.hasOwn(element, "observed_source_urls")
|
|
754
|
+
&& (!Array.isArray(element.observed_source_urls)
|
|
755
|
+
|| element.observed_source_urls.some((source) => typeof source !== "string")))
|
|
756
|
+
|| !Object.hasOwn(element, "preload_attribute")
|
|
757
|
+
|| (element.preload_attribute !== null && typeof element.preload_attribute !== "string")
|
|
758
|
+
|| !validComputedStyle(element.computed_style)
|
|
759
|
+
|| !Array.isArray(element.ancestor_styles)
|
|
760
|
+
|| element.ancestor_styles.some((style) => !validComputedStyle(style))) {
|
|
761
|
+
throw new Error("Page-load media capture needs complete source and computed-style measurements.");
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
const styles = [element.computed_style, ...element.ancestor_styles];
|
|
765
|
+
const hiddenBy = new Set();
|
|
766
|
+
for (const style of styles) {
|
|
767
|
+
if (normalizedToken(style.display) === "none") hiddenBy.add("display_none");
|
|
768
|
+
if (normalizedToken(style.visibility) === "hidden") hiddenBy.add("visibility_hidden");
|
|
769
|
+
if (normalizedToken(style.visibility) === "collapse") hiddenBy.add("visibility_collapse");
|
|
770
|
+
}
|
|
771
|
+
const currentSrc = sourceReference(element.current_src, "current_src", 0, documentUrl);
|
|
772
|
+
const srcAttribute = sourceReference(element.src_attribute, "src_attribute", 0, documentUrl);
|
|
773
|
+
const sourceSrcAttributes = element.source_src_attributes
|
|
774
|
+
.map((source, index) => sourceReference(source, "source_src_attribute", index, documentUrl));
|
|
775
|
+
const observedSourceUrls = (Array.isArray(element.observed_source_urls) ? element.observed_source_urls : [])
|
|
776
|
+
.map((source, index) => sourceReference(source, "observed_source", index, documentUrl));
|
|
777
|
+
const sourceReferences = [currentSrc, srcAttribute, ...sourceSrcAttributes, ...observedSourceUrls]
|
|
778
|
+
.filter(Boolean)
|
|
779
|
+
.sort(sourceReferenceSort);
|
|
780
|
+
const preloadAttribute = normalizedPreloadAttribute(element.preload_attribute);
|
|
781
|
+
const width = Number(element?.bounding_box?.width);
|
|
782
|
+
const height = Number(element?.bounding_box?.height);
|
|
783
|
+
|
|
784
|
+
return {
|
|
785
|
+
tag_name: tagName,
|
|
786
|
+
element_index: elementIndex,
|
|
787
|
+
current_src: currentSrc?.url || null,
|
|
788
|
+
src_attribute: srcAttribute?.url || null,
|
|
789
|
+
source_src_attributes: sourceSrcAttributes.map((source) => source?.url || null),
|
|
790
|
+
observed_source_urls: observedSourceUrls.map((source) => source?.url || null),
|
|
791
|
+
source_references: sourceReferences,
|
|
792
|
+
preload_attribute: preloadAttribute,
|
|
793
|
+
preload_defers_fetch: preloadDefersFetch(preloadAttribute),
|
|
794
|
+
hidden_at_load: hiddenBy.size > 0,
|
|
795
|
+
hidden_by: [...hiddenBy].sort(),
|
|
796
|
+
zero_size_at_load: Number.isFinite(width) && Number.isFinite(height) ? (width <= 0 || height <= 0) : null,
|
|
797
|
+
};
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
export function normalizePageLoadRoute(value) {
|
|
801
|
+
const text = normalizeString(value);
|
|
802
|
+
if (!text) return null;
|
|
803
|
+
try {
|
|
804
|
+
const url = new URL(text, "https://capture.invalid/");
|
|
805
|
+
return url.pathname.startsWith("/") ? url.pathname : `/${url.pathname}`;
|
|
806
|
+
} catch {
|
|
807
|
+
return null;
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
export function normalizedNetworkidle(value) {
|
|
812
|
+
const status = normalizedToken(value?.status);
|
|
813
|
+
const durationMs = value?.duration_ms;
|
|
814
|
+
if ((status !== "settled" && status !== "timeout")
|
|
815
|
+
|| !Number.isInteger(durationMs)
|
|
816
|
+
|| durationMs < 0) return { status: "invalid", duration_ms: null };
|
|
817
|
+
return { status, duration_ms: durationMs };
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
export function normalizedViewport(value) {
|
|
821
|
+
const token = normalizeString(value)?.toLocaleLowerCase("en-US") || null;
|
|
822
|
+
return token && /^[a-z0-9][a-z0-9_-]{0,63}$/.test(token) ? token : null;
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
export function normalizedBuildFingerprint(value) {
|
|
826
|
+
const fingerprint = normalizeString(value);
|
|
827
|
+
return fingerprint && /^sha256:[a-f0-9]{64}$/.test(fingerprint) ? fingerprint : null;
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
export function normalizedCampaignSlug(value) {
|
|
831
|
+
const slug = normalizeString(value);
|
|
832
|
+
return slug && /^[a-z0-9][a-z0-9_-]{0,127}$/i.test(slug) ? slug : null;
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
function captureSubject({ buildFingerprint, slug, requestedRoute, finalDocumentUrl, viewport }) {
|
|
836
|
+
return {
|
|
837
|
+
build_fingerprint: normalizedBuildFingerprint(buildFingerprint),
|
|
838
|
+
campaign_slug: normalizedCampaignSlug(slug),
|
|
839
|
+
requested_route: normalizePageLoadRoute(requestedRoute),
|
|
840
|
+
final_document_route: resolvedCaptureUrl(finalDocumentUrl).status === "http"
|
|
841
|
+
? normalizePageLoadRoute(finalDocumentUrl)
|
|
842
|
+
: null,
|
|
843
|
+
viewport: normalizedViewport(viewport),
|
|
844
|
+
};
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
// The ledger entries a media element fetched: every entry whose identity set
|
|
848
|
+
// meets one of the element's resolved source references.
|
|
849
|
+
export function mediaFetchedResources(media, resources) {
|
|
850
|
+
const sourceIds = new Set(media.source_references
|
|
851
|
+
.map((reference) => reference.resource_id)
|
|
852
|
+
.filter(Boolean));
|
|
853
|
+
return resources.flatMap((resource) => {
|
|
854
|
+
const matchedSourceIds = resource.match_resource_ids.filter((id) => sourceIds.has(id));
|
|
855
|
+
if (!matchedSourceIds.length) return [];
|
|
856
|
+
return [{
|
|
857
|
+
resource_id: resource.resource_id,
|
|
858
|
+
url: resource.url,
|
|
859
|
+
resource_type: resource.resource_type,
|
|
860
|
+
transferred_bytes: resource.transferred_bytes,
|
|
861
|
+
request_count: resource.request_count,
|
|
862
|
+
...(Object.hasOwn(resource, "declared_bytes") ? { declared_bytes: resource.declared_bytes } : {}),
|
|
863
|
+
matched_source_resource_ids: matchedSourceIds.sort(),
|
|
864
|
+
}];
|
|
865
|
+
}).sort(resourceLedgerSort);
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
// The element's byte and request totals over its fetched resources.
|
|
869
|
+
export function mediaFetchTotals(fetchedResources) {
|
|
870
|
+
return {
|
|
871
|
+
fetched_bytes: fetchedResources.reduce((sum, resource) => sum + resource.transferred_bytes, 0),
|
|
872
|
+
fetched_request_count: fetchedResources.reduce((sum, resource) => sum + resource.request_count, 0),
|
|
873
|
+
declared_bytes: fetchedResources.reduce((sum, resource) => sum + (resource.declared_bytes || 0), 0),
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
// A hidden element that fetches eagerly and names a source the ledger cannot
|
|
878
|
+
// identify: the measurement cannot say what it loaded.
|
|
879
|
+
export function hiddenEagerSourceUnresolved(media) {
|
|
880
|
+
return media.hidden_at_load
|
|
881
|
+
&& !media.preload_defers_fetch
|
|
882
|
+
&& media.source_references.some((reference) => reference.resource_id === null);
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
// Media transfers no element claims: bytes the page moved that the media
|
|
886
|
+
// measurement cannot attribute.
|
|
887
|
+
export function unattributedMediaTransfers(media, resources) {
|
|
888
|
+
const attributedResourceIds = new Set(media.flatMap((item) => item.fetched_resources)
|
|
889
|
+
.map((resource) => resource.resource_id));
|
|
890
|
+
return resources.filter((resource) => resource.resource_type === "media"
|
|
891
|
+
&& resource.transferred_bytes > 0
|
|
892
|
+
&& !attributedResourceIds.has(resource.resource_id));
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
// The capture-level statuses stated beside the problems that justify them.
|
|
896
|
+
const PRODUCER_FAILURE_PROBLEM_CODES = Object.freeze([
|
|
897
|
+
"browser_unavailable",
|
|
898
|
+
"producer_failed",
|
|
899
|
+
"producer_timeout",
|
|
900
|
+
]);
|
|
901
|
+
|
|
902
|
+
export function producerStatus(problems) {
|
|
903
|
+
return problems.some((problem) => PRODUCER_FAILURE_PROBLEM_CODES.includes(problem.code)) ? "failed" : "complete";
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
export function responseCollectionProblemCode(status) {
|
|
907
|
+
if (status === "failed") return "response_collection_failed";
|
|
908
|
+
return status === "complete" ? null : "response_collection_status_invalid";
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
export function mediaCollectionStatus({
|
|
912
|
+
failed_element_count,
|
|
913
|
+
omitted_element_count,
|
|
914
|
+
source_overflow_element_count,
|
|
915
|
+
ancestor_overflow_element_count,
|
|
916
|
+
}) {
|
|
917
|
+
return failed_element_count > 0 || omitted_element_count > 0
|
|
918
|
+
|| source_overflow_element_count > 0 || ancestor_overflow_element_count > 0 ? "partial" : "complete";
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
export function buildPageLoadCapture({
|
|
922
|
+
buildFingerprint,
|
|
923
|
+
slug,
|
|
924
|
+
requestedRoute,
|
|
925
|
+
viewport,
|
|
926
|
+
finalDocumentUrl,
|
|
927
|
+
requestedDocumentUrl,
|
|
928
|
+
responseCollectionStatus,
|
|
929
|
+
networkidle,
|
|
930
|
+
mediaElements,
|
|
931
|
+
responses,
|
|
932
|
+
producerError,
|
|
933
|
+
producerProblem,
|
|
934
|
+
} = {}) {
|
|
935
|
+
const network = aggregateCdpResponses(responses, {
|
|
936
|
+
documentUrl: finalDocumentUrl,
|
|
937
|
+
requestedDocumentUrl,
|
|
938
|
+
requireFinalDocumentResponse: true,
|
|
939
|
+
});
|
|
940
|
+
const problemCounts = new Map(network.problems.map(({ code, count }) => [code, count]));
|
|
941
|
+
const addCaptureProblem = (code, count = 1) => addProblemCount(problemCounts, code, count);
|
|
942
|
+
const subject = captureSubject({ buildFingerprint, slug, requestedRoute, finalDocumentUrl, viewport });
|
|
943
|
+
for (const value of [requestedDocumentUrl, finalDocumentUrl, requestedRoute]) {
|
|
944
|
+
if (typeof value === "string" && value.length > MAX_POLISH_CAPTURE_URL_LENGTH) {
|
|
945
|
+
addCaptureProblem("url_length_overflow");
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
if (Object.values(subject).some((value) => value === null)) addCaptureProblem("capture_subject_invalid");
|
|
949
|
+
if (subject.requested_route && subject.final_document_route
|
|
950
|
+
&& subject.requested_route !== subject.final_document_route) {
|
|
951
|
+
addCaptureProblem("final_document_route_mismatch");
|
|
952
|
+
}
|
|
953
|
+
if (!Array.isArray(mediaElements)) addCaptureProblem("media_collection_unavailable");
|
|
954
|
+
if (!Array.isArray(responses)) addCaptureProblem("response_collection_unavailable");
|
|
955
|
+
// The browser collector reports whether it observed the network faithfully.
|
|
956
|
+
// Whether a failed request voids the collection is decided here, from the
|
|
957
|
+
// attributed ledger: a dependency failure (document, first-party, or any
|
|
958
|
+
// script/stylesheet/media) fails it; a cross-origin beacon-class failure is
|
|
959
|
+
// recorded as a warning and leaves the measurement complete.
|
|
960
|
+
const reportedCollectionStatus = normalizedToken(responseCollectionStatus);
|
|
961
|
+
const dependencyFailed = (problemCounts.get("dependency_request_failed") || 0) > 0;
|
|
962
|
+
const collectionStatus = reportedCollectionStatus === "complete" && dependencyFailed
|
|
963
|
+
? "failed"
|
|
964
|
+
: reportedCollectionStatus;
|
|
965
|
+
const collectionProblem = responseCollectionProblemCode(collectionStatus);
|
|
966
|
+
if (collectionProblem) addCaptureProblem(collectionProblem);
|
|
967
|
+
if (collectionStatus === "complete" && Array.isArray(responses) && network.observed_response_count === 0) {
|
|
968
|
+
addCaptureProblem("response_collection_empty");
|
|
969
|
+
}
|
|
970
|
+
const normalizedProducerProblem = producerProblem === "browser_unavailable"
|
|
971
|
+
? "browser_unavailable"
|
|
972
|
+
: producerProblem === "producer_timeout"
|
|
973
|
+
? "producer_timeout"
|
|
974
|
+
: producerProblem === "producer_failed" || producerError ? "producer_failed" : null;
|
|
975
|
+
if (normalizedProducerProblem) addCaptureProblem(normalizedProducerProblem);
|
|
976
|
+
|
|
977
|
+
const media = [];
|
|
978
|
+
let failedMediaElementCount = 0;
|
|
979
|
+
const rawMediaElements = Array.isArray(mediaElements) ? mediaElements : [];
|
|
980
|
+
const declaredObservedElementCount = Number.isInteger(rawMediaElements.observed_element_count)
|
|
981
|
+
&& rawMediaElements.observed_element_count >= rawMediaElements.length
|
|
982
|
+
? rawMediaElements.observed_element_count
|
|
983
|
+
: rawMediaElements.length;
|
|
984
|
+
const boundedMediaElements = rawMediaElements.slice(0, MAX_PAGE_LOAD_MEDIA_ELEMENTS);
|
|
985
|
+
const omittedMediaElementCount = Math.max(0, declaredObservedElementCount - boundedMediaElements.length);
|
|
986
|
+
if (omittedMediaElementCount > 0) addCaptureProblem("media_element_overflow", omittedMediaElementCount);
|
|
987
|
+
let sourceOverflowElementCount = 0;
|
|
988
|
+
let ancestorOverflowElementCount = 0;
|
|
989
|
+
for (const [elementIndex, element] of boundedMediaElements.entries()) {
|
|
990
|
+
try {
|
|
991
|
+
const sourceOverflow = (Array.isArray(element?.source_src_attributes)
|
|
992
|
+
&& element.source_src_attributes.length > MAX_PAGE_LOAD_MEDIA_SOURCES_PER_ELEMENT)
|
|
993
|
+
|| (Array.isArray(element?.observed_source_urls)
|
|
994
|
+
&& element.observed_source_urls.length > MAX_PAGE_LOAD_MEDIA_SOURCES_PER_ELEMENT)
|
|
995
|
+
|| (Number.isInteger(element?.source_overflow_count) && element.source_overflow_count > 0);
|
|
996
|
+
const ancestorOverflow = (Array.isArray(element?.ancestor_styles)
|
|
997
|
+
&& element.ancestor_styles.length > MAX_PAGE_LOAD_MEDIA_ANCESTORS)
|
|
998
|
+
|| (Number.isInteger(element?.ancestor_overflow_count) && element.ancestor_overflow_count > 0);
|
|
999
|
+
if (sourceOverflow) sourceOverflowElementCount += 1;
|
|
1000
|
+
if (ancestorOverflow) ancestorOverflowElementCount += 1;
|
|
1001
|
+
const mediaSourceValues = [element?.current_src, element?.src_attribute,
|
|
1002
|
+
...(Array.isArray(element?.source_src_attributes)
|
|
1003
|
+
? element.source_src_attributes.slice(0, MAX_PAGE_LOAD_MEDIA_SOURCES_PER_ELEMENT) : []),
|
|
1004
|
+
...(Array.isArray(element?.observed_source_urls)
|
|
1005
|
+
? element.observed_source_urls.slice(0, MAX_PAGE_LOAD_MEDIA_SOURCES_PER_ELEMENT) : [])];
|
|
1006
|
+
const overlongSourceCount = mediaSourceValues.filter((value) => typeof value === "string"
|
|
1007
|
+
&& value.length > MAX_POLISH_CAPTURE_URL_LENGTH).length
|
|
1008
|
+
+ (Number.isInteger(element?.url_overflow_count) && element.url_overflow_count > 0
|
|
1009
|
+
? element.url_overflow_count : 0);
|
|
1010
|
+
if (overlongSourceCount > 0) addCaptureProblem("url_length_overflow", overlongSourceCount);
|
|
1011
|
+
const boundedElement = isPlainObject(element) ? {
|
|
1012
|
+
...element,
|
|
1013
|
+
source_src_attributes: Array.isArray(element.source_src_attributes)
|
|
1014
|
+
? element.source_src_attributes.slice(0, MAX_PAGE_LOAD_MEDIA_SOURCES_PER_ELEMENT)
|
|
1015
|
+
: element.source_src_attributes,
|
|
1016
|
+
...(Array.isArray(element.observed_source_urls) ? {
|
|
1017
|
+
observed_source_urls: element.observed_source_urls.slice(0, MAX_PAGE_LOAD_MEDIA_SOURCES_PER_ELEMENT),
|
|
1018
|
+
} : {}),
|
|
1019
|
+
ancestor_styles: Array.isArray(element.ancestor_styles)
|
|
1020
|
+
? element.ancestor_styles.slice(0, MAX_PAGE_LOAD_MEDIA_ANCESTORS)
|
|
1021
|
+
: element.ancestor_styles,
|
|
1022
|
+
} : element;
|
|
1023
|
+
const normalized = normalizeMediaElement(boundedElement, { documentUrl: finalDocumentUrl, elementIndex });
|
|
1024
|
+
if (hiddenEagerSourceUnresolved(normalized)) addCaptureProblem("media_source_unresolvable");
|
|
1025
|
+
const fetchedResources = mediaFetchedResources(normalized, network.resources);
|
|
1026
|
+
const hasDeclaredShape = network.resources.some((resource) => Object.hasOwn(resource, "declared_bytes"));
|
|
1027
|
+
const { declared_bytes: declaredBytes, ...fetchTotals } = mediaFetchTotals(fetchedResources);
|
|
1028
|
+
media.push({
|
|
1029
|
+
...normalized,
|
|
1030
|
+
...fetchTotals,
|
|
1031
|
+
...(hasDeclaredShape ? { declared_bytes: declaredBytes } : {}),
|
|
1032
|
+
fetched_resources: fetchedResources,
|
|
1033
|
+
});
|
|
1034
|
+
} catch {
|
|
1035
|
+
failedMediaElementCount += 1;
|
|
1036
|
+
addCaptureProblem("media_measurement_failed");
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
if (sourceOverflowElementCount > 0) {
|
|
1040
|
+
addCaptureProblem("media_source_overflow", sourceOverflowElementCount);
|
|
1041
|
+
}
|
|
1042
|
+
if (ancestorOverflowElementCount > 0) {
|
|
1043
|
+
addCaptureProblem("media_ancestor_overflow", ancestorOverflowElementCount);
|
|
1044
|
+
}
|
|
1045
|
+
const unattributedTransfers = unattributedMediaTransfers(media, network.resources);
|
|
1046
|
+
if (unattributedTransfers.length) addCaptureProblem("media_transfer_unattributed", unattributedTransfers.length);
|
|
1047
|
+
const settled = normalizedNetworkidle(networkidle);
|
|
1048
|
+
if (settled.status === "invalid") addCaptureProblem("networkidle_measurement_invalid");
|
|
1049
|
+
const problems = projectedProblems(problemCounts);
|
|
1050
|
+
const mediaCollectionCounts = {
|
|
1051
|
+
failed_element_count: failedMediaElementCount,
|
|
1052
|
+
omitted_element_count: omittedMediaElementCount,
|
|
1053
|
+
source_overflow_element_count: sourceOverflowElementCount,
|
|
1054
|
+
ancestor_overflow_element_count: ancestorOverflowElementCount,
|
|
1055
|
+
};
|
|
1056
|
+
|
|
1057
|
+
const capture = {
|
|
1058
|
+
schema_version: POLISH_ROUTE_CAPTURE_SCHEMA_VERSION,
|
|
1059
|
+
performed_by: POLISH_CAPTURE_PRODUCER,
|
|
1060
|
+
subject,
|
|
1061
|
+
measurement_status: polishCaptureMeasurementStatus(problems),
|
|
1062
|
+
producer_status: producerStatus(problems),
|
|
1063
|
+
response_collection: {
|
|
1064
|
+
status: collectionStatus === "complete" || collectionStatus === "failed" ? collectionStatus : "invalid",
|
|
1065
|
+
observed_response_count: network.observed_response_count,
|
|
1066
|
+
unattributed_response_count: network.unattributed_request_count,
|
|
1067
|
+
},
|
|
1068
|
+
media_collection: {
|
|
1069
|
+
status: !Array.isArray(mediaElements) ? "failed" : mediaCollectionStatus(mediaCollectionCounts),
|
|
1070
|
+
observed_element_count: Array.isArray(mediaElements) ? declaredObservedElementCount : 0,
|
|
1071
|
+
...mediaCollectionCounts,
|
|
1072
|
+
},
|
|
1073
|
+
networkidle: settled,
|
|
1074
|
+
document_response: network.document_response,
|
|
1075
|
+
resource_ledger: {
|
|
1076
|
+
...network.resource_ledger,
|
|
1077
|
+
entries: network.resources,
|
|
1078
|
+
},
|
|
1079
|
+
metrics: {
|
|
1080
|
+
total_transferred_bytes: network.total_transferred_bytes,
|
|
1081
|
+
request_count: network.request_count,
|
|
1082
|
+
largest_resource: network.largest_resource,
|
|
1083
|
+
cross_origin_request_count: network.cross_origin_request_count,
|
|
1084
|
+
cache_request_count: network.cache_request_count,
|
|
1085
|
+
service_worker_request_count: network.service_worker_request_count,
|
|
1086
|
+
},
|
|
1087
|
+
media,
|
|
1088
|
+
problems,
|
|
1089
|
+
};
|
|
1090
|
+
return {
|
|
1091
|
+
...capture,
|
|
1092
|
+
integrity: buildPolishCaptureIntegrity(capture),
|
|
1093
|
+
};
|
|
1094
|
+
}
|