@slicedearth/whoisleuth-cli 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +6 -0
- package/README.md +20 -15
- package/bin/whoisleuth.mjs +8 -1
- package/cli/archive-content-digest.mjs +14 -0
- package/cli/archive-inspect.mjs +20 -53
- package/cli/arguments.mjs +454 -1748
- package/cli/artifact-validation/case-response.mjs +66 -35
- package/cli/artifact-validation/investigation-capsule.mjs +52 -31
- package/cli/artifact-validation/lookup-evidence.mjs +12 -6
- package/cli/artifact-validation/signed-review.mjs +112 -72
- package/cli/artifact-validation/structure-primitives.mjs +1 -105
- package/cli/artifact-verify.mjs +90 -24
- package/cli/assurance-command-runner.mjs +39 -4
- package/cli/bagit-folder.mjs +57 -0
- package/cli/bagit-review.mjs +23 -0
- package/cli/bounded-json.mjs +1 -1
- package/cli/bulk-checkpoint.mjs +34 -3
- package/cli/bulk-command-runner.mjs +2 -0
- package/cli/bulk-output.mjs +29 -3
- package/cli/case-command.mjs +195 -0
- package/cli/case-pack.mjs +66 -111
- package/cli/command-argument-grammar.mjs +215 -0
- package/cli/command-reference.mjs +342 -285
- package/cli/completion.mjs +303 -80
- package/cli/config-profile.mjs +7 -5
- package/cli/ct-event-intake.mjs +12 -12
- package/cli/discover.mjs +4 -3
- package/cli/discovery-observation-snapshot.mjs +59 -18
- package/cli/discovery-scan-command-runner.mjs +21 -0
- package/cli/discovery-scan.mjs +13 -4
- package/cli/discovery-workflow.mjs +1 -1
- package/cli/domain-control-monitor.mjs +50 -20
- package/cli/domain-control-observations.mjs +149 -102
- package/cli/errors.mjs +33 -1
- package/cli/evidence-command-runner.mjs +16 -5
- package/cli/evidence-signing.mjs +2 -2
- package/cli/external-observation-mapping.mjs +7 -8
- package/cli/formatters/terminal-lookup.mjs +18 -46
- package/cli/formatters/terminal-shared.mjs +5 -3
- package/cli/formatters/terminal-technology.mjs +114 -0
- package/cli/inline-command-families.mjs +2 -0
- package/cli/input.mjs +116 -0
- package/cli/interactive-launcher.mjs +2 -160
- package/cli/interchange-report.mjs +18 -8
- package/cli/investigation-artifacts.mjs +176 -0
- package/cli/investigation-checkpoint.mjs +10 -0
- package/cli/investigation-folder.mjs +114 -0
- package/cli/investigation-manifest.mjs +1 -112
- package/cli/investigation-package-review.mjs +103 -0
- package/cli/investigation-plan.mjs +32 -19
- package/cli/investigation-run.mjs +188 -33
- package/cli/local-document-checkpoint.mjs +170 -0
- package/cli/lookalike-calibration.mjs +7 -2
- package/cli/lookup-brief.mjs +4 -2
- package/cli/lookup-browser-view.mjs +1 -1
- package/cli/lookup-browser.mjs +5 -12
- package/cli/lookup-command-runner.mjs +17 -6
- package/cli/lookup-diff.mjs +37 -3
- package/cli/lookup-plan.mjs +11 -5
- package/cli/lookup-reconcile.mjs +10 -8
- package/cli/mail-header-review.mjs +467 -0
- package/cli/mail-review.mjs +66 -14
- package/cli/manual.mjs +1 -0
- package/cli/network-command-runner.mjs +2 -0
- package/cli/output-file.mjs +31 -5
- package/cli/registry-doctor.mjs +3 -1
- package/cli/registry-support.mjs +1 -1
- package/cli/retained-artifact-diff.mjs +2 -2
- package/cli/review-command-runner.mjs +70 -0
- package/cli/risk-calibration.mjs +19 -17
- package/cli/runner.mjs +62 -7
- package/cli/sharing-review.mjs +21 -34
- package/cli/signer-trust.mjs +104 -0
- package/cli/source-reliability.mjs +74 -83
- package/cli/terminal-input.mjs +168 -0
- package/cli/workflow-command-runner.mjs +68 -13
- package/docs/cli-reference.md +77 -24
- package/docs/cli.md +300 -25
- package/lib/abort.mjs +4 -1
- package/lib/availability.mjs +161 -112
- package/lib/bounded-file.mjs +3 -1
- package/lib/bounded-html-document.mjs +125 -0
- package/lib/bounded-json.mjs +51 -8
- package/lib/browser-library-profile.mjs +42 -51
- package/lib/browser-library-worker.mjs +112 -0
- package/lib/client-behavior-profile.mjs +3 -3
- package/lib/ct-search.mjs +11 -4
- package/lib/dns-convergence-review.mjs +1 -0
- package/lib/dns-delegation-health.mjs +24 -14
- package/lib/dns-inheritance-review.mjs +217 -0
- package/lib/dns-intelligence.mjs +73 -26
- package/lib/dns-mx.mjs +1 -1
- package/lib/dnssec-chain-validation.mjs +6 -0
- package/lib/dnssec-evidence-validation.mjs +11 -3
- package/lib/domain-change-packet.mjs +49 -16
- package/lib/domain-change-review.mjs +14 -8
- package/lib/domain-control-flight-recorder.mjs +221 -117
- package/lib/domain-control-manifest.mjs +129 -69
- package/lib/domain-posture-analysis.mjs +20 -7
- package/lib/domain-posture-parsers.mjs +201 -8
- package/lib/domain-posture.mjs +147 -29
- package/lib/evidence-export-privacy.mjs +561 -0
- package/lib/evidence-export.mjs +11 -543
- package/lib/evidence-report.mjs +5 -3
- package/lib/favicon.mjs +117 -41
- package/lib/generated/cisa-kev-catalog.mjs +12 -4
- package/lib/generated/registrar-standing-catalogue.mjs +7 -7
- package/lib/generated/retire-browser-catalog.mjs +2335 -940
- package/lib/hostname.mjs +2 -38
- package/lib/html-signals.mjs +111 -108
- package/lib/http-evidence-bounds.mjs +2 -1
- package/lib/http-intelligence.mjs +39 -26
- package/lib/local-geoip-evidence.mjs +19 -3
- package/lib/lookup-cache.mjs +34 -11
- package/lib/lookup-child-profile-contract.mjs +24 -15
- package/lib/lookup-diagnostics.mjs +24 -1
- package/lib/lookup-progress.mjs +30 -13
- package/lib/lookup-response-contract.mjs +1048 -0
- package/lib/lookup-response.mjs +8 -10
- package/lib/lookup-source-progress.mjs +40 -14
- package/lib/lookup.mjs +48 -27
- package/lib/nameserver-preflight-review.mjs +1 -0
- package/lib/observed-network-context.mjs +43 -27
- package/lib/outbound-request-bounds.mjs +4 -1
- package/lib/page-fingerprints.mjs +69 -133
- package/lib/page-role-profile.mjs +12 -8
- package/lib/perceptual-hash.mjs +43 -14
- package/lib/rdap-bootstrap.mjs +72 -22
- package/lib/rdap-capabilities.mjs +1 -1
- package/lib/rdap-client.mjs +15 -7
- package/lib/rdap-normalization.mjs +33 -18
- package/lib/rdap-search-workbench.mjs +1 -1
- package/lib/rdap-transport.mjs +47 -20
- package/lib/rdap-types.mjs +2 -1
- package/lib/rdap.mjs +18 -3
- package/lib/registrable-domain.mjs +21 -1
- package/lib/registrar-standing-contract.mjs +1 -0
- package/lib/registry-capabilities.mjs +5 -1
- package/lib/registry-capability-catalogue.mjs +2 -1
- package/lib/registry-dates.mjs +2 -190
- package/lib/registry-insights.mjs +85 -34
- package/lib/response-policy.mjs +86 -48
- package/lib/risk-calibration-summary.mjs +18 -6
- package/lib/rpki-evidence.mjs +8 -5
- package/lib/safe-fetch.mjs +54 -47
- package/lib/security-txt.mjs +42 -7
- package/lib/service-binding-dns.mjs +8 -0
- package/lib/smtp-transport-review.mjs +27 -15
- package/lib/sslbl-certificates.generated.mjs +136 -135
- package/lib/static-html-analysis.mjs +178 -151
- package/lib/structured-data-identity.mjs +1 -0
- package/lib/technology-markup.mjs +100 -0
- package/lib/threat-intelligence-manifests.mjs +11 -139
- package/lib/threat-intelligence-runtime.mjs +24 -52
- package/lib/threat-intelligence-targets.mjs +6 -34
- package/lib/threat-intelligence-types.mjs +39 -0
- package/lib/typosquat-generator.mjs +19 -15
- package/lib/urlhaus-intelligence.mjs +8 -3
- package/lib/urlscan-intelligence.mjs +1 -1
- package/lib/website-security-posture.mjs +30 -6
- package/lib/website-technology.mjs +153 -45
- package/lib/whois-authority.mjs +14 -2
- package/lib/whois-chain.mjs +9 -2
- package/lib/whois-common-formats.mjs +5 -5
- package/lib/whois-contracts.mjs +2 -1
- package/lib/whois-dialects.mjs +2 -2
- package/lib/whois-parser-support.mjs +8 -1
- package/lib/whois-transport.mjs +23 -7
- package/lib/zone-intent-review.mjs +16 -6
- package/package.json +2 -1
- package/packages/cases/case-attachment-model.mjs +83 -0
- package/packages/cases/case-evidence-model.mjs +156 -62
- package/packages/cases/case-export-input.mjs +53 -0
- package/packages/cases/case-migration-model.mjs +127 -44
- package/packages/cases/case-model.mjs +1 -0
- package/packages/cases/case-recheck-model.mjs +91 -0
- package/packages/cases/case-record-contracts.mjs +5 -20
- package/packages/cases/case-record-core.mjs +12 -22
- package/packages/cases/case-record-decisions.mjs +72 -0
- package/packages/cases/case-record-model.mjs +1 -0
- package/packages/cases/case-record-operations.mjs +95 -31
- package/packages/cases/case-record-projection.mjs +221 -0
- package/packages/cases/case-report.mjs +70 -54
- package/packages/cases/case-response-actions.mjs +654 -0
- package/packages/cases/case-response-model.mjs +64 -1115
- package/packages/cases/case-response-outcomes.mjs +317 -0
- package/packages/cases/case-response-packet-render.mjs +153 -0
- package/packages/cases/case-response-packet-types.mjs +1 -0
- package/packages/cases/case-response-packet-vocabulary.mjs +48 -0
- package/packages/cases/case-response-packet.mjs +74 -572
- package/packages/cases/case-response-records.mjs +94 -0
- package/packages/cases/case-response-review-inputs.mjs +371 -0
- package/packages/cases/case-response-values.mjs +85 -0
- package/packages/cases/case-review-package.mjs +45 -0
- package/packages/cases/case-selection.mjs +46 -0
- package/packages/cases/case-storage-model.mjs +31 -2
- package/packages/cases/response-route-freshness.mjs +16 -0
- package/packages/comparison/capture-context.mjs +61 -0
- package/packages/comparison/comparison-ledger-bulk.mjs +13 -11
- package/packages/comparison/page-similarity.mjs +5 -2
- package/packages/comparison/relationship-evidence.mjs +85 -8
- package/packages/comparison/relationship-provenance.mjs +93 -0
- package/packages/contracts/analyst-interchange.mjs +216 -0
- package/packages/contracts/analyst-review-state-contract.mjs +11 -0
- package/packages/contracts/analyst-review-state.mjs +5 -4
- package/packages/contracts/case-portability.mjs +366 -83
- package/packages/contracts/case-views-contract.mjs +12 -0
- package/packages/contracts/case-views.mjs +26 -0
- package/packages/contracts/cli-command-semantics.mjs +94 -0
- package/packages/contracts/cli-lookup.mjs +0 -1
- package/packages/contracts/domain-control-flight-recorder.mjs +80 -42
- package/packages/contracts/domain-control-manifest.mjs +127 -73
- package/packages/contracts/domain-control-monitor.mjs +48 -45
- package/packages/contracts/domain-control-review.mjs +179 -121
- package/packages/contracts/domain-name.mjs +21 -0
- package/packages/contracts/external-observation-interchange.mjs +2 -1
- package/packages/contracts/extracted-domain-lifecycle.mjs +11 -32
- package/packages/contracts/http-url.mjs +2 -0
- package/packages/contracts/investigation-package-limits.mjs +24 -0
- package/packages/contracts/investigation-portability.mjs +56 -27
- package/packages/contracts/investigation-run.mjs +16 -0
- package/packages/contracts/ip-address.mjs +34 -0
- package/packages/contracts/lookup-evidence.mjs +35 -0
- package/packages/contracts/offline-comparison.mjs +1 -2
- package/packages/contracts/page-fingerprints.mjs +15 -0
- package/packages/contracts/risk-calibration.mjs +15 -11
- package/packages/contracts/schema-compatibility.mjs +1 -1
- package/packages/contracts/schema-lifecycle.mjs +83 -153
- package/packages/contracts/selected-file-limits.mjs +6 -0
- package/packages/contracts/vulnerability-identifiers.mjs +5 -0
- package/packages/contracts/web-capture.mjs +10 -0
- package/packages/contracts/workspace-portability.mjs +183 -211
- package/packages/evidence/artifact-integrity.mjs +8 -2
- package/packages/evidence/artifact-structure.mjs +105 -0
- package/packages/evidence/decision-fact.mjs +4 -1
- package/packages/evidence/domain-control-runtime.mjs +201 -33
- package/packages/evidence/domain-posture-context.mjs +79 -0
- package/packages/evidence/image-regions.mjs +56 -0
- package/packages/evidence/latest-observations.mjs +28 -0
- package/packages/evidence/lookup-target.mjs +104 -0
- package/packages/evidence/observation.mjs +16 -1
- package/packages/evidence/passphrase-encryption.mjs +44 -0
- package/packages/evidence/record-identity.mjs +72 -0
- package/packages/evidence/registry-dates.mjs +190 -0
- package/packages/evidence/retained-file.mjs +69 -0
- package/packages/evidence/review-age.mjs +14 -0
- package/packages/interchange/bagit.mjs +342 -0
- package/packages/interchange/bounded-zip-extraction.mjs +278 -0
- package/packages/interchange/crc32.mjs +17 -0
- package/packages/interchange/external-findings-import.mjs +92 -51
- package/packages/interchange/web-capture-import.mjs +382 -0
- package/packages/investigation/bulk-mail-exposure.mjs +2 -1
- package/packages/investigation/investigation-bagit.mjs +14 -0
- package/packages/investigation/investigation-capsule.mjs +33 -5
- package/packages/investigation/investigation-manifest.mjs +200 -0
- package/packages/investigation/investigation-package-crypto.mjs +98 -0
- package/packages/investigation/investigation-package.mjs +200 -0
- package/packages/investigation/lookup-asset-graph-inputs.mjs +38 -0
- package/packages/investigation/lookup-asset-graph.mjs +193 -165
- package/packages/investigation/lookup-claim-readiness.mjs +18 -0
- package/packages/investigation/lookup-investigation-brief.mjs +3 -3
- package/packages/investigation/lookup-task-guidance.mjs +58 -0
- package/packages/monitoring/analyst-review-state.mjs +38 -90
- package/packages/workspace/brand-profile-model.mjs +276 -22
- package/packages/workspace/bulk-columns.mjs +17 -0
- package/packages/workspace/bulk-review-model.mjs +23 -11
- package/packages/workspace/bulk-session-model.mjs +100 -24
- package/packages/workspace/bulk-sort.mjs +1 -0
- package/packages/workspace/case-views.mjs +102 -0
- package/packages/workspace/detection-rule-model.mjs +70 -22
- package/packages/workspace/hostile-input.mjs +29 -8
- package/packages/workspace/investigation-template-model.mjs +19 -4
- package/packages/workspace/page-baseline.mjs +18 -15
- package/packages/workspace/relationship-observation-model.mjs +41 -23
- package/packages/workspace/watchlist-history.mjs +2 -2
- package/packages/workspace/watchlist-store.mjs +8 -2
- package/packages/workspace/website-snapshot-model.mjs +164 -26
- package/packages/workspace/workspace-archive-crypto.mjs +41 -61
- package/packages/workspace/workspace-archive.mjs +87 -21
- package/third-party-notices.txt +29 -2
- package/cli/evidence-command-arguments.mjs +0 -156
- package/packages/investigation/bulk-review-export.mjs +0 -63
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { CASE_SCHEMA_VERSION, MAX_CASE_STORE_BYTES, } from "./case-record-model.mjs";
|
|
2
2
|
import { normalizeCaseStore } from "./case-migration-model.mjs";
|
|
3
|
+
import { projectCaseForDurableWrite } from "./case-record-projection.mjs";
|
|
3
4
|
/** The exact versioned string persisted by the browser storage adapter. */
|
|
4
5
|
export function serializeCaseStore(cases) {
|
|
5
|
-
return JSON.stringify({
|
|
6
|
+
return JSON.stringify({
|
|
7
|
+
version: CASE_SCHEMA_VERSION,
|
|
8
|
+
cases: cases.map(projectCaseForDurableWrite),
|
|
9
|
+
});
|
|
6
10
|
}
|
|
7
11
|
function byteLength(text) {
|
|
8
12
|
if (typeof TextEncoder !== 'undefined') {
|
|
@@ -84,10 +88,35 @@ export function enforceStoreBudget(cases) {
|
|
|
84
88
|
}
|
|
85
89
|
return { cases: working, pruned };
|
|
86
90
|
}
|
|
91
|
+
/** A read-only, ephemeral preview of the exact existing byte-budget policy. */
|
|
92
|
+
export function prepareCaseStoreSave(current, proposed) {
|
|
93
|
+
const result = enforceStoreBudget(proposed);
|
|
94
|
+
const retainedById = new Map(result.cases.map(record => [record.id, record]));
|
|
95
|
+
const removed = proposed.flatMap(record => {
|
|
96
|
+
const retained = retainedById.get(record.id);
|
|
97
|
+
const ids = new Set(retained?.evidenceHistory.map(snapshot => snapshot.id));
|
|
98
|
+
const snapshots = record.evidenceHistory.filter(snapshot => !ids.has(snapshot.id));
|
|
99
|
+
return snapshots.length ? [{
|
|
100
|
+
caseId: record.id, domain: record.domain,
|
|
101
|
+
snapshotIds: snapshots.map(snapshot => snapshot.id),
|
|
102
|
+
remainingSnapshots: retained?.evidenceHistory.length ?? 0,
|
|
103
|
+
}] : [];
|
|
104
|
+
});
|
|
105
|
+
return {
|
|
106
|
+
...result,
|
|
107
|
+
current: normalizeCaseStore(current).cases,
|
|
108
|
+
removed,
|
|
109
|
+
removedBytes: Math.max(0, byteLength(serializeCaseStore(proposed)) - byteLength(serializeCaseStore(result.cases))),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/** Same-clock peer edits invalidate confirmation, including changes to other Cases. */
|
|
113
|
+
export function caseStoreSavePreviewIsCurrent(current, preview) {
|
|
114
|
+
return serializeCaseStore(current) === serializeCaseStore(preview.current);
|
|
115
|
+
}
|
|
87
116
|
export function buildCaseExport(cases, nowIso) {
|
|
88
117
|
return {
|
|
89
118
|
version: CASE_SCHEMA_VERSION,
|
|
90
119
|
exportedAt: nowIso || new Date().toISOString(),
|
|
91
|
-
cases: normalizeCaseStore(cases).cases,
|
|
120
|
+
cases: normalizeCaseStore(cases).cases.map(projectCaseForDurableWrite),
|
|
92
121
|
};
|
|
93
122
|
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
|
|
2
|
+
import { MAX_RESPONSE_AUTHORISATION_CLOCK_SKEW_MS, RESPONSE_ROUTE_STALE_AFTER_DAYS } from "../contracts/case-portability.mjs";
|
|
3
|
+
/** Source freshness is independent of action deadlines and analyst follow-up. */
|
|
4
|
+
export function responseRouteFreshness(observedAt, reviewAfter, evaluatedAt) {
|
|
5
|
+
const observation = normalizeExplicitIsoTimestamp(observedAt);
|
|
6
|
+
const evaluation = normalizeExplicitIsoTimestamp(evaluatedAt);
|
|
7
|
+
const deadline = normalizeExplicitIsoTimestamp(reviewAfter);
|
|
8
|
+
if (!observation || !evaluation || (reviewAfter !== null && !deadline))
|
|
9
|
+
return 'unknown';
|
|
10
|
+
const age = Date.parse(evaluation) - Date.parse(observation);
|
|
11
|
+
return age < -MAX_RESPONSE_AUTHORISATION_CLOCK_SKEW_MS
|
|
12
|
+
|| (deadline !== null && Date.parse(evaluation) >= Date.parse(deadline))
|
|
13
|
+
|| (deadline === null && age > RESPONSE_ROUTE_STALE_AFTER_DAYS * 86_400_000)
|
|
14
|
+
? 'stale'
|
|
15
|
+
: 'current';
|
|
16
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { exact, text } from "../evidence/artifact-structure.mjs";
|
|
2
|
+
import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
|
|
3
|
+
import { readEvidenceImageDimensions } from "../evidence/image-regions.mjs";
|
|
4
|
+
/** Optional declarations remain unknown for existing manifests; no inference from image dimensions. */
|
|
5
|
+
export function readCaptureConditions(raw) {
|
|
6
|
+
if (raw === undefined || raw === null)
|
|
7
|
+
return null;
|
|
8
|
+
const value = exact(raw, ['browser', 'browserVersion', 'viewport', 'deviceScaleFactor', 'locale', 'timezone', 'colourScheme'], 'Capture conditions');
|
|
9
|
+
const viewport = exact(value.viewport, ['width', 'height'], 'Capture viewport');
|
|
10
|
+
const dimensions = readEvidenceImageDimensions(viewport.width, viewport.height);
|
|
11
|
+
if (typeof value.deviceScaleFactor !== 'number' || !Number.isFinite(value.deviceScaleFactor)
|
|
12
|
+
|| value.deviceScaleFactor < 0.25 || value.deviceScaleFactor > 8)
|
|
13
|
+
throw new TypeError('Capture scale must be between 0.25 and 8.');
|
|
14
|
+
if (value.colourScheme !== 'light' && value.colourScheme !== 'dark')
|
|
15
|
+
throw new TypeError('Capture colour scheme is unsupported.');
|
|
16
|
+
return Object.freeze({ browser: text(value.browser, 'Capture browser', 40), browserVersion: text(value.browserVersion, 'Capture browser version', 80),
|
|
17
|
+
viewport: dimensions, deviceScaleFactor: value.deviceScaleFactor,
|
|
18
|
+
locale: text(value.locale, 'Capture locale', 80), timezone: text(value.timezone, 'Capture timezone', 80), colourScheme: value.colourScheme });
|
|
19
|
+
}
|
|
20
|
+
export function readObservationLabel(raw) {
|
|
21
|
+
if (raw === undefined || raw === null || raw === '')
|
|
22
|
+
return null;
|
|
23
|
+
return text(raw, 'Observation label', 80);
|
|
24
|
+
}
|
|
25
|
+
export function compareObservationContexts(input) {
|
|
26
|
+
if (!Array.isArray(input) || input.length < 2 || input.length > 20)
|
|
27
|
+
throw new TypeError('Compare between 2 and 20 observation contexts.');
|
|
28
|
+
const observations = input.map(value => {
|
|
29
|
+
const observedAt = value.observedAt === null ? null : normalizeExplicitIsoTimestamp(value.observedAt);
|
|
30
|
+
if (value.observedAt !== null && !observedAt)
|
|
31
|
+
throw new TypeError('Observation time must include an explicit timezone.');
|
|
32
|
+
return { observedAt, observerLabel: readObservationLabel(value.observerLabel), vantageLabel: readObservationLabel(value.vantageLabel), conditions: readCaptureConditions(value.conditions) };
|
|
33
|
+
});
|
|
34
|
+
const timestamps = observations.map(value => value.observedAt === null ? null : Date.parse(value.observedAt));
|
|
35
|
+
const knownTimes = timestamps.filter((value) => value !== null);
|
|
36
|
+
const row = (id, label, values) => ({ id, label, values,
|
|
37
|
+
state: values.some(value => value === null) ? 'unknown' : new Set(values).size === 1 ? 'same' : 'different' });
|
|
38
|
+
const conditions = observations.map(value => value.conditions);
|
|
39
|
+
const rows = [
|
|
40
|
+
row('observer', 'Declared observer', observations.map(value => value.observerLabel)),
|
|
41
|
+
row('vantage', 'Declared vantage', observations.map(value => value.vantageLabel)),
|
|
42
|
+
row('browser', 'Declared browser', conditions.map(value => value ? `${value.browser} ${value.browserVersion}` : null)),
|
|
43
|
+
row('viewport', 'Declared viewport', conditions.map(value => value ? `${value.viewport.width} × ${value.viewport.height}` : null)),
|
|
44
|
+
row('scale', 'Declared scale', conditions.map(value => value ? String(value.deviceScaleFactor) : null)),
|
|
45
|
+
row('locale', 'Declared locale', conditions.map(value => value?.locale ?? null)),
|
|
46
|
+
row('timezone', 'Declared timezone', conditions.map(value => value?.timezone ?? null)),
|
|
47
|
+
row('colour', 'Declared colour scheme', conditions.map(value => value?.colourScheme ?? null)),
|
|
48
|
+
];
|
|
49
|
+
const labelPairs = observations.map(value => value.observerLabel && value.vantageLabel ? JSON.stringify([value.observerLabel, value.vantageLabel]) : null);
|
|
50
|
+
return {
|
|
51
|
+
observations, rows, time: { state: knownTimes.length !== observations.length ? 'unknown' : new Set(knownTimes).size === 1 ? 'same_instant' : 'different_instants',
|
|
52
|
+
spanMilliseconds: knownTimes.length === observations.length ? Math.max(...knownTimes) - Math.min(...knownTimes) : null },
|
|
53
|
+
labels: labelPairs.some(value => value === null) ? 'incomplete' : new Set(labelPairs).size === labelPairs.length ? 'distinct' : 'repeated',
|
|
54
|
+
independence: 'not_verified',
|
|
55
|
+
limitations: [
|
|
56
|
+
'Observer, vantage and capture conditions are declarations, not verified network location or independent collection.',
|
|
57
|
+
'Different capture times, browser versions, viewports, scale, locale and shared upstream caches can explain differences. Equal declarations do not establish identical environments.',
|
|
58
|
+
'These selected observations do not establish worldwide availability, removal or takedown.',
|
|
59
|
+
],
|
|
60
|
+
};
|
|
61
|
+
}
|
|
@@ -1,14 +1,8 @@
|
|
|
1
|
-
import { MAX_BULK_SESSIONS, compareBulkSessions, normalizeBulkSession, } from "../workspace/bulk-session-model.mjs";
|
|
1
|
+
import { MAX_BULK_SESSIONS, areBulkRiskScoresComparable, compareBulkSessions, normalizeBulkSession, } from "../workspace/bulk-session-model.mjs";
|
|
2
2
|
import { MAX_COMPARISON_LEDGER_BULK_PAIRS, boundedComparisonLedgerId, comparisonLedgerCollector, comparisonLedgerInputArray, comparisonLedgerRecord, comparisonLedgerValuePresent, comparisonLedgerValueState, makeComparisonLedgerCandidate, stableComparisonLedgerId, } from "./comparison-ledger-contract.mjs";
|
|
3
|
-
function bulkCompleteness(session) {
|
|
4
|
-
|
|
5
|
-
}
|
|
6
|
-
function sourceMap(result) {
|
|
7
|
-
return new Map(result.sourceCoverage.map((item) => [item.source, item.state]));
|
|
8
|
-
}
|
|
9
|
-
function namedBulkSourceState(result, source) {
|
|
10
|
-
return result.sourceCoverage.find((item) => item.source === source)?.state ?? 'not_reported';
|
|
11
|
-
}
|
|
3
|
+
function bulkCompleteness(session) { return session.state === 'complete' ? 'complete' : session.state === 'partial' ? 'partial' : 'unavailable'; }
|
|
4
|
+
function sourceMap(result) { return new Map(result.sourceCoverage.map((item) => [item.source, item.state])); }
|
|
5
|
+
function namedBulkSourceState(result, source) { return result.sourceCoverage.find((item) => item.source === source)?.state ?? 'not_reported'; }
|
|
12
6
|
function bulkFamilyState(result, family) {
|
|
13
7
|
if (family === 'model')
|
|
14
8
|
return 'derived';
|
|
@@ -47,6 +41,9 @@ function bulkFieldDecision(field, earlier, later, before, after) {
|
|
|
47
41
|
if (field.kind === 'registrar')
|
|
48
42
|
return { state: 'not_compared', completeness: 'not_reported', earlierState, laterState, limitation: 'The exact retained registrar texts are shown, but saved Bulk rows do not retain field-level registrar provenance; these values are not compared as a publication or target change.' };
|
|
49
43
|
if (field.kind === 'model') {
|
|
44
|
+
if (field.field === 'Risk score' && !areBulkRiskScoresComparable(earlier, later)) {
|
|
45
|
+
return { state: 'not_compared', completeness: 'partial', earlierState, laterState, limitation: 'Risk scores require the same ready Brand Profile provenance, scan depth, and versioned Risk model before comparison.' };
|
|
46
|
+
}
|
|
50
47
|
const comparable = comparisonLedgerValuePresent(before) && comparisonLedgerValuePresent(after);
|
|
51
48
|
return { state: comparable ? 'different' : 'not_compared', completeness: comparable ? 'complete' : 'partial', earlierState, laterState, limitation: comparable ? null : 'A score absent from either retained row is not represented as an observed score change.' };
|
|
52
49
|
}
|
|
@@ -69,6 +66,8 @@ function bulkPairCompleteness(earlier, later) {
|
|
|
69
66
|
const right = after.get(domain);
|
|
70
67
|
if (!right || left.scanDepth !== right.scanDepth)
|
|
71
68
|
return 'partial';
|
|
69
|
+
if (!areBulkRiskScoresComparable(left, right))
|
|
70
|
+
return 'partial';
|
|
72
71
|
if (left.riskModelVersion === null || right.riskModelVersion === null || left.riskModelVersion !== right.riskModelVersion
|
|
73
72
|
|| left.opportunityModelVersion === null || right.opportunityModelVersion === null || left.opportunityModelVersion !== right.opportunityModelVersion)
|
|
74
73
|
return 'partial';
|
|
@@ -296,6 +295,10 @@ export function comparisonLedgerBulkPairIndexId(earlierSessionId, laterSessionId
|
|
|
296
295
|
return stableComparisonLedgerId('ledger-bulk_session_pair', [String(earlierSessionId ?? ''), String(laterSessionId ?? '')]);
|
|
297
296
|
}
|
|
298
297
|
export function buildBulkComparisonCandidates(sessionsRaw, pairsRaw, counters) {
|
|
298
|
+
const pairs = comparisonLedgerInputArray(pairsRaw);
|
|
299
|
+
// Saved sessions are comparison inputs only after a pair is requested.
|
|
300
|
+
if (!pairs.length)
|
|
301
|
+
return [];
|
|
299
302
|
const input = comparisonLedgerInputArray(sessionsRaw);
|
|
300
303
|
counters.inputRecords += Math.max(0, input.length - MAX_BULK_SESSIONS);
|
|
301
304
|
const sessions = new Map();
|
|
@@ -311,7 +314,6 @@ export function buildBulkComparisonCandidates(sessionsRaw, pairsRaw, counters) {
|
|
|
311
314
|
}
|
|
312
315
|
sessions.set(item.id, item);
|
|
313
316
|
}
|
|
314
|
-
const pairs = comparisonLedgerInputArray(pairsRaw);
|
|
315
317
|
counters.inputRecords += Math.max(0, pairs.length - MAX_COMPARISON_LEDGER_BULK_PAIRS);
|
|
316
318
|
const candidates = [];
|
|
317
319
|
const seen = new Set();
|
|
@@ -141,6 +141,7 @@ export function comparePageBaselines(rawReference, rawObserved) {
|
|
|
141
141
|
const observed = normalizePageBaseline(rawObserved);
|
|
142
142
|
if (!reference || !observed)
|
|
143
143
|
return null;
|
|
144
|
+
const sameFingerprintVersion = reference.fingerprintVersion === observed.fingerprintVersion;
|
|
144
145
|
const components = [
|
|
145
146
|
digestComponent('normalized_html', 'Normalised HTML', 'Normalized-HTML', reference.normalizedHtml, observed.normalizedHtml),
|
|
146
147
|
visibleTextComponent(reference.visibleText, observed.visibleText),
|
|
@@ -149,7 +150,7 @@ export function comparePageBaselines(rawReference, rawObserved) {
|
|
|
149
150
|
faviconComponent(reference, observed),
|
|
150
151
|
setComponent('resource_hosts', 'External resource hosts', 'host', reference.resourceHosts, observed.resourceHosts, (value) => value),
|
|
151
152
|
setComponent('tracking_identifiers', 'Tracking identifiers', 'identifier', reference.trackingIdentifiers, observed.trackingIdentifiers, (value) => `${value.type}:${value.value}`),
|
|
152
|
-
];
|
|
153
|
+
].map((item) => sameFingerprintVersion || item.id === 'favicon' ? item : component(item.id, item.label, item.method, 'unavailable', 'Different fingerprint versions', 'These captures use different HTML fingerprint algorithms. Retain the older baseline and capture a new baseline to compare current page evidence.', true));
|
|
153
154
|
const counts = { same: 0, overlap: 0, different: 0, notObserved: 0, unavailable: 0 };
|
|
154
155
|
for (const item of components) {
|
|
155
156
|
if (item.status === 'not_observed')
|
|
@@ -173,7 +174,9 @@ export function comparePageBaselines(rawReference, rawObserved) {
|
|
|
173
174
|
complete: observed.complete,
|
|
174
175
|
truncated: observed.truncated,
|
|
175
176
|
},
|
|
176
|
-
partial: reference.
|
|
177
|
+
partial: !reference.complete || !observed.complete
|
|
178
|
+
|| reference.truncated || observed.truncated
|
|
179
|
+
|| components.some((item) => item.partial),
|
|
177
180
|
components,
|
|
178
181
|
counts,
|
|
179
182
|
};
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// boundary.
|
|
6
6
|
import { normalizeDomain } from "../cases/case-model.mjs";
|
|
7
7
|
import { groupBySimilarFavicon } from "./favicon-similarity.mjs";
|
|
8
|
+
import { qualifyRelationshipSources, relationshipSourceEvidence, normalizeRelationshipSourceProjection, unknownRelationshipSource, MAX_RELATIONSHIP_SOURCES_PER_DOMAIN, } from "./relationship-provenance.mjs";
|
|
8
9
|
import { RELATIONSHIP_EVIDENCE_SCHEMA, RELATIONSHIP_EVIDENCE_VERSION, SUPPORTED_TLS_RELATIONSHIP_PROFILE_VERSIONS, TLS_RELATIONSHIP_PROFILE_VERSION, } from "../contracts/offline-comparison.mjs";
|
|
9
10
|
export { RELATIONSHIP_EVIDENCE_SCHEMA, RELATIONSHIP_EVIDENCE_VERSION, SUPPORTED_TLS_RELATIONSHIP_PROFILE_VERSIONS, TLS_RELATIONSHIP_PROFILE_VERSION, };
|
|
10
11
|
export const MAX_RELATIONSHIP_ROWS = 2000;
|
|
@@ -16,6 +17,12 @@ export const MAX_TRACKING_IDS_PER_ROW = 30;
|
|
|
16
17
|
export const MAX_OFFICIAL_ASSET_HOSTS_PER_ROW = 30;
|
|
17
18
|
export const MAX_OFFICIAL_DOMAINS = 200;
|
|
18
19
|
export const MAX_FAVICON_ROWS = 250;
|
|
20
|
+
const ARRAY_RELATIONSHIP_FIELDS = {
|
|
21
|
+
nameserver_set: { field: 'nameservers', maximum: MAX_NAMESERVERS_PER_ROW },
|
|
22
|
+
ip_address: { field: 'ipAddresses', maximum: MAX_IPS_PER_ROW },
|
|
23
|
+
tracking_identifier: { field: 'trackingIdentifiers', maximum: MAX_TRACKING_IDS_PER_ROW },
|
|
24
|
+
official_asset: { field: 'officialAssetHosts', maximum: MAX_OFFICIAL_ASSET_HOSTS_PER_ROW },
|
|
25
|
+
};
|
|
19
26
|
const CONTROL_RE = /[\x00-\x1f\x7f]/;
|
|
20
27
|
const FAVICON_SHA_RE = /^[a-f0-9]{64}$/i;
|
|
21
28
|
const FAVICON_PHASH_RE = /^[a-f0-9]{16}$/i;
|
|
@@ -92,7 +99,7 @@ function tlsCertificateFingerprint(availability) {
|
|
|
92
99
|
const certificate = record(tls?.certificate);
|
|
93
100
|
const fingerprint = certificate?.fingerprintSha256;
|
|
94
101
|
const status = tls?.status;
|
|
95
|
-
if (tls?.source !== 'tls' || !SUPPORTED_TLS_RELATIONSHIP_PROFILE_VERSIONS.some((version) => version ===
|
|
102
|
+
if (tls?.source !== 'tls' || !SUPPORTED_TLS_RELATIONSHIP_PROFILE_VERSIONS.some((version) => version === tls?.profileVersion)
|
|
96
103
|
|| typeof status !== 'string' || !['success', 'partial'].includes(status) || typeof fingerprint !== 'string'
|
|
97
104
|
|| fingerprint.length !== 64 || !CERTIFICATE_SHA_RE.test(fingerprint))
|
|
98
105
|
return null;
|
|
@@ -131,6 +138,36 @@ export function relationshipObservation(rawAvailability, rawOfficialDomains = []
|
|
|
131
138
|
? availability.faviconHash.toLowerCase() : null;
|
|
132
139
|
const faviconPHash = typeof availability.faviconPHash === 'string' && FAVICON_PHASH_RE.test(availability.faviconPHash)
|
|
133
140
|
? availability.faviconPHash.toLowerCase() : null;
|
|
141
|
+
const certificateFingerprint = tlsCertificateFingerprint(availability);
|
|
142
|
+
const dnsSource = relationshipSourceEvidence('dns', availability.dns);
|
|
143
|
+
const httpSource = relationshipSourceEvidence('http', availability.http);
|
|
144
|
+
const projectedSource = (source, truncated) => truncated
|
|
145
|
+
? { ...source, complete: false, truncated: true } : source;
|
|
146
|
+
const sourceEvidence = {};
|
|
147
|
+
if (nameservers.values.length)
|
|
148
|
+
sourceEvidence.nameserver_set = [
|
|
149
|
+
// The compact availability nameserver list can be registration data or a
|
|
150
|
+
// DNS fallback. Its source is not distinguishable from this projection.
|
|
151
|
+
...(Array.isArray(availability.nameservers) && availability.nameservers.length
|
|
152
|
+
? [availability.source === 'dns' ? projectedSource(dnsSource, nameservers.truncated)
|
|
153
|
+
: unknownRelationshipSource('registration_or_dns')] : []),
|
|
154
|
+
...(Array.isArray(records.ns) && records.ns.length ? [projectedSource(dnsSource, nameservers.truncated)] : []),
|
|
155
|
+
];
|
|
156
|
+
if (ipAddresses.values.length)
|
|
157
|
+
sourceEvidence.ip_address = [projectedSource(dnsSource, ipAddresses.truncated)];
|
|
158
|
+
if (certificateFingerprint)
|
|
159
|
+
sourceEvidence.certificate = [relationshipSourceEvidence('tls', availability.tls)];
|
|
160
|
+
if (trackingIdentifiers.values.length)
|
|
161
|
+
sourceEvidence.tracking_identifier = [
|
|
162
|
+
projectedSource(httpSource, trackingIdentifiers.truncated),
|
|
163
|
+
relationshipSourceEvidence('page_identity', availability.pageIdentity),
|
|
164
|
+
];
|
|
165
|
+
if (officialAssetHosts.length)
|
|
166
|
+
sourceEvidence.official_asset = [projectedSource(httpSource, assets.truncated
|
|
167
|
+
|| officialDomainSource.length > MAX_OFFICIAL_DOMAINS)];
|
|
168
|
+
// A homepage timestamp does not date the independently fetched favicon.
|
|
169
|
+
if (faviconHash || faviconPHash)
|
|
170
|
+
sourceEvidence.favicon = [unknownRelationshipSource('favicon')];
|
|
134
171
|
return {
|
|
135
172
|
version: RELATIONSHIP_EVIDENCE_VERSION,
|
|
136
173
|
nameservers: nameservers.values,
|
|
@@ -139,7 +176,8 @@ export function relationshipObservation(rawAvailability, rawOfficialDomains = []
|
|
|
139
176
|
officialAssetHosts,
|
|
140
177
|
faviconHash,
|
|
141
178
|
faviconPHash,
|
|
142
|
-
certificateFingerprint
|
|
179
|
+
certificateFingerprint,
|
|
180
|
+
sourceEvidence,
|
|
143
181
|
truncated: nameservers.truncated || ipAddresses.truncated || trackingIdentifiers.truncated || assets.truncated
|
|
144
182
|
|| officialDomainSource.length > MAX_OFFICIAL_DOMAINS,
|
|
145
183
|
};
|
|
@@ -152,7 +190,22 @@ function addBucket(buckets, value, domain) {
|
|
|
152
190
|
buckets.get(value)?.add(domain);
|
|
153
191
|
}
|
|
154
192
|
function group(type, label, method, value, domains, description, normalizedValue = value) {
|
|
155
|
-
return { type, label, method, value, normalizedValue, domains, description };
|
|
193
|
+
return { type, label, method, value, normalizedValue, domains, description, ...qualifyRelationshipSources([], domains) };
|
|
194
|
+
}
|
|
195
|
+
function contributesToGroup(observation, item, domain) {
|
|
196
|
+
if (item.type === 'certificate')
|
|
197
|
+
return typeof observation.certificateFingerprint === 'string'
|
|
198
|
+
&& observation.certificateFingerprint.toLowerCase() === item.normalizedValue;
|
|
199
|
+
if (item.type === 'nameserver_set')
|
|
200
|
+
return Array.isArray(observation.nameservers)
|
|
201
|
+
&& [...new Set(observation.nameservers.slice(0, MAX_NAMESERVERS_PER_ROW).map(hostname).filter(Boolean))].sort().join(' · ') === item.normalizedValue;
|
|
202
|
+
const field = ARRAY_RELATIONSHIP_FIELDS[item.type];
|
|
203
|
+
if (field)
|
|
204
|
+
return Array.isArray(observation[field.field]) && observation[field.field].slice(0, field.maximum).some((value) => (item.type === 'ip_address' ? ipAddress(value) : item.type === 'official_asset' ? hostname(value) : value) === item.normalizedValue);
|
|
205
|
+
return item.type === 'favicon' && item.normalizedValue.split('|').includes(`${domain}=${[
|
|
206
|
+
typeof observation.faviconHash === 'string' && FAVICON_SHA_RE.test(observation.faviconHash) ? `sha256:${observation.faviconHash.toLowerCase()}` : '',
|
|
207
|
+
typeof observation.faviconPHash === 'string' && FAVICON_PHASH_RE.test(observation.faviconPHash) ? `dhash:${observation.faviconPHash.toLowerCase()}` : '',
|
|
208
|
+
].filter(Boolean).join(',')}`);
|
|
156
209
|
}
|
|
157
210
|
/**
|
|
158
211
|
* @param {Array<{domain?:unknown,trusted?:unknown,relationship?:unknown}>} rawRows
|
|
@@ -164,7 +217,7 @@ export function buildScanRelationships(rawRows) {
|
|
|
164
217
|
for (const raw of input.slice(0, MAX_RELATIONSHIP_ROWS)) {
|
|
165
218
|
const domain = normalizeDomain(raw?.domain);
|
|
166
219
|
const observation = record(raw?.relationship);
|
|
167
|
-
if (!domain || raw?.trusted || !observation ||
|
|
220
|
+
if (!domain || raw?.trusted || !observation || ![2, RELATIONSHIP_EVIDENCE_VERSION].some((version) => version === observation.version))
|
|
168
221
|
continue;
|
|
169
222
|
rows.push({ domain, observation });
|
|
170
223
|
if (observation.truncated === true)
|
|
@@ -245,11 +298,35 @@ export function buildScanRelationships(rawRows) {
|
|
|
245
298
|
output.sort((left, right) => (Number(order.get(left.type)) - Number(order.get(right.type))) || left.value.localeCompare(right.value) || left.domains.join('|').localeCompare(right.domains.join('|')));
|
|
246
299
|
if (output.length > MAX_RELATIONSHIP_GROUPS)
|
|
247
300
|
truncated = true;
|
|
301
|
+
const rowsByDomain = new Map();
|
|
302
|
+
for (const row of rows) {
|
|
303
|
+
const members = rowsByDomain.get(row.domain) ?? [];
|
|
304
|
+
members.push(row);
|
|
305
|
+
rowsByDomain.set(row.domain, members);
|
|
306
|
+
}
|
|
248
307
|
const groups = output.slice(0, MAX_RELATIONSHIP_GROUPS).map((item) => {
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
308
|
+
const domains = item.domains.slice(0, MAX_RELATIONSHIP_DOMAINS);
|
|
309
|
+
let groupTruncated = item.domains.length > domains.length || input.length > MAX_RELATIONSHIP_ROWS;
|
|
310
|
+
if (groupTruncated)
|
|
311
|
+
truncated = true;
|
|
312
|
+
const contributors = [];
|
|
313
|
+
for (const domain of domains)
|
|
314
|
+
for (const { observation } of rowsByDomain.get(domain) ?? []) {
|
|
315
|
+
if (!contributesToGroup(observation, item, domain))
|
|
316
|
+
continue;
|
|
317
|
+
const field = ARRAY_RELATIONSHIP_FIELDS[item.type];
|
|
318
|
+
if (field && Array.isArray(observation[field.field]) && observation[field.field].length > field.maximum)
|
|
319
|
+
groupTruncated = true;
|
|
320
|
+
const projection = observation.version === RELATIONSHIP_EVIDENCE_VERSION
|
|
321
|
+
? normalizeRelationshipSourceProjection(observation.sourceEvidence) : {};
|
|
322
|
+
for (const source of projection[item.type] ?? [unknownRelationshipSource()]) {
|
|
323
|
+
if (contributors.length >= MAX_RELATIONSHIP_DOMAINS * MAX_RELATIONSHIP_SOURCES_PER_DOMAIN)
|
|
324
|
+
groupTruncated = true;
|
|
325
|
+
else
|
|
326
|
+
contributors.push({ domain, ...source });
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return { ...item, domains, ...qualifyRelationshipSources(contributors, domains, { truncated: groupTruncated, type: item.type }) };
|
|
253
330
|
});
|
|
254
331
|
return {
|
|
255
332
|
version: RELATIONSHIP_EVIDENCE_VERSION,
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// Only the source metadata needed to qualify an observed relationship crosses
|
|
2
|
+
// this boundary. Collection and retention times are never source-time fallbacks.
|
|
3
|
+
import { normalizeDomain } from "../cases/case-model.mjs";
|
|
4
|
+
import { normalizeExplicitIsoTimestamp, readObservationEnvelope } from "../evidence/observation.mjs";
|
|
5
|
+
export const RELATIONSHIP_TYPES = Object.freeze([
|
|
6
|
+
'nameserver_set', 'ip_address', 'certificate', 'tracking_identifier', 'favicon', 'official_asset',
|
|
7
|
+
]);
|
|
8
|
+
const SOURCES = ['dns', 'tls', 'http', 'page_identity', 'favicon', 'registration_or_dns', 'unknown'];
|
|
9
|
+
const SOURCES_BY_TYPE = {
|
|
10
|
+
nameserver_set: ['dns', 'registration_or_dns'], ip_address: ['dns'], certificate: ['tls'],
|
|
11
|
+
tracking_identifier: ['http', 'page_identity'], favicon: ['favicon'], official_asset: ['http'],
|
|
12
|
+
};
|
|
13
|
+
const STATUSES = ['success', 'partial', 'not_found', 'skipped', 'error', 'unsupported', 'not_applicable', 'unknown'];
|
|
14
|
+
export const MAX_RELATIONSHIP_SOURCES_PER_DOMAIN = 3;
|
|
15
|
+
export const MAX_RELATIONSHIP_SOURCE_DOMAINS = 50;
|
|
16
|
+
function record(value) {
|
|
17
|
+
return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
18
|
+
}
|
|
19
|
+
export function unknownRelationshipSource(source = 'unknown') {
|
|
20
|
+
return { source, status: 'unknown', observedAt: null, complete: false, truncated: null };
|
|
21
|
+
}
|
|
22
|
+
export function relationshipSourceEvidence(source, raw) {
|
|
23
|
+
const result = readObservationEnvelope(raw);
|
|
24
|
+
if (result.state !== 'supported' || result.observation.source !== source) {
|
|
25
|
+
return { ...unknownRelationshipSource(source), status: result.state === 'unsupported' ? 'unsupported' : 'unknown' };
|
|
26
|
+
}
|
|
27
|
+
return normalizeSource({ ...result.observation, truncated: record(raw).truncated });
|
|
28
|
+
}
|
|
29
|
+
function normalizeSource(raw, type) {
|
|
30
|
+
const value = record(raw);
|
|
31
|
+
const source = SOURCES.find((candidate) => candidate === value.source) ?? 'unknown';
|
|
32
|
+
if (type && source !== 'unknown' && !SOURCES_BY_TYPE[type].includes(source))
|
|
33
|
+
return unknownRelationshipSource();
|
|
34
|
+
const status = STATUSES.find((candidate) => candidate === value.status) ?? 'unknown';
|
|
35
|
+
const observedAt = normalizeExplicitIsoTimestamp(value.observedAt);
|
|
36
|
+
const truncated = typeof value.truncated === 'boolean' ? value.truncated : null;
|
|
37
|
+
return {
|
|
38
|
+
source, status, observedAt, truncated,
|
|
39
|
+
complete: source !== 'unknown' && source !== 'registration_or_dns' && status === 'success'
|
|
40
|
+
&& value.complete === true && truncated === false && observedAt !== null,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
export function normalizeRelationshipSourceProjection(raw) {
|
|
44
|
+
const value = record(raw);
|
|
45
|
+
const output = {};
|
|
46
|
+
for (const type of RELATIONSHIP_TYPES) {
|
|
47
|
+
if (!Array.isArray(value[type]))
|
|
48
|
+
continue;
|
|
49
|
+
const candidates = value[type];
|
|
50
|
+
output[type] = candidates.slice(0, MAX_RELATIONSHIP_SOURCES_PER_DOMAIN).map((candidate) => {
|
|
51
|
+
const source = normalizeSource(candidate, type);
|
|
52
|
+
return candidates.length > MAX_RELATIONSHIP_SOURCES_PER_DOMAIN ? { ...source, complete: false, truncated: true } : source;
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
return output;
|
|
56
|
+
}
|
|
57
|
+
/** Missing, malformed or omitted contributors cannot make a group complete. */
|
|
58
|
+
export function qualifyRelationshipSources(raw, rawDomains, options = {}) {
|
|
59
|
+
const domains = [...new Set(rawDomains.slice(0, MAX_RELATIONSHIP_SOURCE_DOMAINS * 2).map(normalizeDomain).filter(Boolean))].slice(0, MAX_RELATIONSHIP_SOURCE_DOMAINS);
|
|
60
|
+
const allowed = new Set(domains);
|
|
61
|
+
const candidates = Array.isArray(raw) ? raw : [];
|
|
62
|
+
const maximum = MAX_RELATIONSHIP_SOURCE_DOMAINS * MAX_RELATIONSHIP_SOURCES_PER_DOMAIN;
|
|
63
|
+
const byDomain = new Map();
|
|
64
|
+
let omitted = options.truncated === true || rawDomains.length > MAX_RELATIONSHIP_SOURCE_DOMAINS || candidates.length > maximum;
|
|
65
|
+
for (const candidate of candidates.slice(0, maximum)) {
|
|
66
|
+
const value = record(candidate);
|
|
67
|
+
const domain = normalizeDomain(value.domain);
|
|
68
|
+
if (!domain || !allowed.has(domain)) {
|
|
69
|
+
omitted = true;
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
const contribution = { domain, ...normalizeSource(value, options.type) };
|
|
73
|
+
const entries = byDomain.get(domain) ?? new Map();
|
|
74
|
+
const key = JSON.stringify(contribution);
|
|
75
|
+
if (!entries.has(key) && entries.size >= MAX_RELATIONSHIP_SOURCES_PER_DOMAIN)
|
|
76
|
+
omitted = true;
|
|
77
|
+
else
|
|
78
|
+
entries.set(key, contribution);
|
|
79
|
+
byDomain.set(domain, entries);
|
|
80
|
+
}
|
|
81
|
+
const sourceEvidence = domains.sort().flatMap((domain) => {
|
|
82
|
+
const entries = byDomain.get(domain);
|
|
83
|
+
return entries?.size ? [...entries.values()].sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)))
|
|
84
|
+
: [{ domain, ...unknownRelationshipSource() }];
|
|
85
|
+
});
|
|
86
|
+
const observedAt = sourceEvidence.length && sourceEvidence.every((entry) => entry.observedAt !== null)
|
|
87
|
+
? sourceEvidence.map((entry) => entry.observedAt).sort().at(-1) ?? null : null;
|
|
88
|
+
return {
|
|
89
|
+
sourceEvidence, observedAt,
|
|
90
|
+
complete: sourceEvidence.length > 0 && !omitted && sourceEvidence.every((entry) => entry.complete),
|
|
91
|
+
truncated: omitted || sourceEvidence.some((entry) => entry.truncated === true),
|
|
92
|
+
};
|
|
93
|
+
}
|