@slicedearth/whoisleuth-cli 2.5.1 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +9 -6
- package/NOTICE +3 -2
- package/README.md +60 -67
- package/cli/artifact-validation/case-response.mjs +6 -814
- package/cli/artifact-validation/lookup-evidence.mjs +30 -3
- package/cli/artifact-validation/signed-review.mjs +2 -51
- package/cli/artifact-verify.mjs +41 -3
- package/cli/bulk-command-runner.mjs +4 -6
- package/cli/case-command.mjs +92 -16
- package/cli/case-pack.mjs +14 -3
- package/cli/ci-report.mjs +17 -5
- package/cli/command-definition.mjs +7 -6
- package/cli/discovery-scan-command-runner.mjs +4 -6
- package/cli/doctor.mjs +37 -3
- package/cli/domain-feed.mjs +61 -0
- package/cli/formatters/html.mjs +3 -0
- package/cli/formatters/json.mjs +9 -14
- package/cli/formatters/terminal-lookup.mjs +24 -1
- package/cli/history-command-definitions.mjs +1 -1
- package/cli/indicator-set-command.mjs +1 -2
- package/cli/intake-command.mjs +43 -3
- package/cli/investigation-artifacts.mjs +2 -3
- package/cli/investigation-package-review.mjs +2 -0
- package/cli/investigation-plan.mjs +15 -0
- package/cli/local-mmdb-review.mjs +27 -4
- package/cli/lookup-brief.mjs +11 -5
- package/cli/lookup-command-runner.mjs +21 -13
- package/cli/offline-evidence-review.mjs +51 -5
- package/cli/review-arguments.mjs +3 -2
- package/cli/review-command-definitions.mjs +23 -17
- package/cli/review-command-runner.mjs +6 -3
- package/cli/risk-calibration.mjs +6 -4
- package/cli/saved-lookup.mjs +22 -7
- package/cli/selected-input-worker.mjs +12 -2
- package/cli/source-reliability.mjs +12 -10
- package/cli/support-arguments.mjs +1 -1
- package/cli/support-command-definitions.mjs +5 -4
- package/cli/support-command-runner.mjs +9 -7
- package/cli/watchlist-review.mjs +43 -0
- package/cli/workflow-arguments.mjs +14 -0
- package/cli/workflow-command-definitions.mjs +27 -1
- package/cli/workflow-command-runner.mjs +32 -0
- package/docs/cli-reference.md +49 -11
- package/docs/cli.md +200 -26
- package/docs/contextual-reviews.md +38 -5
- package/docs/infrastructure-observations.md +135 -0
- package/docs/offline-intake-context.md +213 -0
- package/lib/credential-form-attribution.mjs +136 -0
- package/lib/credential-surface-profile.mjs +6 -2
- package/lib/dmarc-discovery.mjs +4 -2
- package/lib/dns-delegation-health.mjs +23 -7
- package/lib/dns-inheritance-review.mjs +14 -1
- package/lib/dns-intelligence.mjs +18 -4
- package/lib/domain-posture-analysis.mjs +53 -2
- package/lib/domain-posture-parsers.mjs +9 -0
- package/lib/domain-posture.mjs +29 -10
- package/lib/evidence-export-privacy.mjs +5 -2
- package/lib/evidence-export.mjs +10 -4
- package/lib/evidence-report.mjs +3 -1
- package/lib/favicon.mjs +13 -4
- package/lib/generated/cisa-kev-catalog.mjs +17 -4
- package/lib/generated/registrar-standing-catalogue.mjs +7 -7
- package/lib/generated/retire-browser-catalog.mjs +178 -5
- package/lib/html-signals.mjs +6 -0
- package/lib/idn-confusables.mjs +5 -1
- package/lib/lookup-child-profile-contract.mjs +28 -11
- package/lib/lookup-collection-receipt.mjs +52 -0
- package/lib/lookup-diagnostics.mjs +2 -12
- package/lib/lookup-progress.mjs +29 -31
- package/lib/lookup-response-contract.mjs +10 -0
- package/lib/lookup-response.mjs +15 -14
- package/lib/lookup-source-progress.mjs +4 -2
- package/lib/lookup.mjs +33 -2
- package/lib/operation-budget.mjs +4 -0
- package/lib/rdap-capabilities.mjs +43 -9
- package/lib/registry-insights.mjs +17 -6
- package/lib/service-binding-dns.mjs +19 -2
- package/lib/sslbl-certificates.generated.mjs +142 -140
- package/lib/static-html-analysis.mjs +32 -4
- package/lib/technology-evidence-role.mjs +1 -51
- package/lib/tls-intelligence.mjs +30 -5
- package/lib/website-enrichment.mjs +39 -23
- package/lib/whois-authority.mjs +56 -1
- package/lib/whois-chain.mjs +14 -4
- package/lib/whois-common-formats.mjs +3 -1
- package/lib/whois-contacts.mjs +13 -2
- package/lib/whois-contracts.mjs +2 -0
- package/lib/whois-normalization.mjs +4 -4
- package/package.json +9 -7
- package/packages/analysis/bounded-json.mjs +12 -2
- package/packages/analysis/risk-scoring.mjs +12 -9
- package/packages/cases/case-attachment-model.mjs +36 -1
- package/packages/cases/case-checkpoint-receipts.mjs +68 -0
- package/packages/cases/case-containment-handoff.mjs +195 -0
- package/packages/cases/case-evidence-links.mjs +44 -0
- package/packages/cases/case-evidence-model.mjs +51 -7
- package/packages/cases/case-migration-model.mjs +9 -1
- package/packages/cases/case-object-coverage.mjs +119 -0
- package/packages/cases/case-packet-correction.mjs +106 -0
- package/packages/cases/case-recheck-model.mjs +35 -5
- package/packages/cases/case-record-operations.mjs +87 -9
- package/packages/cases/case-record-projection.mjs +43 -5
- package/packages/cases/case-record-version-input.mjs +23 -1
- package/packages/cases/case-report.mjs +35 -4
- package/packages/cases/case-requested-evidence.mjs +1 -1
- package/packages/cases/case-response-actions.mjs +57 -1
- package/packages/cases/case-response-model.mjs +7 -0
- package/packages/cases/case-response-object.mjs +119 -0
- package/packages/cases/case-response-outcomes.mjs +81 -14
- package/packages/cases/case-response-packet-render.mjs +22 -3
- package/packages/cases/case-response-packet-validation.mjs +880 -0
- package/packages/cases/case-response-packet-vocabulary.mjs +12 -1
- package/packages/cases/case-response-packet.mjs +32 -17
- package/packages/cases/case-response-review-inputs.mjs +66 -8
- package/packages/cases/case-type-presentation.mjs +10 -0
- package/packages/cases/case-workflow-metadata.mjs +1 -17
- package/packages/comparison/capture-context.mjs +3 -2
- package/packages/contracts/analyst-interchange.mjs +7 -5
- package/packages/contracts/analyst-review-state-contract.mjs +1 -1
- package/packages/contracts/candidate-watch-review.mjs +54 -0
- package/packages/contracts/case-portability.mjs +292 -43
- package/packages/contracts/cli-command-semantics.mjs +3 -0
- package/packages/contracts/cli-doctor.mjs +4 -0
- package/packages/contracts/cli-lookup.mjs +57 -46
- package/packages/contracts/context-review.mjs +2 -1
- package/packages/contracts/external-observation-interchange.mjs +96 -28
- package/packages/contracts/lookup-evidence.mjs +13 -2
- package/packages/contracts/message-intake.mjs +7 -1
- package/packages/contracts/registry-insights.mjs +6 -0
- package/packages/contracts/risk-calibration.mjs +44 -29
- package/packages/contracts/schema-lifecycle.mjs +3 -1
- package/packages/contracts/web-capture.mjs +1 -1
- package/packages/contracts/workspace-portability.mjs +86 -16
- package/packages/evidence/collection-quality.mjs +1 -1
- package/packages/evidence/credential-form-attribution.mjs +60 -0
- package/packages/evidence/dmarc-policy-context.mjs +80 -0
- package/packages/evidence/image-regions.mjs +15 -0
- package/packages/evidence/lookup-collection-receipt.mjs +140 -0
- package/packages/evidence/lookup-collection-review.mjs +74 -0
- package/packages/evidence/record-identity.mjs +60 -16
- package/packages/evidence/technology-evidence-role.mjs +51 -0
- package/packages/interchange/external-findings-import.mjs +40 -21
- package/packages/interchange/json-output.mjs +14 -0
- package/packages/interchange/managed-indicator-export.mjs +3 -2
- package/packages/interchange/managed-indicator-set.mjs +13 -0
- package/packages/investigation/acquisition-decision-packet.mjs +78 -0
- package/packages/investigation/bulk-mail-exposure.mjs +2 -2
- package/packages/investigation/context-review.mjs +3 -3
- package/packages/investigation/document-intake.mjs +9 -3
- package/packages/investigation/domain-history-review.mjs +50 -12
- package/packages/investigation/domain-transition-review.mjs +2 -0
- package/packages/investigation/infrastructure-observation.mjs +234 -0
- package/packages/investigation/intake-context.mjs +284 -0
- package/packages/investigation/intake-indicators.mjs +116 -0
- package/packages/investigation/intake-phones.mjs +155 -0
- package/packages/investigation/intake-report.mjs +2 -1
- package/packages/investigation/investigation-manifest.mjs +33 -7
- package/packages/investigation/link-intake.mjs +38 -8
- package/packages/investigation/message-intake.mjs +55 -15
- package/packages/monitoring/domain-feed.mjs +321 -0
- package/packages/workspace/brand-candidate-workflow.mjs +351 -0
- package/packages/workspace/brand-keyword-campaign.mjs +241 -0
- package/packages/workspace/brand-profile-model.mjs +46 -2
- package/packages/workspace/bulk-session-model.mjs +12 -0
- package/packages/workspace/candidate-watch-handoff.mjs +195 -0
- package/packages/workspace/detection-rule-model.mjs +10 -2
- package/packages/workspace/shortlist-model.mjs +10 -2
- package/packages/workspace/watchlist-history.mjs +31 -4
- package/packages/workspace/watchlist-store.mjs +136 -7
- package/third-party-notices.txt +30 -30
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { readManagedIndicatorSet } from "./managed-indicator-set.mjs";
|
|
1
|
+
import { readManagedIndicatorSet, serializeManagedIndicatorSet } from "./managed-indicator-set.mjs";
|
|
2
|
+
import { terminalSafeJson } from "./json-output.mjs";
|
|
2
3
|
import { stixIndicatorObjects, stixIndicatorProducer } from "./stix-indicator-export.mjs";
|
|
3
4
|
import { mispIndicatorAttribute, mispIndicatorEvent } from "./misp-indicator-export.mjs";
|
|
4
5
|
/** Re-exporting a revision is deterministic; revoked object versions never advance. */
|
|
@@ -29,7 +30,7 @@ export async function exportManagedIndicators(input, format) {
|
|
|
29
30
|
}
|
|
30
31
|
else if (format !== 'manifest')
|
|
31
32
|
throw new TypeError('Unsupported managed indicator export format.');
|
|
32
|
-
return { document, content: `${
|
|
33
|
+
return { document, content: format === 'manifest' ? serializeManagedIndicatorSet(manifest) : `${terminalSafeJson(document, 2)}\n`,
|
|
33
34
|
filename: `whoisleuth-indicators-${manifest.id}-r${manifest.revision}.${format}.json`,
|
|
34
35
|
mimeType: format === 'stix' ? 'application/stix+json;charset=utf-8' : 'application/json;charset=utf-8' };
|
|
35
36
|
}
|
|
@@ -5,6 +5,7 @@ import { parseBoundedJson, boundedJsonLimitsForBytes } from "../analysis/bounded
|
|
|
5
5
|
import { MANAGED_INDICATOR_SET_SCHEMA, MANAGED_INDICATOR_SET_VERSION, MAX_MANAGED_INDICATOR_SET_BYTES, MAX_MANAGED_INDICATOR_PLAN_BYTES } from "../contracts/analyst-interchange.mjs";
|
|
6
6
|
import { defensiveIndicatorProvenance, prepareDefensiveIndicatorExport, MAX_DEFENSIVE_INDICATOR_INPUTS } from "./defensive-indicator-export.mjs";
|
|
7
7
|
import { MAX_STIX_INDICATORS } from "./stix-indicator-export.mjs";
|
|
8
|
+
import { terminalSafeJson } from "./json-output.mjs";
|
|
8
9
|
export const MAX_MANAGED_INDICATORS = MAX_STIX_INDICATORS;
|
|
9
10
|
export const MAX_INDICATOR_REVIEW_BASIS = 1_000;
|
|
10
11
|
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u;
|
|
@@ -96,6 +97,17 @@ export function parseManagedIndicatorJson(raw, plan = false) {
|
|
|
96
97
|
const maximumBytes = plan ? MAX_MANAGED_INDICATOR_PLAN_BYTES : MAX_MANAGED_INDICATOR_SET_BYTES;
|
|
97
98
|
return parseBoundedJson(raw, { maximumBytes, label: 'Managed indicator input', limits: boundedJsonLimitsForBytes(maximumBytes) });
|
|
98
99
|
}
|
|
100
|
+
/** Both writers enforce the reader's budget on final UTF-8 bytes, including escapes. */
|
|
101
|
+
export function serializeManagedIndicatorSet(manifest) {
|
|
102
|
+
const pretty = `${terminalSafeJson(manifest, 2)}\n`;
|
|
103
|
+
if (new TextEncoder().encode(pretty).byteLength <= MAX_MANAGED_INDICATOR_SET_BYTES)
|
|
104
|
+
return pretty;
|
|
105
|
+
const compact = terminalSafeJson(manifest);
|
|
106
|
+
if (new TextEncoder().encode(compact).byteLength > MAX_MANAGED_INDICATOR_SET_BYTES) {
|
|
107
|
+
throw new TypeError('The indicator revision exceeds the file byte limit after safe JSON encoding; no partial file was created.');
|
|
108
|
+
}
|
|
109
|
+
return compact;
|
|
110
|
+
}
|
|
99
111
|
export function managedIndicatorState(entry, at) {
|
|
100
112
|
iso(at, 'Review time');
|
|
101
113
|
return entry.withdrawal ? 'withdrawn' : Date.parse(entry.expiresAt) <= Date.parse(at) ? 'expired' : 'active';
|
|
@@ -205,6 +217,7 @@ export async function buildManagedIndicatorRevision(input, now = new Date().toIS
|
|
|
205
217
|
revision: (previous?.revision ?? 0) + 1, createdAt: previous?.createdAt ?? generatedAt, modifiedAt: generatedAt,
|
|
206
218
|
previous: previous ? { revisionId: previous.revisionId, digestSha256: previous.integrity.digestSha256 } : null, entries };
|
|
207
219
|
const manifest = validateManagedIndicatorSet({ ...unsigned, integrity: { algorithm: 'SHA-256', canonicalization: SORTED_JSON_V2, digestSha256: await sha256ArtifactDigestV2(unsigned) } });
|
|
220
|
+
serializeManagedIndicatorSet(manifest);
|
|
208
221
|
const changes = previous ? compareManagedIndicatorRevisions(previous, manifest) : manifest.entries.map(entry => ({ id: entry.id, domain: entry.domain, kind: 'added' }));
|
|
209
222
|
return { manifest, changes, unchanged: manifest.entries.length - changes.length, exclusions: preflight.exclusions };
|
|
210
223
|
}
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import { SORTED_JSON_V2, sha256ArtifactDigestV2 } from "../evidence/artifact-integrity.mjs";
|
|
2
2
|
import { ACQUISITION_DECISION_PACKET_SCHEMA, ACQUISITION_DECISION_PACKET_VERSION, } from "../contracts/investigation-portability.mjs";
|
|
3
3
|
import { isValidAsciiDomainName } from "../contracts/domain-name.mjs";
|
|
4
|
+
import { parseBoundedJsonObject } from "../analysis/bounded-json.mjs";
|
|
5
|
+
import { MAX_INVESTIGATION_PORTABLE_BYTES } from "../contracts/investigation-portability.mjs";
|
|
6
|
+
import { array, boolean, domain, enumeration, exact, fail, iso, sameValues, strings, text as structureText, validateIntegrity } from "../evidence/artifact-structure.mjs";
|
|
4
7
|
export { ACQUISITION_DECISION_PACKET_SCHEMA, ACQUISITION_DECISION_PACKET_VERSION };
|
|
8
|
+
export { MAX_INVESTIGATION_PORTABLE_BYTES as MAX_ACQUISITION_DECISION_PACKET_BYTES };
|
|
5
9
|
export const ACQUISITION_DECISIONS = [
|
|
6
10
|
'unresolved',
|
|
7
11
|
'continue_manual_review',
|
|
@@ -18,6 +22,80 @@ export const ACQUISITION_MANUAL_CHECKS = [
|
|
|
18
22
|
const DECISION_SET = new Set(ACQUISITION_DECISIONS);
|
|
19
23
|
const CHECK_SET = new Set(ACQUISITION_MANUAL_CHECKS);
|
|
20
24
|
const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f]/gu;
|
|
25
|
+
function validateAcquisitionItem(value, label) {
|
|
26
|
+
const item = exact(value, ['id', 'label', 'state', 'detail', 'provenance'], label);
|
|
27
|
+
const id = enumeration(item.id, ['availability', 'contacts', 'lifecycle', 'mail', 'nameservers', 'operations', 'policy_eligibility', 'policy_lifecycle', 'policy_transfer', 'tls', 'transfer', 'web'], label);
|
|
28
|
+
enumeration(item.state, ['authoritative', 'observed', 'review', 'unavailable'], label);
|
|
29
|
+
structureText(item.label, label, 200);
|
|
30
|
+
structureText(item.detail, label, 2_000);
|
|
31
|
+
structureText(item.provenance, label, 300);
|
|
32
|
+
return id;
|
|
33
|
+
}
|
|
34
|
+
export function validateAcquisitionDecisionPacketStructure(value) {
|
|
35
|
+
const root = exact(value, ['schema', 'version', 'generatedAt', 'target', 'synthetic', 'evidenceObservedAt', 'analystReview', 'evidenceReview', 'limitations', 'integrity'], 'Acquisition decision packet');
|
|
36
|
+
if (root.schema !== ACQUISITION_DECISION_PACKET_SCHEMA)
|
|
37
|
+
fail('Acquisition decision packet schema');
|
|
38
|
+
iso(root.generatedAt, 'Acquisition decision packet generatedAt');
|
|
39
|
+
domain(root.target, 'Acquisition decision packet target');
|
|
40
|
+
boolean(root.synthetic, 'Acquisition decision packet synthetic');
|
|
41
|
+
iso(root.evidenceObservedAt, 'Acquisition decision packet evidenceObservedAt', true);
|
|
42
|
+
const review = exact(root.analystReview, ['decision', 'rationale', 'reviewedChecks', 'outstandingChecks', 'state'], 'Acquisition analyst review');
|
|
43
|
+
const decision = enumeration(review.decision, ACQUISITION_DECISIONS, 'Acquisition analyst review decision');
|
|
44
|
+
structureText(review.rationale, 'Acquisition analyst review rationale', 2_000, true);
|
|
45
|
+
const reviewed = strings(review.reviewedChecks, 'Acquisition reviewed checks', ACQUISITION_MANUAL_CHECKS.length, 40);
|
|
46
|
+
const outstanding = strings(review.outstandingChecks, 'Acquisition outstanding checks', ACQUISITION_MANUAL_CHECKS.length, 40);
|
|
47
|
+
const expectedReviewed = ACQUISITION_MANUAL_CHECKS.filter((item) => reviewed.includes(item));
|
|
48
|
+
const expectedOutstanding = ACQUISITION_MANUAL_CHECKS.filter((item) => !reviewed.includes(item));
|
|
49
|
+
if (!sameValues(reviewed, expectedReviewed) || !sameValues(outstanding, expectedOutstanding))
|
|
50
|
+
fail('Acquisition analyst review checks');
|
|
51
|
+
const expectedState = decision === 'unresolved' || reviewed.length < ACQUISITION_MANUAL_CHECKS.length ? 'draft' : 'reviewed';
|
|
52
|
+
if (review.state !== expectedState)
|
|
53
|
+
fail('Acquisition analyst review state');
|
|
54
|
+
const evidence = exact(root.evidenceReview, ['version', 'label', 'state', 'items', 'transitionDependencies', 'policyChecks', 'nextSteps', 'limitations'], 'Acquisition evidence review');
|
|
55
|
+
if (evidence.version !== 2)
|
|
56
|
+
fail('Acquisition evidence review');
|
|
57
|
+
structureText(evidence.label, 'Acquisition evidence review label', 200);
|
|
58
|
+
enumeration(evidence.state, ['incomplete', 'registered', 'review_transition', 'sale_signal', 'unregistered_observation'], 'Acquisition evidence review state');
|
|
59
|
+
const expectedEvidenceIds = {
|
|
60
|
+
items: ['availability', 'lifecycle', 'transfer', 'operations', 'contacts'],
|
|
61
|
+
transitionDependencies: ['nameservers', 'web', 'mail', 'tls'],
|
|
62
|
+
policyChecks: ['policy_eligibility', 'policy_lifecycle', 'policy_transfer'],
|
|
63
|
+
};
|
|
64
|
+
for (const key of ['items', 'transitionDependencies', 'policyChecks']) {
|
|
65
|
+
const expected = expectedEvidenceIds[key];
|
|
66
|
+
const values = array(evidence[key], `Acquisition evidence review ${key}`, expected.length, expected.length);
|
|
67
|
+
const ids = values.map((item, index) => validateAcquisitionItem(item, `Acquisition evidence review ${key}[${index}]`));
|
|
68
|
+
if (!sameValues(ids, expected))
|
|
69
|
+
fail(`Acquisition evidence review ${key}`);
|
|
70
|
+
}
|
|
71
|
+
strings(evidence.nextSteps, 'Acquisition next steps', 6, 600);
|
|
72
|
+
strings(evidence.limitations, 'Acquisition evidence limitations', 12, 600);
|
|
73
|
+
strings(root.limitations, 'Acquisition limitations', 8, 600);
|
|
74
|
+
validateIntegrity(root.integrity, 'Acquisition integrity', root.version, ACQUISITION_DECISION_PACKET_VERSION);
|
|
75
|
+
}
|
|
76
|
+
function freezePacket(value) {
|
|
77
|
+
if (value && typeof value === 'object') {
|
|
78
|
+
for (const item of Object.values(value))
|
|
79
|
+
freezePacket(item);
|
|
80
|
+
Object.freeze(value);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** Validate the existing supported format and full canonical digest before preview. */
|
|
84
|
+
export async function readAcquisitionDecisionPacket(raw, expected) {
|
|
85
|
+
const value = parseBoundedJsonObject(raw, { label: 'Acquisition decision packet', maximumBytes: MAX_INVESTIGATION_PORTABLE_BYTES });
|
|
86
|
+
if (value.schema !== ACQUISITION_DECISION_PACKET_SCHEMA)
|
|
87
|
+
fail('Acquisition decision packet schema');
|
|
88
|
+
validateAcquisitionDecisionPacketStructure(value);
|
|
89
|
+
const { integrity, ...unsigned } = value;
|
|
90
|
+
if (await sha256ArtifactDigestV2(unsigned) !== integrity.digestSha256) {
|
|
91
|
+
throw new TypeError('The acquisition decision packet failed its integrity check.');
|
|
92
|
+
}
|
|
93
|
+
if (value.target !== target(expected.target) || value.synthetic !== expected.synthetic) {
|
|
94
|
+
throw new TypeError('This packet belongs to a different target or demonstration context. The current review is unchanged.');
|
|
95
|
+
}
|
|
96
|
+
freezePacket(value);
|
|
97
|
+
return value;
|
|
98
|
+
}
|
|
21
99
|
function text(value, maximum) {
|
|
22
100
|
return typeof value === 'string'
|
|
23
101
|
? value.replace(CONTROL_CHARACTERS, ' ').replace(/\s+/gu, ' ').trim().slice(0, maximum)
|
|
@@ -99,8 +99,8 @@ function relation(state, profile, profileSourceState, profileContextComparable,
|
|
|
99
99
|
? {
|
|
100
100
|
baselineRelation: state === 'authenticated_mail' ? 'aligned' : 'review',
|
|
101
101
|
baselineDetail: state === 'authenticated_mail'
|
|
102
|
-
? '
|
|
103
|
-
: '
|
|
102
|
+
? 'Published MX, SPF and DMARC records match the configured standard-mail baseline.'
|
|
103
|
+
: 'MX publication was observed with a mail-authentication policy gap relative to the configured baseline.',
|
|
104
104
|
}
|
|
105
105
|
: {
|
|
106
106
|
baselineRelation: 'review',
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { parseBoundedJson } from "../analysis/bounded-json.mjs";
|
|
2
2
|
import { exact, text } from "../evidence/artifact-structure.mjs";
|
|
3
3
|
import { readEditableCaseExport } from "../cases/case-export-input.mjs";
|
|
4
|
-
import { MAX_CONTEXT_INPUT_BYTES } from "../contracts/context-review.mjs";
|
|
4
|
+
import { MAX_CONTEXT_INPUT_BYTES, DOMAIN_HISTORY_LEGACY_INPUT_VERSION } from "../contracts/context-review.mjs";
|
|
5
5
|
import { DOMAIN_HISTORY_INPUT_SCHEMA, DOMAIN_HISTORY_INPUT_VERSION, reviewDomainHistory } from "./domain-history-review.mjs";
|
|
6
6
|
import { PLATFORM_CONTINUITY_INPUT_SCHEMA, PLATFORM_CONTINUITY_INPUT_VERSION, reviewPlatformContinuity } from "./platform-continuity-review.mjs";
|
|
7
7
|
import { STOREFRONT_INPUT_SCHEMA, STOREFRONT_INPUT_VERSION, reviewStorefront } from "./storefront-review.mjs";
|
|
@@ -11,14 +11,14 @@ export { CONTEXT_INPUT_SCHEMAS } from "../contracts/context-review.mjs";
|
|
|
11
11
|
/** Adapters share the same explicit dispatch; each domain owns its validation. */
|
|
12
12
|
export function reviewContextInput(raw, reviewedAt) {
|
|
13
13
|
const input = exact(raw, ['schema', 'version', 'evidence'], 'Context review input');
|
|
14
|
-
if (input.schema === DOMAIN_HISTORY_INPUT_SCHEMA && input.version === DOMAIN_HISTORY_INPUT_VERSION) {
|
|
14
|
+
if (input.schema === DOMAIN_HISTORY_INPUT_SCHEMA && (input.version === DOMAIN_HISTORY_INPUT_VERSION || input.version === DOMAIN_HISTORY_LEGACY_INPUT_VERSION)) {
|
|
15
15
|
const evidence = exact(input.evidence, ['caseExport', 'caseId', 'declarations'], 'Domain history input');
|
|
16
16
|
const caseId = text(evidence.caseId, 'Selected Case ID', 240);
|
|
17
17
|
const cases = readEditableCaseExport(JSON.stringify(evidence.caseExport));
|
|
18
18
|
const selected = cases.find(record => record.id === caseId);
|
|
19
19
|
if (!selected)
|
|
20
20
|
throw new TypeError('The selected Case is not present in this export.');
|
|
21
|
-
return reviewDomainHistory(selected, evidence.declarations, reviewedAt);
|
|
21
|
+
return reviewDomainHistory(selected, evidence.declarations, reviewedAt, input.version);
|
|
22
22
|
}
|
|
23
23
|
if (input.schema === PLATFORM_CONTINUITY_INPUT_SCHEMA && input.version === PLATFORM_CONTINUITY_INPUT_VERSION)
|
|
24
24
|
return reviewPlatformContinuity(input.evidence, reviewedAt);
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { createIntakeReport } from "./intake-report.mjs";
|
|
2
2
|
import { createLinkIntake } from "./link-intake.mjs";
|
|
3
|
+
import { createIndicatorIntake } from "./intake-indicators.mjs";
|
|
3
4
|
import { decodeQrPixels } from "./qr-intake.mjs";
|
|
4
5
|
import { sha256ArtifactBytes } from "../evidence/artifact-integrity.mjs";
|
|
5
6
|
import { MAX_DOCUMENT_PARTS, MAX_DOCUMENT_TEXT_BYTES, MAX_DOCUMENT_TOTAL_IMAGE_PIXELS } from "../contracts/document-intake.mjs";
|
|
6
7
|
/** The format adapters provide inert text/pixels; this owner retains provenance and links. */
|
|
7
8
|
export async function createDocumentIntake(bytes, kind, reviewedAt) {
|
|
8
|
-
const base = await createIntakeReport(bytes, kind, reviewedAt), links = createLinkIntake();
|
|
9
|
+
const base = await createIntakeReport(bytes, kind, reviewedAt), links = createLinkIntake(), indicators = createIndicatorIntake();
|
|
9
10
|
const parts = [], notes = new Set();
|
|
10
11
|
let textBytes = 0, imagePixels = 0;
|
|
11
12
|
let state = 'reviewed', pageCount = null, reviewedPages = 0;
|
|
@@ -38,8 +39,10 @@ export async function createDocumentIntake(bytes, kind, reviewedAt) {
|
|
|
38
39
|
return;
|
|
39
40
|
}
|
|
40
41
|
const location = await part({ kind: 'text', page, identity: original ? 'original_part_bytes' : 'extracted_text', state: 'reviewed' }, original ?? encoded);
|
|
41
|
-
if (location)
|
|
42
|
+
if (location) {
|
|
42
43
|
links.addText(value, 'document_text', location);
|
|
44
|
+
indicators.addText(value, 'document_text', location);
|
|
45
|
+
}
|
|
43
46
|
},
|
|
44
47
|
async destinations(values, page, original) {
|
|
45
48
|
const encoded = encoder.encode(JSON.stringify(values));
|
|
@@ -73,7 +76,10 @@ export async function createDocumentIntake(bytes, kind, reviewedAt) {
|
|
|
73
76
|
partial(limitation);
|
|
74
77
|
if (result.bounded)
|
|
75
78
|
partial('Further extracted links exceed the review bound.');
|
|
76
|
-
|
|
79
|
+
const indicatorResult = indicators.result();
|
|
80
|
+
if (indicatorResult.indicatorCoverage.state === 'partial')
|
|
81
|
+
partial('Further indicator extraction exceeds the review bound.');
|
|
82
|
+
return { targets: result.targets, report: { ...base, ...indicatorResult, links: result.links,
|
|
77
83
|
coverage: { ...base.coverage, state: state === 'reviewed' ? 'reviewed' : 'partial', reviewedParts: parts.length, rejectedLinks: result.rejected },
|
|
78
84
|
documentReview: { state, pageCount, reviewedPages, parts, notes: [...notes] } } };
|
|
79
85
|
},
|
|
@@ -1,12 +1,21 @@
|
|
|
1
|
-
import { caseEvidenceIncomparableReasons, compareCaseEvidence } from "../cases/case-evidence-model.mjs";
|
|
1
|
+
import { caseEvidenceFieldScope, caseEvidenceIncomparableReasons, compareCaseEvidence } from "../cases/case-evidence-model.mjs";
|
|
2
2
|
import { domainTransitionReview } from "./domain-transition-review.mjs";
|
|
3
3
|
import { MAX_CASE_EVIDENCE_PINS, MAX_EVIDENCE_SNAPSHOTS_PER_CASE } from "../contracts/case-portability.mjs";
|
|
4
4
|
import { array, domain, enumeration, exact, iso, text } from "../evidence/artifact-structure.mjs";
|
|
5
5
|
import { CONTEXT_REVIEW_SCHEMA, CONTEXT_REVIEW_VERSION, MAX_CONTEXT_RECORDS } from "../contracts/context-review.mjs";
|
|
6
6
|
export { DOMAIN_HISTORY_INPUT_SCHEMA, DOMAIN_HISTORY_INPUT_VERSION } from "../contracts/context-review.mjs";
|
|
7
7
|
export const DOMAIN_CHANGE_FAMILIES = ['registration', 'dns', 'mail', 'certificate', 'web'];
|
|
8
|
-
export
|
|
9
|
-
|
|
8
|
+
export const RETIRED_DEPENDENCY_REVIEW_STEPS = Object.freeze([
|
|
9
|
+
'DNS aliases: review owned CNAME records and delegated nameservers for references to the retired hostname. Record the configuration source and a deliberate cleanup plan; a remaining reference alone does not establish takeover availability.',
|
|
10
|
+
'Recovery email: review account-recovery and administrator-contact dependencies on the retired mail domain. Confirm a replacement through the account owner before removing access; record a source label, not email addresses or recovery secrets.',
|
|
11
|
+
'Integration references: review retained application, sign-in, webhook and third-party service configuration for the retired hostname. Record the responsible service and change reference without credentials or complete URLs; retained configuration does not prove a reference is still active.',
|
|
12
|
+
]);
|
|
13
|
+
export const REGISTRATION_BOUNDARY_KINDS = ['registration_reported', 'deletion_reported', 'reregistration_reported', 'transfer_reported', 'review_boundary'];
|
|
14
|
+
export const MAX_REGISTRATION_BOUNDARIES = 40;
|
|
15
|
+
export function readDomainHistoryDeclarations(raw, version = 1) {
|
|
16
|
+
if (version !== 1 && version !== 2)
|
|
17
|
+
throw new TypeError('Unsupported domain-history declaration version.');
|
|
18
|
+
const input = exact(raw, ['expectedChanges', 'retiredDependencies', ...(version === 2 ? ['registrationBoundaries'] : [])], 'Domain history declarations');
|
|
10
19
|
const expectedChanges = array(input.expectedChanges, 'Expected changes', MAX_CONTEXT_RECORDS).map(value => {
|
|
11
20
|
const item = exact(value, ['family', 'start', 'end', 'reason'], 'Expected change');
|
|
12
21
|
iso(item.start, 'Expected window start');
|
|
@@ -22,7 +31,22 @@ export function readDomainHistoryDeclarations(raw) {
|
|
|
22
31
|
iso(item.retiredAt, 'Retirement time');
|
|
23
32
|
return { asset: item.asset, dependency: item.dependency, family: enumeration(item.family, DOMAIN_CHANGE_FAMILIES, 'Dependency family'), retiredAt: item.retiredAt, source: text(item.source, 'Retirement source', 500) };
|
|
24
33
|
});
|
|
25
|
-
|
|
34
|
+
const registrationBoundaries = version === 1 ? [] : array(input.registrationBoundaries, 'Registration boundaries', MAX_REGISTRATION_BOUNDARIES).map(value => {
|
|
35
|
+
const item = exact(value, ['kind', 'occurredAt', 'source', 'rationale', 'snapshotIds', 'evidencePinIds'], 'Registration boundary');
|
|
36
|
+
iso(item.occurredAt, 'Declared boundary time');
|
|
37
|
+
const references = (raw, label) => {
|
|
38
|
+
const values = array(raw, label, 20).map(id => text(id, label, 240));
|
|
39
|
+
if (new Set(values).size !== values.length)
|
|
40
|
+
throw new TypeError('Registration boundary references must be unique.');
|
|
41
|
+
return values;
|
|
42
|
+
};
|
|
43
|
+
const snapshotIds = references(item.snapshotIds, 'Snapshot references'), evidencePinIds = references(item.evidencePinIds, 'Evidence pin references');
|
|
44
|
+
if (!snapshotIds.length && !evidencePinIds.length)
|
|
45
|
+
throw new TypeError('Link a retained snapshot or evidence pin to the declared boundary.');
|
|
46
|
+
return { kind: enumeration(item.kind, REGISTRATION_BOUNDARY_KINDS, 'Registration boundary kind'), occurredAt: item.occurredAt,
|
|
47
|
+
source: text(item.source, 'Boundary source', 160), rationale: text(item.rationale, 'Boundary rationale', 500), snapshotIds, evidencePinIds };
|
|
48
|
+
});
|
|
49
|
+
return { expectedChanges, retiredDependencies, registrationBoundaries };
|
|
26
50
|
}
|
|
27
51
|
function family(field) {
|
|
28
52
|
if (['registrar', 'createdDate', 'expiryDate', 'availability'].includes(field))
|
|
@@ -32,19 +56,21 @@ function family(field) {
|
|
|
32
56
|
if (['hasMx', 'hasSpf', 'hasDmarc'].includes(field))
|
|
33
57
|
return 'mail';
|
|
34
58
|
if (field.startsWith('http') || ['activityStatus', 'pageTitle', 'websiteProbeDetail', 'faviconMatch', 'faviconNearMatch',
|
|
35
|
-
'reusesOfficialAssets', 'hasPasswordField', 'hasExternalFormAction', 'pageBaselineMatch'].includes(field))
|
|
59
|
+
'reusesOfficialAssets', 'hasPasswordField', 'hasExternalFormAction', 'hasExternalPasswordForm', 'pageBaselineMatch'].includes(field))
|
|
36
60
|
return 'web';
|
|
37
61
|
return null;
|
|
38
62
|
}
|
|
39
63
|
/** Input is a canonical Case from storage or the existing exact Case-export reader. */
|
|
40
|
-
export function reviewDomainHistory(record, declarations, reviewedAt) {
|
|
64
|
+
export function reviewDomainHistory(record, declarations, reviewedAt, declarationVersion = 1) {
|
|
41
65
|
iso(reviewedAt, 'Review time');
|
|
42
|
-
const declared = readDomainHistoryDeclarations(declarations);
|
|
66
|
+
const declared = readDomainHistoryDeclarations(declarations, declarationVersion);
|
|
43
67
|
const related = (hostname) => hostname === record.domain || hostname.endsWith(`.${record.domain}`);
|
|
44
68
|
if (declared.retiredDependencies.some(row => !related(row.asset) && !related(row.dependency)))
|
|
45
69
|
throw new TypeError('A retired dependency must involve this Case domain or one of its hostnames.');
|
|
46
70
|
array(record.evidenceHistory, 'Case evidence history', MAX_EVIDENCE_SNAPSHOTS_PER_CASE);
|
|
47
71
|
array(record.evidencePins, 'Case evidence pins', MAX_CASE_EVIDENCE_PINS);
|
|
72
|
+
if (declared.registrationBoundaries.some(row => row.snapshotIds.some(id => record.evidenceHistory.filter(item => item.id === id).length !== 1) || row.evidencePinIds.some(id => record.evidencePins.filter(item => item.id === id).length !== 1)))
|
|
73
|
+
throw new TypeError('A declared registration boundary refers to evidence not uniquely retained in this Case.');
|
|
48
74
|
const snapshots = [...record.evidenceHistory].sort((a, b) => Date.parse(a.capturedAt) - Date.parse(b.capturedAt) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
49
75
|
const observations = [];
|
|
50
76
|
let orderedPairs = 0;
|
|
@@ -81,20 +107,32 @@ export function reviewDomainHistory(record, declarations, reviewedAt) {
|
|
|
81
107
|
const windows = declared.expectedChanges.filter(row => row.family === category && Date.parse(row.start) <= Date.parse(after.capturedAt) && Date.parse(row.end) >= Date.parse(before.capturedAt));
|
|
82
108
|
observations.push({ label: `${category} · ${change.label}`, state: 'changed',
|
|
83
109
|
detail: `${JSON.stringify(change.before)} → ${JSON.stringify(change.after)}${windows.length ? ` · Overlaps ${windows.length} analyst-declared expected-change window(s); not automatically approved.` : ''}`,
|
|
84
|
-
source: `${before.source} ${before.id} → ${after.source} ${after.id}`, observedAt: after.capturedAt,
|
|
110
|
+
source: `${before.source} ${before.id} → ${after.source} ${after.id}`, observedAt: after.capturedAt,
|
|
111
|
+
hostname: caseEvidenceFieldScope(change.field) === 'registration' ? record.domain : after.observationHostname ?? null });
|
|
85
112
|
}
|
|
86
113
|
}
|
|
87
114
|
for (const pin of record.evidencePins.filter(pin => pin.certificateObservation)) {
|
|
88
115
|
observations.push({ label: `Retained certificate · ${pin.label}`, state: pin.completeness === 'complete' && pin.truncated !== true ? 'observed' : 'partial', detail: pin.value,
|
|
89
|
-
source: `${pin.source} · pin ${pin.id}`, observedAt: pin.observedAt, hostname:
|
|
116
|
+
source: `${pin.source} · pin ${pin.id}`, observedAt: pin.observedAt, hostname: pin.observationHostname ?? null });
|
|
90
117
|
}
|
|
91
118
|
for (const row of declared.expectedChanges)
|
|
92
119
|
observations.push({ label: `Expected ${row.family} change`, state: 'reported', detail: `${row.start} – ${row.end} · ${row.reason}`, source: 'Analyst declaration', observedAt: null, hostname: record.domain });
|
|
93
120
|
for (const row of declared.retiredDependencies)
|
|
94
121
|
observations.push({ label: `Declared retired ${row.family} dependency`, state: 'reported', detail: `${row.asset} depended on ${row.dependency}. Check whether owned configuration still references it.`, source: row.source, observedAt: row.retiredAt, hostname: row.dependency });
|
|
122
|
+
for (const row of declared.registrationBoundaries) {
|
|
123
|
+
const earlier = snapshots.filter(snapshot => Date.parse(snapshot.capturedAt) < Date.parse(row.occurredAt)).length;
|
|
124
|
+
const later = snapshots.filter(snapshot => Date.parse(snapshot.capturedAt) >= Date.parse(row.occurredAt)).length;
|
|
125
|
+
observations.push({ label: `Declared registration boundary · ${row.kind.replaceAll('_', ' ')}`, state: 'reported',
|
|
126
|
+
detail: `${row.rationale} Retained snapshots before this declared boundary: ${earlier}; at or after: ${later}. This partitions observation times for review, not verified ownership or registration epochs.`,
|
|
127
|
+
source: `${row.source} · snapshots: ${row.snapshotIds.join(', ') || 'none'} · evidence pins: ${row.evidencePinIds.join(', ') || 'none'}`, observedAt: row.occurredAt, hostname: record.domain });
|
|
128
|
+
}
|
|
129
|
+
if (declared.registrationBoundaries.length)
|
|
130
|
+
prompts.add('Review earlier assertions, exceptions and response baselines against each declared registration boundary. Earlier evidence and open follow-ups remain unchanged; only a deliberate analyst update can revise a decision.');
|
|
95
131
|
const certificateCount = observations.filter(row => row.label.startsWith('Retained certificate')).length;
|
|
96
132
|
return { schema: CONTEXT_REVIEW_SCHEMA, version: CONTEXT_REVIEW_VERSION, kind: 'domain_history', reviewedAt, title: 'Domain history and retired dependencies', state: orderedPairs < 1 || limitedPairs > 0 ? 'partial' : 'reviewed',
|
|
97
|
-
summary: `Retained snapshots: ${snapshots.length}; time-ordered pairs: ${orderedPairs}; limited pairs: ${limitedPairs}; certificate pins: ${certificateCount}; declared retired dependencies: ${declared.retiredDependencies.length}
|
|
98
|
-
nextSteps: [...prompts, 'Review changed registration, DNS, mail and web evidence against declared maintenance windows.', 'Recheck owned references to retired dependencies before changing configuration; preserve evidence of legitimate transfers or rebrands.'
|
|
99
|
-
|
|
133
|
+
summary: `Retained snapshots: ${snapshots.length}; time-ordered pairs: ${orderedPairs}; limited pairs: ${limitedPairs}; certificate pins: ${certificateCount}; declared retired dependencies: ${declared.retiredDependencies.length}.${declarationVersion === 2 ? ` Analyst-declared registration boundaries: ${declared.registrationBoundaries.length}.` : ''}`, observations,
|
|
134
|
+
nextSteps: [...prompts, 'Review changed registration, DNS, mail and web evidence against declared maintenance windows.', 'Recheck owned references to retired dependencies before changing configuration; preserve evidence of legitimate transfers or rebrands.',
|
|
135
|
+
...(declared.retiredDependencies.length ? RETIRED_DEPENDENCY_REVIEW_STEPS : [])],
|
|
136
|
+
limitations: ['This review uses retained Case evidence, not fresh collection. Missing or incompatible fields are not compared.', 'Certificate pins retain their own source and completeness; they are not equivalent to live TLS observations.', 'Changes and retired dependencies do not establish ownership transfer, malicious repurposing or takeover availability.',
|
|
137
|
+
...(declarationVersion === 2 ? ['Registration boundaries are analyst declarations linked to retained records, not independently verified deletion, re-registration or ownership events. Expiry and registrar changes do not establish those events.'] : [])] };
|
|
100
138
|
}
|
|
@@ -27,6 +27,8 @@ export function domainTransitionReview(field, before, after) {
|
|
|
27
27
|
: 'Review the current page evidence before changing the assessment; removal of one form does not establish that the incident is resolved.';
|
|
28
28
|
case 'hasExternalFormAction':
|
|
29
29
|
return 'Check the observed form destination and legitimate service use before updating the assessment. No form was submitted by this review.';
|
|
30
|
+
case 'hasExternalPasswordForm':
|
|
31
|
+
return 'Review the password form’s declared destination against the recorded concern and expected identity provider. A changed declaration is not an observed submission.';
|
|
30
32
|
case 'faviconMatch':
|
|
31
33
|
case 'faviconNearMatch':
|
|
32
34
|
case 'reusesOfficialAssets':
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
// Source-qualified, privacy-minimised retained infrastructure evidence. No collector
|
|
2
|
+
// or store lives here; supplied observations never initiate network work.
|
|
3
|
+
import { exact, text, enumeration, array, strings, iso, boolean } from "../evidence/artifact-structure.mjs";
|
|
4
|
+
import { normalizeDomain } from "../evidence/domain-name.mjs";
|
|
5
|
+
import { canonicalIpAddress } from "../contracts/ip-address.mjs";
|
|
6
|
+
import { parseBoundedJson } from "../analysis/bounded-json.mjs";
|
|
7
|
+
import { technologyEvidenceRoles } from "../evidence/technology-evidence-role.mjs";
|
|
8
|
+
import { INFRASTRUCTURE_OBSERVATION_SCHEMA, INFRASTRUCTURE_OBSERVATION_VERSION, MAX_INFRASTRUCTURE_OBSERVATION_BYTES, MAX_INFRASTRUCTURE_HOSTS, MAX_INFRASTRUCTURE_DNS_ROWS, MAX_INFRASTRUCTURE_CERTIFICATES, MAX_INFRASTRUCTURE_ROLES, MAX_INFRASTRUCTURE_SOURCES, MAX_INFRASTRUCTURE_LIMITATIONS, INFRASTRUCTURE_DNS_TYPES, } from "../contracts/external-observation-interchange.mjs";
|
|
9
|
+
export { INFRASTRUCTURE_OBSERVATION_SCHEMA, INFRASTRUCTURE_OBSERVATION_VERSION, MAX_INFRASTRUCTURE_OBSERVATION_BYTES };
|
|
10
|
+
export { MAX_INFRASTRUCTURE_HOSTS, MAX_INFRASTRUCTURE_DNS_ROWS, MAX_INFRASTRUCTURE_CERTIFICATES, MAX_INFRASTRUCTURE_ROLES, INFRASTRUCTURE_DNS_TYPES };
|
|
11
|
+
export const INFRASTRUCTURE_SOURCE_FAMILIES = ['dns', 'certificate_log', 'tls', 'technology', 'ip_registration', 'routing', 'origin_observation', 'analyst'];
|
|
12
|
+
export const INFRASTRUCTURE_PROVIDER_ROLES = ['dns_operator', 'observed_edge', 'application_platform', 'framework_runtime', 'embedded_dependency', 'address_registration', 'routing_origin', 'observed_origin'];
|
|
13
|
+
export function infrastructureCohortLabel(cohort) {
|
|
14
|
+
switch (cohort.kind) {
|
|
15
|
+
case 'dns': return `Query ${cohort.queriedName} · owner ${cohort.ownerName} · ${cohort.type} ${cohort.aspect}`;
|
|
16
|
+
case 'certificate_names': return `Certificate SHA-256 ${cohort.fingerprintSha256}`;
|
|
17
|
+
case 'provider_role': return `${cohort.subjectType} ${cohort.subject} · ${cohort.role}`;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
function hostname(raw) {
|
|
21
|
+
const value = text(raw, 'Infrastructure hostname', 253);
|
|
22
|
+
if (normalizeDomain(value) !== value)
|
|
23
|
+
throw new TypeError('Infrastructure hostnames must be exact canonical hostnames.');
|
|
24
|
+
return value;
|
|
25
|
+
}
|
|
26
|
+
function name(raw) {
|
|
27
|
+
const value = text(raw, 'Certificate name', 255);
|
|
28
|
+
hostname(value.startsWith('*.') ? value.slice(2) : value);
|
|
29
|
+
return value;
|
|
30
|
+
}
|
|
31
|
+
function identifier(raw, label) {
|
|
32
|
+
const value = text(raw, label, 120);
|
|
33
|
+
if (!/^[a-zA-Z0-9_-]+$/u.test(value))
|
|
34
|
+
throw new TypeError(`${label} must use a bounded portable identity.`);
|
|
35
|
+
return value;
|
|
36
|
+
}
|
|
37
|
+
function unique(values, label, key = value => String(value)) {
|
|
38
|
+
if (new Set(values.map(key)).size !== values.length)
|
|
39
|
+
throw new TypeError(`${label} contains duplicate or ambiguous identities.`);
|
|
40
|
+
return values;
|
|
41
|
+
}
|
|
42
|
+
function at(raw) { iso(raw, 'Infrastructure observation time'); return new Date(raw).toISOString(); }
|
|
43
|
+
function flag(raw) { boolean(raw, 'Infrastructure completeness'); return raw; }
|
|
44
|
+
/** Exact current reader. Old Case pins lacking this field remain unknown, never reconstructed. */
|
|
45
|
+
export function readInfrastructureObservation(raw) {
|
|
46
|
+
// Bound hostile object traversal before serialisation or parsing work.
|
|
47
|
+
const root = exact(raw, ['schema', 'version', 'id', 'target', 'observedAt', 'mode', 'scope', 'coverage', 'sources', 'dns', 'certificates', 'roles', 'limitations'], 'Infrastructure snapshot');
|
|
48
|
+
if (root.schema !== INFRASTRUCTURE_OBSERVATION_SCHEMA || root.version !== INFRASTRUCTURE_OBSERVATION_VERSION)
|
|
49
|
+
throw new TypeError('Unsupported infrastructure snapshot schema or version.');
|
|
50
|
+
const scope = exact(root.scope, ['hostnames', 'dnsTypes', 'selection'], 'Infrastructure scope');
|
|
51
|
+
const selection = enumeration(scope.selection, ['explicit_hosts', 'certificate_names'], 'Selection scope');
|
|
52
|
+
const hostnames = unique(array(scope.hostnames, 'Selected hostnames', MAX_INFRASTRUCTURE_HOSTS, selection === 'explicit_hosts' ? 1 : 0).map(hostname), 'Selected hostnames');
|
|
53
|
+
const dnsTypes = unique(array(scope.dnsTypes, 'Selected DNS types', INFRASTRUCTURE_DNS_TYPES.length).map(value => enumeration(value, INFRASTRUCTURE_DNS_TYPES, 'DNS type')), 'Selected DNS types');
|
|
54
|
+
const coverage = exact(root.coverage, ['state', 'truncated', 'detail'], 'Infrastructure coverage');
|
|
55
|
+
const sources = unique(array(root.sources, 'Infrastructure sources', MAX_INFRASTRUCTURE_SOURCES, 1).map(rawSource => {
|
|
56
|
+
const source = exact(rawSource, ['id', 'name', 'family', 'evidenceClass', 'reference'], 'Infrastructure source');
|
|
57
|
+
const reference = source.reference === null ? null : text(source.reference, 'Source reference', 300);
|
|
58
|
+
if (reference !== null && /[?#]|\b(?:cookie|password|bearer)\s*[:=]/iu.test(reference))
|
|
59
|
+
throw new TypeError('Source references must not retain query strings, fragments or credentials.');
|
|
60
|
+
if (reference?.includes('://')) {
|
|
61
|
+
let url;
|
|
62
|
+
try {
|
|
63
|
+
url = new URL(reference);
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
throw new TypeError('Source reference URL is invalid.');
|
|
67
|
+
}
|
|
68
|
+
if (!['https:', 'http:'].includes(url.protocol) || url.username || url.password)
|
|
69
|
+
throw new TypeError('Source references cannot retain credentials or unsafe URL schemes.');
|
|
70
|
+
}
|
|
71
|
+
return { id: identifier(source.id, 'Source ID'), name: text(source.name, 'Source name', 80), family: enumeration(source.family, INFRASTRUCTURE_SOURCE_FAMILIES, 'Source family'), evidenceClass: enumeration(source.evidenceClass, ['local_observation', 'provider_report'], 'Evidence class'), reference };
|
|
72
|
+
}), 'Source IDs', value => value.id);
|
|
73
|
+
const sourceMap = new Map(sources.map(value => [value.id, value]));
|
|
74
|
+
function sourceId(rawId, families) {
|
|
75
|
+
const id = identifier(rawId, 'Observation source ID'), source = sourceMap.get(id);
|
|
76
|
+
if (!source || !families.includes(source.family))
|
|
77
|
+
throw new TypeError('An observation requires its own matching source family.');
|
|
78
|
+
return id;
|
|
79
|
+
}
|
|
80
|
+
const dns = array(root.dns, 'DNS observations', MAX_INFRASTRUCTURE_DNS_ROWS).map(rawRow => {
|
|
81
|
+
const row = exact(rawRow, ['sourceId', 'queriedName', 'ownerName', 'type', 'observedAt', 'outcome', 'values', 'complete', 'truncated'], 'DNS observation');
|
|
82
|
+
const type = enumeration(row.type, INFRASTRUCTURE_DNS_TYPES, 'DNS type');
|
|
83
|
+
const queriedName = hostname(row.queriedName), ownerName = hostname(row.ownerName);
|
|
84
|
+
if (!hostnames.includes(queriedName) || !dnsTypes.includes(type))
|
|
85
|
+
throw new TypeError('DNS observation falls outside its explicit selected scope.');
|
|
86
|
+
const outcome = enumeration(row.outcome, ['answered', 'no_data', 'nxdomain', 'failed', 'not_checked'], 'DNS outcome');
|
|
87
|
+
const values = unique(array(row.values, 'DNS record values', 32).map(rawValue => {
|
|
88
|
+
const value = text(rawValue, 'DNS value', 300);
|
|
89
|
+
if (type === 'A' || type === 'AAAA') {
|
|
90
|
+
const address = canonicalIpAddress(value);
|
|
91
|
+
if (address !== value || (type === 'A') !== !value.includes(':'))
|
|
92
|
+
throw new TypeError('DNS address value does not match its type.');
|
|
93
|
+
}
|
|
94
|
+
else if (type === 'MX') {
|
|
95
|
+
const match = /^(\d{1,5}) ([a-z0-9.-]+)$/u.exec(value);
|
|
96
|
+
if (!match || Number(match[1]) > 65535)
|
|
97
|
+
throw new TypeError('MX records require an explicit preference and target.');
|
|
98
|
+
hostname(match[2]);
|
|
99
|
+
}
|
|
100
|
+
else
|
|
101
|
+
hostname(value);
|
|
102
|
+
return value;
|
|
103
|
+
}), 'DNS values');
|
|
104
|
+
if ((outcome === 'answered') !== (values.length > 0))
|
|
105
|
+
throw new TypeError('Answered DNS observations require values; other outcomes cannot retain answers.');
|
|
106
|
+
if (type === 'CNAME' && values.length > 1)
|
|
107
|
+
throw new TypeError('One CNAME owner cannot have competing alias targets.');
|
|
108
|
+
return { sourceId: sourceId(row.sourceId, ['dns']), queriedName, ownerName, type, observedAt: at(row.observedAt), outcome, values, complete: flag(row.complete), truncated: flag(row.truncated) };
|
|
109
|
+
});
|
|
110
|
+
unique(dns, 'DNS observation identities', row => JSON.stringify([row.sourceId, row.queriedName, row.ownerName, row.type, row.observedAt]));
|
|
111
|
+
const certificates = array(root.certificates, 'Certificate observations', MAX_INFRASTRUCTURE_CERTIFICATES).map(rawRow => {
|
|
112
|
+
const row = exact(rawRow, ['sourceId', 'fingerprintSha256', 'observedAt', 'names', 'namesComplete'], 'Certificate observation');
|
|
113
|
+
const fingerprintSha256 = text(row.fingerprintSha256, 'Certificate digest', 64);
|
|
114
|
+
if (!/^[a-f0-9]{64}$/u.test(fingerprintSha256))
|
|
115
|
+
throw new TypeError('Certificate identity requires an exact SHA-256 digest.');
|
|
116
|
+
return { sourceId: sourceId(row.sourceId, ['tls', 'certificate_log']), fingerprintSha256, observedAt: at(row.observedAt), names: unique(array(row.names, 'Certificate names', MAX_INFRASTRUCTURE_HOSTS, 1).map(name), 'Certificate names'), namesComplete: flag(row.namesComplete) };
|
|
117
|
+
});
|
|
118
|
+
unique(certificates, 'Certificate observation identities', row => JSON.stringify([row.sourceId, row.fingerprintSha256, row.observedAt]));
|
|
119
|
+
const roleFamilies = {
|
|
120
|
+
dns_operator: ['dns'], observed_edge: ['technology'], application_platform: ['technology'], framework_runtime: ['technology'], embedded_dependency: ['technology'], address_registration: ['ip_registration'], routing_origin: ['routing'], observed_origin: ['origin_observation'],
|
|
121
|
+
};
|
|
122
|
+
const roles = array(root.roles, 'Provider role observations', MAX_INFRASTRUCTURE_ROLES).map(rawRow => {
|
|
123
|
+
const row = exact(rawRow, ['sourceId', 'subject', 'subjectType', 'role', 'providerId', 'providerLabel', 'value', 'observedAt', 'complete'], 'Provider role observation');
|
|
124
|
+
const subjectType = enumeration(row.subjectType, ['hostname', 'address'], 'Provider subject type');
|
|
125
|
+
const subject = subjectType === 'hostname' ? hostname(row.subject) : text(row.subject, 'Provider address', 45);
|
|
126
|
+
if (subjectType === 'address' && canonicalIpAddress(subject) !== subject)
|
|
127
|
+
throw new TypeError('Provider role address is invalid.');
|
|
128
|
+
if (subjectType === 'hostname' && !hostnames.includes(subject))
|
|
129
|
+
throw new TypeError('Provider-role hostname falls outside the selected scope.');
|
|
130
|
+
const role = enumeration(row.role, INFRASTRUCTURE_PROVIDER_ROLES, 'Provider role');
|
|
131
|
+
const value = text(row.value, 'Provider role value', 300);
|
|
132
|
+
if (role === 'routing_origin') {
|
|
133
|
+
if (subjectType !== 'address' || !/^AS(?:[1-9]\d{0,9})$/u.test(value) || Number(value.slice(2)) > 4_294_967_295)
|
|
134
|
+
throw new TypeError('Routing evidence requires an address and independently sourced ASN.');
|
|
135
|
+
}
|
|
136
|
+
if (role === 'observed_origin' && normalizeDomain(value) !== value && canonicalIpAddress(value) !== value)
|
|
137
|
+
throw new TypeError('Independently observed origin must retain an exact hostname or address.');
|
|
138
|
+
return { sourceId: sourceId(row.sourceId, roleFamilies[role]), subject, subjectType, role, providerId: identifier(row.providerId, 'Provider ID'), providerLabel: text(row.providerLabel, 'Provider label', 100), value, observedAt: at(row.observedAt), complete: flag(row.complete) };
|
|
139
|
+
});
|
|
140
|
+
unique(roles, 'Provider-role observation identities', row => JSON.stringify([row.sourceId, row.subjectType, row.subject, row.role, row.providerId, row.observedAt]));
|
|
141
|
+
if (roles.some(row => row.subjectType === 'address' && !dns.some(answer => answer.values.includes(row.subject))))
|
|
142
|
+
throw new TypeError('Provider-role addresses must have explicit supporting DNS address observations within the snapshot.');
|
|
143
|
+
const result = { schema: INFRASTRUCTURE_OBSERVATION_SCHEMA, version: INFRASTRUCTURE_OBSERVATION_VERSION, id: identifier(root.id, 'Snapshot ID'), target: hostname(root.target), observedAt: at(root.observedAt), mode: enumeration(root.mode, ['supplied', 'selected_lookup', 'certificate_log'], 'Collection mode'), scope: { hostnames, dnsTypes, selection: enumeration(scope.selection, ['explicit_hosts', 'certificate_names'], 'Selection scope') }, coverage: { state: enumeration(coverage.state, ['complete', 'partial', 'failed'], 'Coverage state'), truncated: flag(coverage.truncated), detail: text(coverage.detail, 'Coverage detail', 500) }, sources, dns, certificates, roles, limitations: unique(strings(root.limitations, 'Infrastructure limitations', MAX_INFRASTRUCTURE_LIMITATIONS, 300), 'Infrastructure limitations') };
|
|
144
|
+
if ([...dns, ...certificates, ...roles].some(row => row.observedAt > result.observedAt))
|
|
145
|
+
throw new TypeError('Snapshot time cannot precede one of its retained observations.');
|
|
146
|
+
if (result.coverage.state === 'complete' && (result.coverage.truncated || dns.some(row => !row.complete || row.truncated || ['failed', 'not_checked'].includes(row.outcome)) || certificates.some(row => !row.namesComplete) || roles.some(row => !row.complete)))
|
|
147
|
+
throw new TypeError('Complete snapshot coverage cannot contain incomplete observations.');
|
|
148
|
+
if (result.coverage.state === 'complete' && hostnames.some(host => dnsTypes.some(type => !dns.some(row => row.queriedName === host && row.type === type))))
|
|
149
|
+
throw new TypeError('Complete DNS coverage requires every explicitly selected host/type pair.');
|
|
150
|
+
if (new TextEncoder().encode(`${JSON.stringify(result)}\n`).byteLength > MAX_INFRASTRUCTURE_OBSERVATION_BYTES)
|
|
151
|
+
throw new TypeError('Infrastructure snapshot exceeds its retained byte bound.');
|
|
152
|
+
return result;
|
|
153
|
+
}
|
|
154
|
+
export function normalizeInfrastructureObservation(raw) {
|
|
155
|
+
if (raw === undefined || raw === null)
|
|
156
|
+
return null;
|
|
157
|
+
return readInfrastructureObservation(raw);
|
|
158
|
+
}
|
|
159
|
+
export function parseInfrastructureObservation(raw) {
|
|
160
|
+
return readInfrastructureObservation(parseBoundedJson(raw, { label: 'Infrastructure snapshot', maximumBytes: MAX_INFRASTRUCTURE_OBSERVATION_BYTES }));
|
|
161
|
+
}
|
|
162
|
+
export function serialiseInfrastructureObservation(raw) { return `${JSON.stringify(readInfrastructureObservation(raw))}\n`; }
|
|
163
|
+
function dnsFactKey(row) { return JSON.stringify([row.sourceId, row.queriedName, row.ownerName, row.type]); }
|
|
164
|
+
function roleFactKey(row) { return JSON.stringify([row.sourceId, row.subjectType, row.subject, row.role]); }
|
|
165
|
+
/** A negative DNS answer has a clock too; a missing row has none. */
|
|
166
|
+
function observationCohorts(snapshot) {
|
|
167
|
+
const cohorts = new Map();
|
|
168
|
+
const add = (key, observedAt, complete) => {
|
|
169
|
+
const cohort = cohorts.get(key) ?? { times: new Set(), complete: true };
|
|
170
|
+
cohort.times.add(observedAt);
|
|
171
|
+
cohort.complete &&= complete;
|
|
172
|
+
cohorts.set(key, cohort);
|
|
173
|
+
};
|
|
174
|
+
for (const row of snapshot.dns) {
|
|
175
|
+
const complete = row.complete && !row.truncated && !['failed', 'not_checked'].includes(row.outcome);
|
|
176
|
+
add(dnsFactKey(row), row.observedAt, complete);
|
|
177
|
+
add(JSON.stringify([row.sourceId, row.queriedName, row.ownerName, row.type, 'outcome']), row.observedAt, complete);
|
|
178
|
+
}
|
|
179
|
+
for (const row of snapshot.certificates)
|
|
180
|
+
add(JSON.stringify([row.sourceId, row.fingerprintSha256, 'certificate_names']), row.observedAt, row.namesComplete);
|
|
181
|
+
for (const row of snapshot.roles)
|
|
182
|
+
add(roleFactKey(row), row.observedAt, row.complete);
|
|
183
|
+
return cohorts;
|
|
184
|
+
}
|
|
185
|
+
export function infrastructureObservationFacts(raw) {
|
|
186
|
+
const snapshot = readInfrastructureObservation(raw);
|
|
187
|
+
const sources = new Map(snapshot.sources.map(source => [source.id, source]));
|
|
188
|
+
const facts = [];
|
|
189
|
+
const dnsCohort = (row, aspect) => ({ kind: 'dns', sourceId: row.sourceId, queriedName: row.queriedName, ownerName: row.ownerName, type: row.type, aspect });
|
|
190
|
+
for (const row of snapshot.dns)
|
|
191
|
+
for (const value of row.values)
|
|
192
|
+
facts.push({ key: dnsFactKey(row), cohort: dnsCohort(row, 'values'), hostname: row.ownerName, family: row.type, value, observedAt: row.observedAt, source: sources.get(row.sourceId), complete: row.complete && !row.truncated });
|
|
193
|
+
for (const row of snapshot.dns)
|
|
194
|
+
facts.push({ key: JSON.stringify([row.sourceId, row.queriedName, row.ownerName, row.type, 'outcome']), cohort: dnsCohort(row, 'outcome'), hostname: row.ownerName, family: `${row.type}_outcome`, value: row.outcome, observedAt: row.observedAt, source: sources.get(row.sourceId), complete: row.complete && !row.truncated && !['failed', 'not_checked'].includes(row.outcome) });
|
|
195
|
+
for (const row of snapshot.certificates)
|
|
196
|
+
for (const value of row.names)
|
|
197
|
+
facts.push({ key: JSON.stringify([row.sourceId, row.fingerprintSha256, 'certificate_names']), cohort: { kind: 'certificate_names', sourceId: row.sourceId, fingerprintSha256: row.fingerprintSha256 }, hostname: value, family: 'certificate_names', value, observedAt: row.observedAt, source: sources.get(row.sourceId), complete: row.namesComplete });
|
|
198
|
+
for (const row of snapshot.roles)
|
|
199
|
+
facts.push({ key: roleFactKey(row), cohort: { kind: 'provider_role', sourceId: row.sourceId, subjectType: row.subjectType, subject: row.subject, role: row.role }, hostname: row.subject, family: row.role, value: `${row.providerId}: ${row.value}`, observedAt: row.observedAt, source: sources.get(row.sourceId), complete: row.complete });
|
|
200
|
+
return { snapshot, facts };
|
|
201
|
+
}
|
|
202
|
+
export function compareInfrastructureObservations(earlierRaw, laterRaw) {
|
|
203
|
+
const earlier = infrastructureObservationFacts(earlierRaw), later = infrastructureObservationFacts(laterRaw);
|
|
204
|
+
const scope = (row) => JSON.stringify({ target: row.target, mode: row.mode, selection: row.scope.selection, hostnames: [...row.scope.hostnames].sort(), dnsTypes: [...row.scope.dnsTypes].sort(), sources: [...row.sources].sort((a, b) => a.id.localeCompare(b.id)) });
|
|
205
|
+
const comparable = scope(earlier.snapshot) === scope(later.snapshot) && earlier.snapshot.observedAt < later.snapshot.observedAt;
|
|
206
|
+
const complete = comparable && [earlier.snapshot, later.snapshot].every(row => row.coverage.state === 'complete' && !row.coverage.truncated);
|
|
207
|
+
const beforeCohorts = observationCohorts(earlier.snapshot), afterCohorts = observationCohorts(later.snapshot);
|
|
208
|
+
const groups = new Map();
|
|
209
|
+
for (const [side, values] of [['before', earlier.facts], ['after', later.facts]])
|
|
210
|
+
for (const fact of values) {
|
|
211
|
+
const group = groups.get(fact.key) ?? { cohort: fact.cohort, hostname: fact.hostname, family: fact.family, source: fact.source, before: [], after: [], complete: true };
|
|
212
|
+
if (fact.hostname < group.hostname)
|
|
213
|
+
group.hostname = fact.hostname;
|
|
214
|
+
group[side].push(fact.value);
|
|
215
|
+
group.complete &&= fact.complete;
|
|
216
|
+
groups.set(fact.key, group);
|
|
217
|
+
}
|
|
218
|
+
const rows = [...groups.entries()].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([key, row]) => {
|
|
219
|
+
row.before = [...new Set(row.before)].sort();
|
|
220
|
+
row.after = [...new Set(row.after)].sort();
|
|
221
|
+
const before = beforeCohorts.get(key), after = afterCohorts.get(key);
|
|
222
|
+
const beforeTimes = [...before?.times ?? []].sort(), afterTimes = [...after?.times ?? []].sort();
|
|
223
|
+
// Multiple historical cohorts are retained, not flattened into a fictitious
|
|
224
|
+
// single response. A later envelope never advances a source observation.
|
|
225
|
+
const ordered = beforeTimes.length === 1 && afterTimes.length === 1 && beforeTimes[0] < afterTimes[0];
|
|
226
|
+
row.complete &&= Boolean(before?.complete && after?.complete);
|
|
227
|
+
const same = JSON.stringify(row.before) === JSON.stringify(row.after);
|
|
228
|
+
const state = !comparable ? 'incomparable' : !ordered || !complete || !row.complete ? 'unknown' : same ? 'unchanged' : !row.before.length ? 'newly_observed' : !row.after.length ? 'not_returned' : 'changed';
|
|
229
|
+
return { ...row, beforeTimes, afterTimes, state, detail: state === 'not_returned' ? 'Not returned by this comparable bounded collection; this does not establish disappearance.' : state === 'newly_observed' ? 'Newly present in retained evidence; not necessarily newly created.' : state === 'unknown' ? !ordered ? 'Source observation times are missing, equal, reversed or span multiple cohorts; envelope time cannot establish a temporal change.' : 'Incomplete evidence cannot establish a removal or change.' : state === 'incomparable' ? 'Source, scope, mode or observation ordering differs; no temporal change is inferred.' : 'Source-qualified retained observation comparison.' };
|
|
230
|
+
});
|
|
231
|
+
return { state: comparable ? complete && !rows.some(row => row.state === 'unknown') ? 'compared' : 'partial' : 'incomparable', earlier: earlier.snapshot.id, later: later.snapshot.id, rows, limitations: ['No new collection was made. Retained changes do not establish ownership, safety, control or maliciousness.', 'Provider-reported history remains labelled separately from local observations. Empty, failed, limited and changed-source snapshots cannot replace a stronger baseline.'] };
|
|
232
|
+
}
|
|
233
|
+
/** Technology roles reuse the canonical role owner, not a new signature catalogue. */
|
|
234
|
+
export function infrastructureTechnologyRoles(value) { return technologyEvidenceRoles(value); }
|