@slicedearth/whoisleuth-cli 1.43.0 → 1.47.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +7 -3
- package/NOTICE +5 -4
- package/README.md +67 -10
- package/bin/whoisleuth.mjs +22 -9
- package/cli/arguments.mjs +256 -9
- package/cli/artifact-structure.mjs +1833 -0
- package/cli/artifact-verify.mjs +242 -49
- package/cli/bounded-json.mjs +1 -0
- package/cli/bulk-checkpoint.mjs +67 -31
- package/cli/bulk-command-runner.mjs +16 -6
- package/cli/bulk.mjs +18 -7
- package/cli/case-pack.mjs +819 -34
- package/cli/command-reference.mjs +48 -24
- package/cli/completion.mjs +50 -11
- package/cli/config-profile.mjs +15 -12
- package/cli/ct-event-intake.mjs +32 -2
- package/cli/discovery-observation-snapshot.mjs +8 -14
- package/cli/discovery-scan-command-runner.mjs +18 -6
- package/cli/discovery-snapshot.mjs +8 -15
- package/cli/discovery-workflow.mjs +2 -3
- package/cli/doctor.mjs +2 -2
- package/cli/domain-control-monitor.mjs +21 -15
- package/cli/domain-control-observations.mjs +25 -2
- package/cli/errors.mjs +7 -1
- package/cli/evidence-command-runner.mjs +17 -11
- package/cli/evidence-signing.mjs +73 -45
- package/cli/export-evidence.mjs +11 -4
- package/cli/formatters/html.mjs +3 -3
- package/cli/formatters/json.mjs +12 -4
- package/cli/formatters/terminal.mjs +4 -1
- package/cli/http.mjs +2 -1
- package/cli/input.mjs +57 -0
- package/cli/interchange-report.mjs +226 -0
- package/cli/investigation-manifest.mjs +16 -17
- package/cli/investigation-run.mjs +71 -23
- package/cli/local-mmdb-review.mjs +45 -14
- package/cli/lookup-command-runner.mjs +2 -2
- package/cli/lookup-diff.mjs +9 -0
- package/cli/lookup-reconcile.mjs +1 -1
- package/cli/mail-review.mjs +5 -5
- package/cli/manual.mjs +1 -1
- package/cli/network-command-runner.mjs +78 -0
- package/cli/offline-evidence-review.mjs +28 -8
- package/cli/page-compare.mjs +8 -5
- package/cli/registry-cohort.mjs +385 -61
- package/cli/registry-fixture-scaffold.mjs +4 -3
- package/cli/registry-support.mjs +2 -1
- package/cli/retained-artifact-diff.mjs +376 -0
- package/cli/risk-calibration.mjs +7 -5
- package/cli/runner.mjs +150 -177
- package/cli/saved-lookup.mjs +26 -3
- package/cli/sharing-review.mjs +12 -6
- package/docs/cli-reference.md +467 -74
- package/docs/cli.md +25 -5
- package/frontend/src/lib/analysis/acquisition-decision-packet.js +4 -4
- package/frontend/src/lib/analysis/acquisition-due-diligence.js +1 -1
- package/frontend/src/lib/analysis/artifact-integrity.js +42 -0
- package/frontend/src/lib/analysis/brand-profile-model.js +33 -7
- package/frontend/src/lib/analysis/bulk-domain-comparison.js +25 -5
- package/frontend/src/lib/analysis/bulk-mail-exposure.js +66 -15
- package/frontend/src/lib/analysis/bulk-retry-plan.js +1 -1
- package/frontend/src/lib/analysis/bulk-review-export.js +7 -6
- package/frontend/src/lib/analysis/bulk-review-model.js +3 -2
- package/frontend/src/lib/analysis/bulk-session-model.js +192 -26
- package/frontend/src/lib/analysis/bulk-sort.js +4 -0
- package/frontend/src/lib/analysis/case-brand-profile-references.js +85 -0
- package/frontend/src/lib/analysis/case-evidence-model.js +9 -1
- package/frontend/src/lib/analysis/case-investigation-branch-model.js +134 -0
- package/frontend/src/lib/analysis/case-migration-model.js +83 -31
- package/frontend/src/lib/analysis/case-record-contracts.js +4 -2
- package/frontend/src/lib/analysis/case-record-core.js +1 -1
- package/frontend/src/lib/analysis/case-record-model.js +2 -0
- package/frontend/src/lib/analysis/case-record-operations.js +43 -13
- package/frontend/src/lib/analysis/case-report.js +39 -6
- package/frontend/src/lib/analysis/case-response-model.js +66 -7
- package/frontend/src/lib/analysis/case-response-packet.js +15 -5
- package/frontend/src/lib/analysis/comparison-ledger-bulk.js +366 -0
- package/frontend/src/lib/analysis/comparison-ledger-contract.js +241 -0
- package/frontend/src/lib/analysis/comparison-ledger-json.js +79 -0
- package/frontend/src/lib/analysis/comparison-ledger-serialization.js +51 -0
- package/frontend/src/lib/analysis/domain-control-manifest-core.js +145 -0
- package/frontend/src/lib/analysis/domain-control-records.js +88 -0
- package/frontend/src/lib/analysis/external-findings-import.js +70 -6
- package/frontend/src/lib/analysis/investigation-capsule.js +47 -13
- package/frontend/src/lib/analysis/investigation-guide.js +52 -10
- package/frontend/src/lib/analysis/investigation-template-model.js +9 -6
- package/frontend/src/lib/analysis/lookup-asset-graph.js +8 -8
- package/frontend/src/lib/analysis/lookup-claim-passport.js +236 -0
- package/frontend/src/lib/analysis/lookup-claim-readiness.js +280 -0
- package/frontend/src/lib/analysis/lookup-decision-support.js +63 -8
- package/frontend/src/lib/analysis/lookup-presentation.js +70 -14
- package/frontend/src/lib/analysis/lookup-source-refresh.js +36 -63
- package/frontend/src/lib/analysis/opaque-reference-id.js +9 -0
- package/frontend/src/lib/analysis/utils.js +27 -4
- package/frontend/src/lib/analysis/visualization-matrix.js +71 -28
- package/frontend/src/lib/analysis/visualization-models.js +63 -94
- package/frontend/src/lib/analysis/visualization-time-series.js +53 -12
- package/frontend/src/lib/analysis/website-snapshot-model.js +6 -0
- package/frontend/src/lib/analysis/workspace-archive-crypto.js +1 -4
- package/frontend/src/lib/analysis/workspace-archive.js +105 -19
- package/lib/application-version.mjs +3 -0
- package/lib/availability.mjs +14 -12
- package/lib/bounded-file.mjs +82 -0
- package/lib/bounded-json-response.mjs +170 -0
- package/lib/bounded-json.mjs +249 -0
- package/lib/browser-library-profile.mjs +167 -38
- package/lib/certificate-extension-profile.mjs +2 -1
- package/lib/cryptographic-assurance.mjs +155 -0
- package/lib/ct-search.mjs +87 -47
- package/lib/dns-intelligence.mjs +3 -2
- package/lib/dns-resolver-selection.mjs +6 -4
- package/lib/dnssec-chain-validation.mjs +1053 -0
- package/lib/dnssec-evidence-validation.mjs +10 -3
- package/lib/domain-change-packet.mjs +7 -5
- package/lib/domain-control-manifest.mjs +17 -129
- package/lib/domain-posture.mjs +7 -4
- package/lib/evidence-export.mjs +751 -78
- package/lib/evidence-report-markdown.mjs +3 -0
- package/lib/evidence-report.mjs +5 -0
- package/lib/favicon.mjs +8 -5
- package/lib/http-evidence-bounds.mjs +3 -0
- package/lib/http-intelligence.mjs +2 -5
- package/lib/interchange-fidelity-registry.mjs +122 -0
- package/lib/local-geoip-evidence.mjs +2 -1
- package/lib/lookup-cache.mjs +5 -1
- package/lib/lookup-network-evidence-bounds.mjs +9 -0
- package/lib/lookup-response-contract.mjs +691 -3
- package/lib/lookup.mjs +2 -2
- package/lib/observation.mjs +7 -6
- package/lib/outbound-identity.mjs +2 -3
- package/lib/perceptual-hash.mjs +12 -4
- package/lib/portable-generator.mjs +21 -0
- package/lib/provider-policy-admission.mjs +9 -4
- package/lib/rdap-normalization.mjs +18 -5
- package/lib/rdap-registrar.mjs +5 -1
- package/lib/rdap-transport.mjs +6 -6
- package/lib/registry-capabilities.mjs +3 -0
- package/lib/registry-capability-catalogue.mjs +12 -2
- package/lib/registry-comparison.mjs +28 -11
- package/lib/risk-calibration-summary.mjs +338 -0
- package/lib/rpki-evidence.mjs +9 -8
- package/lib/safe-fetch.mjs +96 -42
- package/lib/security-txt.mjs +26 -3
- package/lib/service-binding-dns.mjs +6 -1
- package/lib/smtp-transport-review.mjs +784 -0
- package/lib/sslbl-certificates.generated.mjs +136 -134
- package/lib/threat-intelligence-runtime.mjs +4 -13
- package/lib/threat-intelligence-types.mjs +18 -1
- package/lib/tls-intelligence.mjs +11 -5
- package/lib/tlsa-evidence.mjs +37 -7
- package/lib/website-technology.mjs +2 -6
- package/package.json +1 -1
- package/third-party-notices.txt +95 -5817
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { canonicalArtifactJsonV2 } from "../frontend/src/lib/analysis/artifact-integrity.js";
|
|
4
|
+
import { BULK_SESSION_SCHEMA, MAX_BULK_SESSIONS, MAX_BULK_SESSION_ROWS, SUPPORTED_BULK_SESSION_SCHEMA_VERSIONS, mergeBulkSessions, normalizeBulkSession, } from "../frontend/src/lib/analysis/bulk-session-model.js";
|
|
5
|
+
import { buildBulkComparisonCandidates } from "../frontend/src/lib/analysis/comparison-ledger-bulk.js";
|
|
6
|
+
import { MAX_COMPARISON_LEDGER_DETAIL_ROWS, boundedComparisonLedgerLimitations, comparisonLedgerCollector, comparisonLedgerValuePresent, freshComparisonLedgerCounters, makeComparisonLedgerCandidate, normaliseComparisonLedgerRowWithCounters, stableComparisonLedgerId, stableComparisonLedgerJson, } from "../frontend/src/lib/analysis/comparison-ledger-contract.js";
|
|
7
|
+
import { DOMAIN_PORTFOLIO_INPUT_SCHEMA, DOMAIN_PORTFOLIO_REVIEW_SCHEMA, DOMAIN_PORTFOLIO_REVIEW_VERSION, reviewDomainPortfolio, } from "../lib/domain-portfolio-review.mjs";
|
|
8
|
+
import { parseBoundedJsonObject } from "./bounded-json.mjs";
|
|
9
|
+
import { CliUsageError } from "./errors.mjs";
|
|
10
|
+
import { buildCliLookupDiff, formatCliLookupDiff } from "./lookup-diff.mjs";
|
|
11
|
+
import { SAVED_LOOKUP_SCHEMA } from "./saved-lookup.mjs";
|
|
12
|
+
export const CLI_COMPARISON_LEDGER_SCHEMA = 'whoisleuth.cli.comparison-ledger';
|
|
13
|
+
export const CLI_COMPARISON_LEDGER_VERSION = 1;
|
|
14
|
+
export const MAX_RETAINED_ARTIFACT_DIFF_BYTES = 8 * 1024 * 1024;
|
|
15
|
+
const BULK_ROOT_KEYS = new Set(['schema', 'version', 'generatedAt', 'sessions', 'limitations']);
|
|
16
|
+
const PORTFOLIO_ROOT_KEYS = new Set(['schema', 'version', 'generatedAt', 'portfolioLabel', 'assets', 'simulations', 'renewalQueue', 'recoveryCycles', 'unknownCounts', 'limitations']);
|
|
17
|
+
const SAFE_SESSION_ID_RE = /^[A-Za-z0-9_-]{1,128}$/u;
|
|
18
|
+
function record(value) {
|
|
19
|
+
return value && typeof value === 'object' && !Array.isArray(value) ? value : null;
|
|
20
|
+
}
|
|
21
|
+
function exactKeys(value, allowed, label) {
|
|
22
|
+
if (Object.keys(value).some((key) => !allowed.has(key)) || [...allowed].some((key) => !Object.hasOwn(value, key))) {
|
|
23
|
+
throw new CliUsageError(`${label} does not match its exact retained export contract.`);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
function timestamp(value, label) {
|
|
27
|
+
if (typeof value !== 'string' || value.length > 64 || /[\u0000-\u001f\u007f]/u.test(value)) {
|
|
28
|
+
throw new CliUsageError(`${label} is missing or invalid.`);
|
|
29
|
+
}
|
|
30
|
+
const parsed = Date.parse(value);
|
|
31
|
+
if (!Number.isFinite(parsed))
|
|
32
|
+
throw new CliUsageError(`${label} is missing or invalid.`);
|
|
33
|
+
return new Date(parsed).toISOString();
|
|
34
|
+
}
|
|
35
|
+
function digest(value) {
|
|
36
|
+
return `sha256:${createHash('sha256').update(value, 'utf8').digest('hex')}`;
|
|
37
|
+
}
|
|
38
|
+
function parseDocument(raw, label) {
|
|
39
|
+
try {
|
|
40
|
+
return parseBoundedJsonObject(raw, { label, maximumBytes: MAX_RETAINED_ARTIFACT_DIFF_BYTES });
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
throw new CliUsageError(error instanceof Error ? error.message : `${label} is invalid.`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function assertLimitations(value, label) {
|
|
47
|
+
if (!Array.isArray(value) || value.length > 8 || value.some((item) => (typeof item !== 'string' || item.length > 600 || /[\u0000-\u001f\u007f]/u.test(item))))
|
|
48
|
+
throw new CliUsageError(`${label} limitations are invalid.`);
|
|
49
|
+
}
|
|
50
|
+
function parseBulkExport(root) {
|
|
51
|
+
exactKeys(root, BULK_ROOT_KEYS, 'Bulk session export');
|
|
52
|
+
const version = Number(root.version);
|
|
53
|
+
if (root.schema !== BULK_SESSION_SCHEMA || !SUPPORTED_BULK_SESSION_SCHEMA_VERSIONS.includes(version)) {
|
|
54
|
+
throw new CliUsageError(`Bulk session exports must use a supported ${BULK_SESSION_SCHEMA} version.`);
|
|
55
|
+
}
|
|
56
|
+
const generatedAt = timestamp(root.generatedAt, 'Bulk session export generatedAt');
|
|
57
|
+
assertLimitations(root.limitations, 'Bulk session export');
|
|
58
|
+
if (!Array.isArray(root.sessions) || root.sessions.length < 1 || root.sessions.length > MAX_BULK_SESSIONS) {
|
|
59
|
+
throw new CliUsageError(`Bulk session exports used by diff must contain from 1 to ${MAX_BULK_SESSIONS} sessions.`);
|
|
60
|
+
}
|
|
61
|
+
const ids = new Set();
|
|
62
|
+
for (const [index, candidate] of root.sessions.entries()) {
|
|
63
|
+
const rawSession = record(candidate);
|
|
64
|
+
const rawResults = Array.isArray(rawSession?.results) ? rawSession.results : null;
|
|
65
|
+
const rawDomains = Array.isArray(rawSession?.domains) ? rawSession.domains : null;
|
|
66
|
+
const session = normalizeBulkSession(candidate, version);
|
|
67
|
+
if (!session || !rawResults || !rawDomains
|
|
68
|
+
|| rawResults.length > MAX_BULK_SESSION_ROWS
|
|
69
|
+
|| rawResults.length !== session.results.length
|
|
70
|
+
|| rawDomains.length !== session.domains.length
|
|
71
|
+
|| ids.has(session.id)) {
|
|
72
|
+
throw new CliUsageError(`Bulk session export row ${index + 1} is malformed, duplicated, or would be silently omitted.`);
|
|
73
|
+
}
|
|
74
|
+
ids.add(session.id);
|
|
75
|
+
}
|
|
76
|
+
const imported = mergeBulkSessions([], root);
|
|
77
|
+
if (imported.skipped || imported.pruned || imported.sessions.length !== root.sessions.length) {
|
|
78
|
+
throw new CliUsageError('Bulk session export cannot be compared without omitting or pruning retained sessions.');
|
|
79
|
+
}
|
|
80
|
+
return Object.freeze({ version, generatedAt, sessions: Object.freeze(imported.sessions) });
|
|
81
|
+
}
|
|
82
|
+
function selectSession(document, requested, side) {
|
|
83
|
+
if (requested !== null && requested !== undefined && !SAFE_SESSION_ID_RE.test(requested)) {
|
|
84
|
+
throw new CliUsageError(`The ${side} saved-session ID is invalid.`);
|
|
85
|
+
}
|
|
86
|
+
if (!requested && document.sessions.length !== 1) {
|
|
87
|
+
throw new CliUsageError(`The ${side} Bulk export contains multiple sessions; select one with --${side}-session.`);
|
|
88
|
+
}
|
|
89
|
+
const selected = requested
|
|
90
|
+
? document.sessions.find((session) => session.id === requested)
|
|
91
|
+
: document.sessions[0];
|
|
92
|
+
if (!selected)
|
|
93
|
+
throw new CliUsageError(`The selected ${side} saved session was not found.`);
|
|
94
|
+
return selected;
|
|
95
|
+
}
|
|
96
|
+
function withoutOwnerHref(index) {
|
|
97
|
+
return Object.freeze({
|
|
98
|
+
...index,
|
|
99
|
+
items: Object.freeze(index.items.map((item) => Object.freeze({ ...item, ownerHref: '' }))),
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
function withoutDetailOwnerHref(details) {
|
|
103
|
+
return Object.freeze({
|
|
104
|
+
...details,
|
|
105
|
+
selectedItems: Object.freeze(details.selectedItems.map((item) => Object.freeze({ ...item, ownerHref: '' }))),
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
function buildBulkLedger(leftRaw, rightRaw, leftRoot, rightRoot, options, generatedAt) {
|
|
109
|
+
const leftDocument = parseBulkExport(leftRoot);
|
|
110
|
+
const rightDocument = parseBulkExport(rightRoot);
|
|
111
|
+
const left = selectSession(leftDocument, options.leftSessionId, 'left');
|
|
112
|
+
const right = selectSession(rightDocument, options.rightSessionId, 'right');
|
|
113
|
+
if (left.updatedAt > right.updatedAt)
|
|
114
|
+
throw new CliUsageError('Bulk diff requires the left saved session to be no later than the right saved session.');
|
|
115
|
+
const leftTransientId = stableComparisonLedgerId('cli-bulk-left', [left.id, left.updatedAt, digest(leftRaw)]);
|
|
116
|
+
const rightTransientId = stableComparisonLedgerId('cli-bulk-right', [right.id, right.updatedAt, digest(rightRaw)]);
|
|
117
|
+
const transientLeft = { ...left, id: leftTransientId };
|
|
118
|
+
const transientRight = { ...right, id: rightTransientId };
|
|
119
|
+
const counters = freshComparisonLedgerCounters();
|
|
120
|
+
const candidates = buildBulkComparisonCandidates([transientLeft, transientRight], [{ earlierSessionId: leftTransientId, laterSessionId: rightTransientId }], counters);
|
|
121
|
+
const candidate = candidates[0];
|
|
122
|
+
if (!candidate || candidates.length !== 1)
|
|
123
|
+
throw new CliUsageError('The selected Bulk sessions are not a compatible chronological pair.');
|
|
124
|
+
const projected = projectSingleCandidate(candidate, counters);
|
|
125
|
+
return Object.freeze({
|
|
126
|
+
schema: CLI_COMPARISON_LEDGER_SCHEMA,
|
|
127
|
+
version: CLI_COMPARISON_LEDGER_VERSION,
|
|
128
|
+
generatedAt: timestamp(generatedAt, 'Comparison ledger generatedAt'),
|
|
129
|
+
artifactFamily: 'bulk_sessions',
|
|
130
|
+
left: Object.freeze({ schema: BULK_SESSION_SCHEMA, version: leftDocument.version, id: left.id, label: left.name, retainedAt: left.updatedAt }),
|
|
131
|
+
right: Object.freeze({ schema: BULK_SESSION_SCHEMA, version: rightDocument.version, id: right.id, label: right.name, retainedAt: right.updatedAt }),
|
|
132
|
+
index: withoutOwnerHref(projected.index),
|
|
133
|
+
details: withoutDetailOwnerHref(projected.details),
|
|
134
|
+
limitations: Object.freeze([
|
|
135
|
+
'The ledger compares one explicit chronological pair of compact retained Bulk sessions without making a request or reading source file paths.',
|
|
136
|
+
'Portable session claims are treated as unauthenticated imports: profile-derived trust, matches, and Risk remain quarantined while independent retained source fields can still be reviewed.',
|
|
137
|
+
'Missing later rows are not domain removals, releases, resolution changes, or availability findings.',
|
|
138
|
+
]),
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
function parsePortfolioReview(root) {
|
|
142
|
+
exactKeys(root, PORTFOLIO_ROOT_KEYS, 'Domain portfolio review');
|
|
143
|
+
if (root.schema !== DOMAIN_PORTFOLIO_REVIEW_SCHEMA || root.version !== DOMAIN_PORTFOLIO_REVIEW_VERSION) {
|
|
144
|
+
throw new CliUsageError(`Domain portfolio reviews must use ${DOMAIN_PORTFOLIO_REVIEW_SCHEMA} version ${DOMAIN_PORTFOLIO_REVIEW_VERSION}.`);
|
|
145
|
+
}
|
|
146
|
+
const rebuilt = reviewDomainPortfolio({
|
|
147
|
+
schema: DOMAIN_PORTFOLIO_INPUT_SCHEMA,
|
|
148
|
+
version: 1,
|
|
149
|
+
portfolioLabel: root.portfolioLabel,
|
|
150
|
+
assets: root.assets,
|
|
151
|
+
}, timestamp(root.generatedAt, 'Domain portfolio review generatedAt'));
|
|
152
|
+
if (canonicalArtifactJsonV2(rebuilt) !== canonicalArtifactJsonV2(root)) {
|
|
153
|
+
throw new CliUsageError('Domain portfolio review derived fields do not match its bounded source asset projection.');
|
|
154
|
+
}
|
|
155
|
+
return rebuilt;
|
|
156
|
+
}
|
|
157
|
+
const PORTFOLIO_FIELDS = Object.freeze([
|
|
158
|
+
['Criticality', 'portfolio', (item) => item.criticality],
|
|
159
|
+
['Registrar', 'registration', (item) => item.registrar],
|
|
160
|
+
['Registrar account label', 'registration', (item) => item.registrarAccount],
|
|
161
|
+
['Expiry', 'registration', (item) => item.expiresAt],
|
|
162
|
+
['Auto-renew', 'registration', (item) => item.autoRenew],
|
|
163
|
+
['DNS providers', 'dns', (item) => item.dnsProviders],
|
|
164
|
+
['Mail providers', 'mail', (item) => item.mailProviders],
|
|
165
|
+
['Certificate providers', 'certificate', (item) => item.certificateProviders],
|
|
166
|
+
['Recovery domains', 'recovery', (item) => item.recoveryDomains],
|
|
167
|
+
]);
|
|
168
|
+
function portfolioSide(report, value, observedAt = null) {
|
|
169
|
+
return {
|
|
170
|
+
source: `Retained domain portfolio · ${report.portfolioLabel}`,
|
|
171
|
+
sourceState: 'analyst_supplied',
|
|
172
|
+
value,
|
|
173
|
+
observedAt,
|
|
174
|
+
retainedAt: report.generatedAt,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
function portfolioRows(ownerId, earlier, later) {
|
|
178
|
+
const output = comparisonLedgerCollector();
|
|
179
|
+
let incomplete = false;
|
|
180
|
+
const before = new Map(earlier.assets.map((asset) => [asset.domain, asset]));
|
|
181
|
+
const after = new Map(later.assets.map((asset) => [asset.domain, asset]));
|
|
182
|
+
for (const domain of [...new Set([...before.keys(), ...after.keys()])].sort()) {
|
|
183
|
+
const left = before.get(domain);
|
|
184
|
+
const right = after.get(domain);
|
|
185
|
+
if (!left && right) {
|
|
186
|
+
output.add({ comparisonId: ownerId, ownerId, entityId: domain, mode: 'membership', state: 'added', field: 'Retained portfolio membership', family: 'membership', earlier: portfolioSide(earlier, null), later: portfolioSide(later, 'Included', right.reviewedAt), completeness: 'complete', limitations: ['Membership applies only to the two analyst-supplied retained portfolios; it is not evidence of registration, ownership, control, or activity.'] });
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
if (left && !right) {
|
|
190
|
+
incomplete = true;
|
|
191
|
+
output.add({ comparisonId: ownerId, ownerId, entityId: domain, mode: 'membership', state: 'not_compared', field: 'Retained portfolio membership', family: 'membership', earlier: portfolioSide(earlier, 'Included', left.reviewedAt), later: portfolioSide(later, null), completeness: 'partial', limitations: ['The later retained portfolio omits this asset. That omission is not evidence of registration removal, release, loss of control, or inactivity.'] });
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
if (!left || !right)
|
|
195
|
+
continue;
|
|
196
|
+
const leftRecord = left;
|
|
197
|
+
const rightRecord = right;
|
|
198
|
+
for (const [field, family, read] of PORTFOLIO_FIELDS) {
|
|
199
|
+
const beforeValue = read(leftRecord);
|
|
200
|
+
const afterValue = read(rightRecord);
|
|
201
|
+
if (stableComparisonLedgerJson(beforeValue) === stableComparisonLedgerJson(afterValue))
|
|
202
|
+
continue;
|
|
203
|
+
const afterPresent = comparisonLedgerValuePresent(afterValue);
|
|
204
|
+
const beforePresent = comparisonLedgerValuePresent(beforeValue);
|
|
205
|
+
if (beforePresent && !afterPresent)
|
|
206
|
+
incomplete = true;
|
|
207
|
+
output.add({
|
|
208
|
+
comparisonId: ownerId,
|
|
209
|
+
ownerId,
|
|
210
|
+
entityId: domain,
|
|
211
|
+
mode: 'temporal',
|
|
212
|
+
state: !beforePresent && afterPresent ? 'added' : beforePresent && !afterPresent ? 'incomplete' : 'different',
|
|
213
|
+
field,
|
|
214
|
+
family,
|
|
215
|
+
earlier: portfolioSide(earlier, beforeValue, left.reviewedAt),
|
|
216
|
+
later: portfolioSide(later, afterValue, right.reviewedAt),
|
|
217
|
+
completeness: beforePresent && !afterPresent ? 'partial' : 'complete',
|
|
218
|
+
limitations: ['Values are bounded analyst-supplied portfolio assertions, not current provider observations or proof of ownership, configuration, or responsibility.'],
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
if (left.reviewedAt !== right.reviewedAt) {
|
|
222
|
+
output.add({ comparisonId: ownerId, ownerId, entityId: domain, mode: 'temporal', state: 'collection_changed', field: 'Analyst review time', family: 'collection', earlier: portfolioSide(earlier, left.reviewedAt, left.reviewedAt), later: portfolioSide(later, right.reviewedAt, right.reviewedAt), completeness: 'complete', limitations: ['Review-time movement records analyst retention context and is not a change to the domain or provider.'] });
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (!output.totalRows) {
|
|
226
|
+
output.add({ comparisonId: ownerId, ownerId, entityId: 'portfolio-summary', mode: 'temporal', state: 'equivalent', field: 'Bounded retained portfolio assertions', family: 'summary', earlier: portfolioSide(earlier, 'No bounded material difference'), later: portfolioSide(later, 'No bounded material difference'), completeness: 'complete', limitations: ['Equivalence applies only to the exact bounded analyst-supplied fields retained in both portfolio review documents.'] });
|
|
227
|
+
}
|
|
228
|
+
return Object.freeze({
|
|
229
|
+
projection: Object.freeze({ rows: output.rows, totalRows: output.totalRows, sourceOmittedRows: 0 }),
|
|
230
|
+
incomplete,
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
function projectSingleCandidate(candidate, counters) {
|
|
234
|
+
const indexOmissions = Object.freeze({
|
|
235
|
+
inputRecords: counters.inputRecords,
|
|
236
|
+
inputScanTruncations: counters.inputScanTruncations,
|
|
237
|
+
invalidRecords: counters.invalidRecords,
|
|
238
|
+
duplicateRecords: counters.duplicateRecords,
|
|
239
|
+
indexItems: 0,
|
|
240
|
+
limitations: counters.limitations,
|
|
241
|
+
truncatedStrings: counters.truncatedStrings,
|
|
242
|
+
});
|
|
243
|
+
const index = Object.freeze({ items: Object.freeze([candidate.item]), counts: Object.freeze({ candidates: 1, retained: 1 }), omissions: indexOmissions, truncated: candidate.item.truncated || Object.values(indexOmissions).some((count) => count > 0) });
|
|
244
|
+
const projection = candidate.buildDetails();
|
|
245
|
+
const rows = [];
|
|
246
|
+
const ids = new Set();
|
|
247
|
+
let invalidRows = 0;
|
|
248
|
+
let duplicateRows = 0;
|
|
249
|
+
for (const rawRow of projection.rows) {
|
|
250
|
+
const row = normaliseComparisonLedgerRowWithCounters(rawRow, counters);
|
|
251
|
+
if (!row) {
|
|
252
|
+
invalidRows += 1;
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
if (ids.has(row.id)) {
|
|
256
|
+
duplicateRows += 1;
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
ids.add(row.id);
|
|
260
|
+
if (rows.length < MAX_COMPARISON_LEDGER_DETAIL_ROWS)
|
|
261
|
+
rows.push(row);
|
|
262
|
+
}
|
|
263
|
+
rows.sort((left, right) => left.entityId.localeCompare(right.entityId) || left.family.localeCompare(right.family) || left.field.localeCompare(right.field) || left.id.localeCompare(right.id));
|
|
264
|
+
const totalRows = Math.max(0, projection.totalRows - duplicateRows);
|
|
265
|
+
const limitations = boundedComparisonLedgerLimitations(candidate.item.limitations, counters);
|
|
266
|
+
const omissions = Object.freeze({
|
|
267
|
+
inputRecords: counters.inputRecords,
|
|
268
|
+
inputScanTruncations: counters.inputScanTruncations,
|
|
269
|
+
invalidRecords: counters.invalidRecords + invalidRows,
|
|
270
|
+
duplicateRecords: counters.duplicateRecords,
|
|
271
|
+
indexItems: 0,
|
|
272
|
+
entityRequests: 0,
|
|
273
|
+
duplicateEntityRequests: 0,
|
|
274
|
+
invalidEntityRequests: 0,
|
|
275
|
+
missingEntities: 0,
|
|
276
|
+
duplicateDetailRows: duplicateRows,
|
|
277
|
+
detailRows: Math.max(0, totalRows - rows.length),
|
|
278
|
+
sourceRows: projection.sourceOmittedRows,
|
|
279
|
+
limitations: counters.limitations,
|
|
280
|
+
truncatedStrings: counters.truncatedStrings,
|
|
281
|
+
});
|
|
282
|
+
const details = Object.freeze({ selectedItems: Object.freeze([candidate.item]), rows: Object.freeze(rows), totalRows, limitations: limitations.values, omissions, truncated: candidate.item.truncated || Object.values(omissions).some((count) => count > 0) });
|
|
283
|
+
return Object.freeze({ index, details });
|
|
284
|
+
}
|
|
285
|
+
function buildPortfolioLedger(leftRaw, rightRaw, leftRoot, rightRoot, generatedAt) {
|
|
286
|
+
const left = parsePortfolioReview(leftRoot);
|
|
287
|
+
const right = parsePortfolioReview(rightRoot);
|
|
288
|
+
if (left.generatedAt > right.generatedAt)
|
|
289
|
+
throw new CliUsageError('Domain portfolio diff requires the left review to be no later than the right review.');
|
|
290
|
+
const counters = freshComparisonLedgerCounters();
|
|
291
|
+
const ownerId = stableComparisonLedgerId('retained-portfolio-pair', [digest(leftRaw), digest(rightRaw)]);
|
|
292
|
+
const portfolioProjection = portfolioRows(ownerId, left, right);
|
|
293
|
+
const { projection } = portfolioProjection;
|
|
294
|
+
const completeness = portfolioProjection.incomplete ? 'partial' : 'complete';
|
|
295
|
+
const projectionTruncated = projection.totalRows > projection.rows.length;
|
|
296
|
+
const candidate = makeComparisonLedgerCandidate({
|
|
297
|
+
idParts: [digest(leftRaw), digest(rightRaw)],
|
|
298
|
+
ownerType: 'retained_artifact_pair',
|
|
299
|
+
ownerId,
|
|
300
|
+
entityId: 'domain-portfolio-pair',
|
|
301
|
+
label: `${left.portfolioLabel} → ${right.portfolioLabel} · retained portfolio pair`,
|
|
302
|
+
mode: 'temporal',
|
|
303
|
+
earlier: portfolioSide(left, null),
|
|
304
|
+
later: portfolioSide(right, null),
|
|
305
|
+
completeness,
|
|
306
|
+
truncated: projectionTruncated,
|
|
307
|
+
limitations: [
|
|
308
|
+
'This offline ledger compares bounded retained analyst-supplied portfolio assertions and makes no provider request.',
|
|
309
|
+
...(completeness === 'partial' ? ['At least one later omission is not represented as a real-world removal or resolved dependency.'] : []),
|
|
310
|
+
],
|
|
311
|
+
ownerHref: '',
|
|
312
|
+
buildDetails: () => projection,
|
|
313
|
+
}, counters);
|
|
314
|
+
if (!candidate)
|
|
315
|
+
throw new CliUsageError('The retained domain portfolio pair could not be projected safely.');
|
|
316
|
+
const projected = projectSingleCandidate(candidate, counters);
|
|
317
|
+
return Object.freeze({
|
|
318
|
+
schema: CLI_COMPARISON_LEDGER_SCHEMA,
|
|
319
|
+
version: CLI_COMPARISON_LEDGER_VERSION,
|
|
320
|
+
generatedAt: timestamp(generatedAt, 'Comparison ledger generatedAt'),
|
|
321
|
+
artifactFamily: 'domain_portfolio',
|
|
322
|
+
left: Object.freeze({ schema: DOMAIN_PORTFOLIO_REVIEW_SCHEMA, version: DOMAIN_PORTFOLIO_REVIEW_VERSION, id: digest(leftRaw), label: left.portfolioLabel, retainedAt: left.generatedAt }),
|
|
323
|
+
right: Object.freeze({ schema: DOMAIN_PORTFOLIO_REVIEW_SCHEMA, version: DOMAIN_PORTFOLIO_REVIEW_VERSION, id: digest(rightRaw), label: right.portfolioLabel, retainedAt: right.generatedAt }),
|
|
324
|
+
...projected,
|
|
325
|
+
limitations: Object.freeze([
|
|
326
|
+
'Only the two explicitly selected retained review documents are compared; source file paths are not retained.',
|
|
327
|
+
'Portfolio values are analyst-supplied context. They do not prove live configuration, registration, ownership, control, availability, or provider responsibility.',
|
|
328
|
+
]),
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
export function buildCliRetainedArtifactDiff(leftRaw, rightRaw, options = {}, generatedAt = new Date().toISOString()) {
|
|
332
|
+
if (Buffer.byteLength(leftRaw, 'utf8') + Buffer.byteLength(rightRaw, 'utf8') > MAX_RETAINED_ARTIFACT_DIFF_BYTES * 2) {
|
|
333
|
+
throw new CliUsageError(`Retained diff input is limited to ${MAX_RETAINED_ARTIFACT_DIFF_BYTES * 2} bytes in total.`);
|
|
334
|
+
}
|
|
335
|
+
const left = parseDocument(leftRaw, 'Left retained diff input');
|
|
336
|
+
const right = parseDocument(rightRaw, 'Right retained diff input');
|
|
337
|
+
if (left.schema !== right.schema)
|
|
338
|
+
throw new CliUsageError('diff requires two retained documents from the same supported artifact family.');
|
|
339
|
+
if (left.schema === SAVED_LOOKUP_SCHEMA) {
|
|
340
|
+
if (options.leftSessionId || options.rightSessionId)
|
|
341
|
+
throw new CliUsageError('Saved-session selectors apply only to Bulk session exports.');
|
|
342
|
+
return buildCliLookupDiff(leftRaw, rightRaw, generatedAt);
|
|
343
|
+
}
|
|
344
|
+
if (left.schema === BULK_SESSION_SCHEMA)
|
|
345
|
+
return buildBulkLedger(leftRaw, rightRaw, left, right, options, generatedAt);
|
|
346
|
+
if (left.schema === DOMAIN_PORTFOLIO_REVIEW_SCHEMA) {
|
|
347
|
+
if (options.leftSessionId || options.rightSessionId)
|
|
348
|
+
throw new CliUsageError('Saved-session selectors apply only to Bulk session exports.');
|
|
349
|
+
return buildPortfolioLedger(leftRaw, rightRaw, left, right, generatedAt);
|
|
350
|
+
}
|
|
351
|
+
throw new CliUsageError('diff supports saved Lookups, Bulk session exports, or retained domain portfolio reviews.');
|
|
352
|
+
}
|
|
353
|
+
export function formatCliRetainedArtifactDiff(document) {
|
|
354
|
+
if (document.schema !== CLI_COMPARISON_LEDGER_SCHEMA)
|
|
355
|
+
return formatCliLookupDiff(document);
|
|
356
|
+
const lines = [
|
|
357
|
+
'Retained artifact comparison ledger',
|
|
358
|
+
`Family ${document.artifactFamily.replaceAll('_', ' ')}`,
|
|
359
|
+
`Left ${document.left.label} · ${document.left.id}`,
|
|
360
|
+
`Right ${document.right.label} · ${document.right.id}`,
|
|
361
|
+
`Candidates ${document.index.counts.retained}`,
|
|
362
|
+
`Exact rows ${document.details.rows.length} of ${document.details.totalRows}`,
|
|
363
|
+
'',
|
|
364
|
+
];
|
|
365
|
+
for (const row of document.details.rows) {
|
|
366
|
+
lines.push(`${row.entityId} · ${row.field} [${row.state.replaceAll('_', ' ')}]`);
|
|
367
|
+
lines.push(` Earlier: ${row.earlier.value ?? 'not retained'} · ${row.earlier.sourceState}`);
|
|
368
|
+
lines.push(` Later: ${row.later.value ?? 'not retained'} · ${row.later.sourceState}`);
|
|
369
|
+
}
|
|
370
|
+
if (document.details.omissions.detailRows)
|
|
371
|
+
lines.push(`Omitted exact rows: ${document.details.omissions.detailRows}`);
|
|
372
|
+
lines.push('', 'Limitations:');
|
|
373
|
+
for (const limitation of document.limitations)
|
|
374
|
+
lines.push(` - ${limitation}`);
|
|
375
|
+
return `${lines.join('\n')}\n`;
|
|
376
|
+
}
|
package/cli/risk-calibration.mjs
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
2
|
import { isValidAsciiDomainName } from "../lib/hostname.mjs";
|
|
3
3
|
import { ANALYST_REVIEW_REASON_VALUES, analystInteroperabilityTags } from "../lib/analyst-taxonomy.mjs";
|
|
4
|
+
import { RISK_CALIBRATION_SUMMARY_SCHEMA, RISK_CALIBRATION_SUMMARY_VERSION, RISK_CALIBRATION_SUMMARY_INTERPRETATION, RISK_CALIBRATION_SUMMARY_THRESHOLDS, } from "../lib/risk-calibration-summary.mjs";
|
|
4
5
|
import { CliUsageError } from "./arguments.mjs";
|
|
5
|
-
import { RISK_MUTATION_TYPES
|
|
6
|
+
import { RISK_MUTATION_TYPES } from "../lib/risk-scoring.mjs";
|
|
6
7
|
export const RISK_CALIBRATION_DATASET_SCHEMA = 'whoisleuth.risk-calibration-dataset';
|
|
7
8
|
export const RISK_CALIBRATION_DATASET_VERSION = 2;
|
|
8
9
|
export const SUPPORTED_RISK_CALIBRATION_DATASET_VERSIONS = Object.freeze([1, RISK_CALIBRATION_DATASET_VERSION]);
|
|
9
|
-
export const RISK_CALIBRATION_REPORT_SCHEMA =
|
|
10
|
-
export const RISK_CALIBRATION_REPORT_VERSION =
|
|
10
|
+
export const RISK_CALIBRATION_REPORT_SCHEMA = RISK_CALIBRATION_SUMMARY_SCHEMA;
|
|
11
|
+
export const RISK_CALIBRATION_REPORT_VERSION = RISK_CALIBRATION_SUMMARY_VERSION;
|
|
11
12
|
export const MAX_RISK_CALIBRATION_INPUT_BYTES = 2 * 1024 * 1024;
|
|
12
13
|
export const MAX_RISK_CALIBRATION_RECORDS = 500;
|
|
13
14
|
export const MAX_RISK_CALIBRATION_STRING_LENGTH = 256;
|
|
14
|
-
export const RISK_CALIBRATION_THRESHOLDS =
|
|
15
|
+
export const RISK_CALIBRATION_THRESHOLDS = RISK_CALIBRATION_SUMMARY_THRESHOLDS;
|
|
15
16
|
const MAX_MUTATIONS = 30;
|
|
16
17
|
const MAX_PROVIDERS = 10;
|
|
17
18
|
const MAX_FINDINGS_PER_PROVIDER = 100;
|
|
@@ -375,6 +376,7 @@ export function buildRiskCalibrationReport(dataset, explainRiskScore, options) {
|
|
|
375
376
|
return {
|
|
376
377
|
schema: RISK_CALIBRATION_REPORT_SCHEMA,
|
|
377
378
|
version: RISK_CALIBRATION_REPORT_VERSION,
|
|
379
|
+
mode: 'detailed',
|
|
378
380
|
generatedAt: options.generatedAt || new Date().toISOString(),
|
|
379
381
|
dataset: { schema: dataset.schema, version: dataset.version, recordCount: records.length },
|
|
380
382
|
riskModelVersion: options.modelVersion,
|
|
@@ -393,7 +395,7 @@ export function buildRiskCalibrationReport(dataset, explainRiskScore, options) {
|
|
|
393
395
|
records,
|
|
394
396
|
interpretation: {
|
|
395
397
|
authority: 'analyst_context_only',
|
|
396
|
-
statement:
|
|
398
|
+
statement: RISK_CALIBRATION_SUMMARY_INTERPRETATION,
|
|
397
399
|
automaticTuning: false,
|
|
398
400
|
networkRequests: false,
|
|
399
401
|
persisted: false,
|