@slicedearth/whoisleuth-cli 2.5.1 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +9 -6
- package/NOTICE +3 -2
- package/README.md +60 -67
- package/cli/artifact-validation/case-response.mjs +6 -814
- package/cli/artifact-validation/lookup-evidence.mjs +30 -3
- package/cli/artifact-validation/signed-review.mjs +2 -51
- package/cli/artifact-verify.mjs +41 -3
- package/cli/bulk-command-runner.mjs +4 -6
- package/cli/case-command.mjs +92 -16
- package/cli/case-pack.mjs +14 -3
- package/cli/ci-report.mjs +17 -5
- package/cli/command-definition.mjs +7 -6
- package/cli/discovery-scan-command-runner.mjs +4 -6
- package/cli/doctor.mjs +37 -3
- package/cli/domain-feed.mjs +61 -0
- package/cli/formatters/html.mjs +3 -0
- package/cli/formatters/json.mjs +9 -14
- package/cli/formatters/terminal-lookup.mjs +24 -1
- package/cli/history-command-definitions.mjs +1 -1
- package/cli/indicator-set-command.mjs +1 -2
- package/cli/intake-command.mjs +43 -3
- package/cli/investigation-artifacts.mjs +2 -3
- package/cli/investigation-package-review.mjs +2 -0
- package/cli/investigation-plan.mjs +15 -0
- package/cli/local-mmdb-review.mjs +27 -4
- package/cli/lookup-brief.mjs +11 -5
- package/cli/lookup-command-runner.mjs +21 -13
- package/cli/offline-evidence-review.mjs +51 -5
- package/cli/review-arguments.mjs +3 -2
- package/cli/review-command-definitions.mjs +23 -17
- package/cli/review-command-runner.mjs +6 -3
- package/cli/risk-calibration.mjs +6 -4
- package/cli/saved-lookup.mjs +22 -7
- package/cli/selected-input-worker.mjs +12 -2
- package/cli/source-reliability.mjs +12 -10
- package/cli/support-arguments.mjs +1 -1
- package/cli/support-command-definitions.mjs +5 -4
- package/cli/support-command-runner.mjs +9 -7
- package/cli/watchlist-review.mjs +43 -0
- package/cli/workflow-arguments.mjs +14 -0
- package/cli/workflow-command-definitions.mjs +27 -1
- package/cli/workflow-command-runner.mjs +32 -0
- package/docs/cli-reference.md +49 -11
- package/docs/cli.md +200 -26
- package/docs/contextual-reviews.md +38 -5
- package/docs/infrastructure-observations.md +135 -0
- package/docs/offline-intake-context.md +213 -0
- package/lib/credential-form-attribution.mjs +136 -0
- package/lib/credential-surface-profile.mjs +6 -2
- package/lib/dmarc-discovery.mjs +4 -2
- package/lib/dns-delegation-health.mjs +23 -7
- package/lib/dns-inheritance-review.mjs +14 -1
- package/lib/dns-intelligence.mjs +18 -4
- package/lib/domain-posture-analysis.mjs +53 -2
- package/lib/domain-posture-parsers.mjs +9 -0
- package/lib/domain-posture.mjs +29 -10
- package/lib/evidence-export-privacy.mjs +5 -2
- package/lib/evidence-export.mjs +10 -4
- package/lib/evidence-report.mjs +3 -1
- package/lib/favicon.mjs +13 -4
- package/lib/generated/cisa-kev-catalog.mjs +17 -4
- package/lib/generated/registrar-standing-catalogue.mjs +7 -7
- package/lib/generated/retire-browser-catalog.mjs +178 -5
- package/lib/html-signals.mjs +6 -0
- package/lib/idn-confusables.mjs +5 -1
- package/lib/lookup-child-profile-contract.mjs +28 -11
- package/lib/lookup-collection-receipt.mjs +52 -0
- package/lib/lookup-diagnostics.mjs +2 -12
- package/lib/lookup-progress.mjs +29 -31
- package/lib/lookup-response-contract.mjs +10 -0
- package/lib/lookup-response.mjs +15 -14
- package/lib/lookup-source-progress.mjs +4 -2
- package/lib/lookup.mjs +33 -2
- package/lib/operation-budget.mjs +4 -0
- package/lib/rdap-capabilities.mjs +43 -9
- package/lib/registry-insights.mjs +17 -6
- package/lib/service-binding-dns.mjs +19 -2
- package/lib/sslbl-certificates.generated.mjs +142 -140
- package/lib/static-html-analysis.mjs +32 -4
- package/lib/technology-evidence-role.mjs +1 -51
- package/lib/tls-intelligence.mjs +30 -5
- package/lib/website-enrichment.mjs +39 -23
- package/lib/whois-authority.mjs +56 -1
- package/lib/whois-chain.mjs +14 -4
- package/lib/whois-common-formats.mjs +3 -1
- package/lib/whois-contacts.mjs +13 -2
- package/lib/whois-contracts.mjs +2 -0
- package/lib/whois-normalization.mjs +4 -4
- package/package.json +9 -7
- package/packages/analysis/bounded-json.mjs +12 -2
- package/packages/analysis/risk-scoring.mjs +12 -9
- package/packages/cases/case-attachment-model.mjs +36 -1
- package/packages/cases/case-checkpoint-receipts.mjs +68 -0
- package/packages/cases/case-containment-handoff.mjs +195 -0
- package/packages/cases/case-evidence-links.mjs +44 -0
- package/packages/cases/case-evidence-model.mjs +51 -7
- package/packages/cases/case-migration-model.mjs +9 -1
- package/packages/cases/case-object-coverage.mjs +119 -0
- package/packages/cases/case-packet-correction.mjs +106 -0
- package/packages/cases/case-recheck-model.mjs +35 -5
- package/packages/cases/case-record-operations.mjs +87 -9
- package/packages/cases/case-record-projection.mjs +43 -5
- package/packages/cases/case-record-version-input.mjs +23 -1
- package/packages/cases/case-report.mjs +35 -4
- package/packages/cases/case-requested-evidence.mjs +1 -1
- package/packages/cases/case-response-actions.mjs +57 -1
- package/packages/cases/case-response-model.mjs +7 -0
- package/packages/cases/case-response-object.mjs +119 -0
- package/packages/cases/case-response-outcomes.mjs +81 -14
- package/packages/cases/case-response-packet-render.mjs +22 -3
- package/packages/cases/case-response-packet-validation.mjs +880 -0
- package/packages/cases/case-response-packet-vocabulary.mjs +12 -1
- package/packages/cases/case-response-packet.mjs +32 -17
- package/packages/cases/case-response-review-inputs.mjs +66 -8
- package/packages/cases/case-type-presentation.mjs +10 -0
- package/packages/cases/case-workflow-metadata.mjs +1 -17
- package/packages/comparison/capture-context.mjs +3 -2
- package/packages/contracts/analyst-interchange.mjs +7 -5
- package/packages/contracts/analyst-review-state-contract.mjs +1 -1
- package/packages/contracts/candidate-watch-review.mjs +54 -0
- package/packages/contracts/case-portability.mjs +292 -43
- package/packages/contracts/cli-command-semantics.mjs +3 -0
- package/packages/contracts/cli-doctor.mjs +4 -0
- package/packages/contracts/cli-lookup.mjs +57 -46
- package/packages/contracts/context-review.mjs +2 -1
- package/packages/contracts/external-observation-interchange.mjs +96 -28
- package/packages/contracts/lookup-evidence.mjs +13 -2
- package/packages/contracts/message-intake.mjs +7 -1
- package/packages/contracts/registry-insights.mjs +6 -0
- package/packages/contracts/risk-calibration.mjs +44 -29
- package/packages/contracts/schema-lifecycle.mjs +3 -1
- package/packages/contracts/web-capture.mjs +1 -1
- package/packages/contracts/workspace-portability.mjs +86 -16
- package/packages/evidence/collection-quality.mjs +1 -1
- package/packages/evidence/credential-form-attribution.mjs +60 -0
- package/packages/evidence/dmarc-policy-context.mjs +80 -0
- package/packages/evidence/image-regions.mjs +15 -0
- package/packages/evidence/lookup-collection-receipt.mjs +140 -0
- package/packages/evidence/lookup-collection-review.mjs +74 -0
- package/packages/evidence/record-identity.mjs +60 -16
- package/packages/evidence/technology-evidence-role.mjs +51 -0
- package/packages/interchange/external-findings-import.mjs +40 -21
- package/packages/interchange/json-output.mjs +14 -0
- package/packages/interchange/managed-indicator-export.mjs +3 -2
- package/packages/interchange/managed-indicator-set.mjs +13 -0
- package/packages/investigation/acquisition-decision-packet.mjs +78 -0
- package/packages/investigation/bulk-mail-exposure.mjs +2 -2
- package/packages/investigation/context-review.mjs +3 -3
- package/packages/investigation/document-intake.mjs +9 -3
- package/packages/investigation/domain-history-review.mjs +50 -12
- package/packages/investigation/domain-transition-review.mjs +2 -0
- package/packages/investigation/infrastructure-observation.mjs +234 -0
- package/packages/investigation/intake-context.mjs +284 -0
- package/packages/investigation/intake-indicators.mjs +116 -0
- package/packages/investigation/intake-phones.mjs +155 -0
- package/packages/investigation/intake-report.mjs +2 -1
- package/packages/investigation/investigation-manifest.mjs +33 -7
- package/packages/investigation/link-intake.mjs +38 -8
- package/packages/investigation/message-intake.mjs +55 -15
- package/packages/monitoring/domain-feed.mjs +321 -0
- package/packages/workspace/brand-candidate-workflow.mjs +351 -0
- package/packages/workspace/brand-keyword-campaign.mjs +241 -0
- package/packages/workspace/brand-profile-model.mjs +46 -2
- package/packages/workspace/bulk-session-model.mjs +12 -0
- package/packages/workspace/candidate-watch-handoff.mjs +195 -0
- package/packages/workspace/detection-rule-model.mjs +10 -2
- package/packages/workspace/shortlist-model.mjs +10 -2
- package/packages/workspace/watchlist-history.mjs +31 -4
- package/packages/workspace/watchlist-store.mjs +136 -7
- package/third-party-notices.txt +30 -30
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
2
|
import { scanBoundedJson } from "../packages/analysis/bounded-json.mjs";
|
|
3
3
|
import { CONTEXT_INPUT_SCHEMAS, reviewContextInput } from "../packages/investigation/context-review.mjs";
|
|
4
|
-
import { CONTEXT_REVIEW_KINDS } from "../packages/contracts/context-review.mjs";
|
|
4
|
+
import { CONTEXT_REVIEW_KINDS, DOMAIN_HISTORY_INPUT_SCHEMA, DOMAIN_HISTORY_INPUT_VERSION } from "../packages/contracts/context-review.mjs";
|
|
5
5
|
import { normalizeEncryptedDnsAdapter, planEncryptedDnsQuery, } from "../lib/encrypted-dns-contract.mjs";
|
|
6
6
|
import { validateDnssecEvidence } from "../lib/dnssec-evidence-validation.mjs";
|
|
7
7
|
import { reviewDomainChange } from "../lib/domain-change-review.mjs";
|
|
@@ -18,6 +18,10 @@ import { compareTrustStoreEvidence } from "../lib/trust-store-comparison.mjs";
|
|
|
18
18
|
import { CliUsageError } from "./errors.mjs";
|
|
19
19
|
import { safeTerminalValue } from "./formatters/terminal.mjs";
|
|
20
20
|
import { LOCAL_MMDB_QUERY_SCHEMA, LOCAL_MMDB_QUERY_VERSION, LOCAL_MMDB_REVIEW_SCHEMA, LOCAL_MMDB_REVIEW_VERSION, reviewLocalMmdb } from "./local-mmdb-review.mjs";
|
|
21
|
+
import { INFRASTRUCTURE_OBSERVATION_SCHEMA, MAX_INFRASTRUCTURE_OBSERVATION_BYTES, infrastructureObservationFacts, compareInfrastructureObservations } from "../packages/investigation/infrastructure-observation.mjs";
|
|
22
|
+
import { exact } from "../packages/evidence/artifact-structure.mjs";
|
|
23
|
+
import { INFRASTRUCTURE_COMPARISON_INPUT_SCHEMA, MAX_INFRASTRUCTURE_COMPARISON_INPUT_BYTES } from "../packages/contracts/external-observation-interchange.mjs";
|
|
24
|
+
import { CASE_CONTAINMENT_INPUT_SCHEMA, readCaseContainmentInput, formatCaseContainmentHandoff } from "../packages/cases/case-containment-handoff.mjs";
|
|
21
25
|
const OFFLINE_EVIDENCE_REVIEW_SCHEMA = 'whoisleuth.cli.offline-evidence-review';
|
|
22
26
|
const OFFLINE_EVIDENCE_REVIEW_VERSION = 1;
|
|
23
27
|
const MAX_OFFLINE_EVIDENCE_INPUT_BYTES = 16 * 1024 * 1024;
|
|
@@ -46,7 +50,8 @@ function parseInput(value) {
|
|
|
46
50
|
}
|
|
47
51
|
const document = record(parsed);
|
|
48
52
|
if (typeof document.schema !== 'string' || document.version !== 1
|
|
49
|
-
&& !(document.schema === LOCAL_MMDB_QUERY_SCHEMA && document.version === LOCAL_MMDB_QUERY_VERSION)
|
|
53
|
+
&& !(document.schema === LOCAL_MMDB_QUERY_SCHEMA && document.version === LOCAL_MMDB_QUERY_VERSION)
|
|
54
|
+
&& !(document.schema === DOMAIN_HISTORY_INPUT_SCHEMA && document.version === DOMAIN_HISTORY_INPUT_VERSION)) {
|
|
50
55
|
throw new CliUsageError('Offline evidence review requires a supported versioned input schema.');
|
|
51
56
|
}
|
|
52
57
|
return document;
|
|
@@ -57,7 +62,27 @@ function buildOfflineEvidenceReview(value, generatedAt = new Date().toISOString(
|
|
|
57
62
|
throw new CliUsageError('Local MMDB review requires --mmdb <database-file>.');
|
|
58
63
|
let kind;
|
|
59
64
|
let result;
|
|
60
|
-
if (
|
|
65
|
+
if (input.schema === CASE_CONTAINMENT_INPUT_SCHEMA) {
|
|
66
|
+
kind = 'internal_containment';
|
|
67
|
+
result = readCaseContainmentInput(input, generatedAt);
|
|
68
|
+
}
|
|
69
|
+
else if (input.schema === INFRASTRUCTURE_OBSERVATION_SCHEMA) {
|
|
70
|
+
if (typeof value === 'string' && new TextEncoder().encode(value).byteLength > MAX_INFRASTRUCTURE_OBSERVATION_BYTES)
|
|
71
|
+
throw new TypeError('Infrastructure snapshot input exceeds its byte bound.');
|
|
72
|
+
kind = 'infrastructure';
|
|
73
|
+
const retained = infrastructureObservationFacts(input);
|
|
74
|
+
result = { ...retained, state: retained.snapshot.coverage.state === 'complete' && !retained.snapshot.coverage.truncated ? 'complete' : 'partial' };
|
|
75
|
+
}
|
|
76
|
+
else if (input.schema === INFRASTRUCTURE_COMPARISON_INPUT_SCHEMA) {
|
|
77
|
+
if (typeof value === 'string' && new TextEncoder().encode(value).byteLength > MAX_INFRASTRUCTURE_COMPARISON_INPUT_BYTES)
|
|
78
|
+
throw new TypeError('Infrastructure comparison input exceeds its byte bound.');
|
|
79
|
+
const pair = exact(input, ['schema', 'version', 'earlier', 'later'], 'Infrastructure comparison input');
|
|
80
|
+
if (new TextEncoder().encode(JSON.stringify(pair)).byteLength > MAX_INFRASTRUCTURE_COMPARISON_INPUT_BYTES)
|
|
81
|
+
throw new TypeError('Infrastructure comparison input exceeds its byte bound.');
|
|
82
|
+
kind = 'infrastructure_comparison';
|
|
83
|
+
result = compareInfrastructureObservations(pair.earlier, pair.later);
|
|
84
|
+
}
|
|
85
|
+
else if (CONTEXT_INPUT_SCHEMAS.includes(input.schema)) {
|
|
61
86
|
const review = reviewContextInput(input, generatedAt);
|
|
62
87
|
kind = review.kind;
|
|
63
88
|
result = review;
|
|
@@ -173,13 +198,15 @@ async function buildOfflineEvidenceReviewWithLocalResources(value, generatedAt =
|
|
|
173
198
|
version: OFFLINE_EVIDENCE_REVIEW_VERSION,
|
|
174
199
|
generatedAt,
|
|
175
200
|
kind: 'geoip',
|
|
176
|
-
result: await reviewLocalMmdb(input, options.mmdbPath, generatedAt),
|
|
201
|
+
result: await reviewLocalMmdb(input, options.mmdbPath, generatedAt, options.signal ? { signal: options.signal } : {}),
|
|
177
202
|
limitations: Object.freeze([
|
|
178
203
|
'The review is local and uses only the supplied query metadata and analyst-supplied database. It does not transmit the address or refresh the database.',
|
|
179
204
|
]),
|
|
180
205
|
});
|
|
181
206
|
}
|
|
182
207
|
function formatOfflineEvidenceReview(document) {
|
|
208
|
+
if (document.kind === 'internal_containment')
|
|
209
|
+
return formatCaseContainmentHandoff(document.result);
|
|
183
210
|
const result = record(document.result);
|
|
184
211
|
if (document.kind === 'geoip' && result.schema === LOCAL_MMDB_REVIEW_SCHEMA && result.version !== LOCAL_MMDB_REVIEW_VERSION) {
|
|
185
212
|
throw new CliUsageError('Offline evidence review does not support this MMDB result version.');
|
|
@@ -204,7 +231,26 @@ function formatOfflineEvidenceReview(document) {
|
|
|
204
231
|
`Kind ${document.kind.replaceAll('_', ' ')}`,
|
|
205
232
|
`State ${state.replaceAll('_', ' ')}`,
|
|
206
233
|
];
|
|
207
|
-
if (
|
|
234
|
+
if (document.kind === 'infrastructure') {
|
|
235
|
+
const snapshot = record(result.snapshot), scope = record(snapshot.scope), coverage = record(snapshot.coverage);
|
|
236
|
+
lines.push(`Snapshot ${safeTerminalValue(snapshot.id, '')} · ${safeTerminalValue(snapshot.target, '')} · ${safeTerminalValue(snapshot.observedAt, '')}`, `Coverage ${safeTerminalValue(coverage.state, 'unknown')} · ${safeTerminalValue(coverage.detail, '')}`, `Selected hosts ${listLength(scope.hostnames)}`);
|
|
237
|
+
for (const item of Array.isArray(result.facts) ? result.facts : []) {
|
|
238
|
+
const row = record(item), source = record(row.source);
|
|
239
|
+
lines.push(`${safeTerminalValue(row.hostname, '')} · ${safeTerminalValue(row.family, '')}: ${safeTerminalValue(row.value, '')} · ${safeTerminalValue(source.name, '')} (${safeTerminalValue(source.evidenceClass, '')}) · ${safeTerminalValue(row.observedAt, '')}`);
|
|
240
|
+
}
|
|
241
|
+
lines.push('Wildcard patterns are not enumerated hosts. DNS failure, missing values and history do not establish current absence. No requests or watchlist changes were made.');
|
|
242
|
+
}
|
|
243
|
+
else if (document.kind === 'infrastructure_comparison') {
|
|
244
|
+
for (const item of Array.isArray(result.rows) ? result.rows : []) {
|
|
245
|
+
const row = record(item), source = record(row.source);
|
|
246
|
+
// Bound individual identities, not the combined label: a valid DNS query
|
|
247
|
+
// and owner may each occupy all 253 hostname characters.
|
|
248
|
+
const cohort = Object.entries(record(row.cohort)).map(([key, value]) => `${safeTerminalValue(key)}=${safeTerminalValue(value, '', 253)}`).join(' · ');
|
|
249
|
+
const times = (value) => Array.isArray(value) ? value.map(time => safeTerminalValue(time, '')).join(', ') : '';
|
|
250
|
+
lines.push(`${safeTerminalValue(row.hostname, '')} · ${safeTerminalValue(row.family, '')} · ${safeTerminalValue(source.name, '')} (${safeTerminalValue(source.evidenceClass, '')}) · ${safeTerminalValue(row.state, '')}`, ` Cohort: ${cohort}`, ` Earlier observations: ${times(row.beforeTimes)}; later observations: ${times(row.afterTimes)}`, ` ${safeTerminalValue(JSON.stringify(row.before), '')} → ${safeTerminalValue(JSON.stringify(row.after), '')}`, ` ${safeTerminalValue(row.detail, '')}`);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
else if (CONTEXT_REVIEW_KINDS.includes(document.kind)) {
|
|
208
254
|
lines.push(safeTerminalValue(result.summary, ''));
|
|
209
255
|
for (const item of Array.isArray(result.observations) ? result.observations : []) {
|
|
210
256
|
const row = record(item);
|
package/cli/review-arguments.mjs
CHANGED
|
@@ -45,10 +45,10 @@ function parseCaseArguments(parsed) {
|
|
|
45
45
|
if (operation === 'note' ? text === null && !noteSource : text !== null || noteSource !== null) {
|
|
46
46
|
throw new CliUsageError('Only case note accepts and requires --text or --note-file.');
|
|
47
47
|
}
|
|
48
|
-
if (['pin', 'link', 'withdraw-link', 'assess', 'recheck'].includes(operation)
|
|
48
|
+
if (['pin', 'link', 'withdraw-link', 'assess', 'recheck', 'incident-link', 'action', 'action-event', 'recheck-question', 'close-object'].includes(operation)
|
|
49
49
|
? !inputSource
|
|
50
50
|
: inputSource !== null) {
|
|
51
|
-
throw new CliUsageError('
|
|
51
|
+
throw new CliUsageError('This Case mutation requires --input; show, open and note do not accept it.');
|
|
52
52
|
}
|
|
53
53
|
if (expectedFileDigest !== null && (!source || !/^sha256:[a-f0-9]{64}$/u.test(expectedFileDigest))) {
|
|
54
54
|
throw new CliUsageError('--expect-file-digest requires a source file and sha256:<64 lowercase hexadecimal characters>.');
|
|
@@ -95,6 +95,7 @@ export const REVIEW_ARGUMENT_PARSERS = Object.freeze({
|
|
|
95
95
|
action: 'intake',
|
|
96
96
|
kind: parsed.positionalValue('kind'),
|
|
97
97
|
reportedActions: parsed.optionValues('--reported-action'),
|
|
98
|
+
...(parsed.hasOption('--intake-context') ? { intakeContextSource: parsed.optionValue('--intake-context') } : {}),
|
|
98
99
|
...(parsed.hasOption('--trusted-auth-header')
|
|
99
100
|
? { trustedAuthHeaders: parsed.optionValues('--trusted-auth-header') }
|
|
100
101
|
: {}),
|
|
@@ -2,12 +2,14 @@ import { CASE_SCHEMA_VERSION, CLI_CASE_PACK_INPUT_CASE_VERSIONS, MAX_CASE_STORE_
|
|
|
2
2
|
import { MESSAGE_INTAKE_KINDS, MESSAGE_INTAKE_SCHEMA } from "../packages/contracts/message-intake.mjs";
|
|
3
3
|
import { IDENTITY_EVENTS_INPUT_SCHEMA } from "../packages/contracts/identity-events.mjs";
|
|
4
4
|
import { CONTEXT_INPUT_SCHEMAS, CONTEXT_REVIEW_SCHEMA } from "../packages/contracts/context-review.mjs";
|
|
5
|
+
import { CASE_CONTAINMENT_INPUT_SCHEMA, CASE_CONTAINMENT_HANDOFF_SCHEMA } from "../packages/cases/case-containment-handoff.mjs";
|
|
6
|
+
import { INTAKE_CONTEXT_SCHEMA } from "../packages/investigation/intake-context.mjs";
|
|
5
7
|
import { MANAGED_INDICATOR_SET_SCHEMA, MAX_MANAGED_INDICATOR_SET_BYTES, MAX_MANAGED_INDICATOR_PLAN_BYTES } from "../packages/contracts/analyst-interchange.mjs";
|
|
6
8
|
import { CLI_CASE_OPERATIONS, CLI_INDICATOR_OPERATIONS, positional, OPTIONAL_FILE_POSITIONAL, constraint, EMPTY_CONSTRAINTS, commandSeed } from "./command-definition.mjs";
|
|
7
9
|
export const REVIEW_COMMAND_DEFINITIONS = Object.freeze({
|
|
8
10
|
"verify-artifact": commandSeed({
|
|
9
11
|
reference: {
|
|
10
|
-
description: 'Validate a supported archive, ordinary
|
|
12
|
+
description: 'Validate a supported archive, ordinary case export, claim passport, packet, manifest, saved Lookup or Lookup-evidence export without printing evidence contents. Use --package for an evidence ZIP or encrypted package, with --passphrase-file to unlock it; use --folder ./evidence for an unencrypted evidence folder. Add --bagit with --package or --folder to verify BagIt 1.0.',
|
|
11
13
|
example: 'whoisleuth verify-artifact report.json --manifest manifest.json --manifest-entry artifact-2 --json --strict-exit',
|
|
12
14
|
boundary: 'Verification is offline and redacted. ZIP and folder entries are reported separately without importing them. Case exports are checked without repairing content; ordinary package review also counts original references with matching bytes. Ordinary folders allow only the declared layout; BagIt allows bounded nested payloads and checks SHA-256/SHA-512 manifests without interpreting payloads. Symbolic links are refused. BagIt fetch.txt is never fetched; missing files, mismatches and unsupported algorithms remain explicit. Package digests describe bytes, not filesystem metadata or authenticity. In scripts, use --strict-exit: incomplete verification returns 4. Unencrypted workspace archives always return 4 under --strict-exit because section digests do not cover mutable root metadata; structure-only formats can pass their narrower contract. Default exit 0 means the report was produced, not that its checks passed.',
|
|
13
15
|
},
|
|
@@ -54,9 +56,9 @@ export const REVIEW_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
54
56
|
}),
|
|
55
57
|
"source-report": commandSeed({
|
|
56
58
|
reference: {
|
|
57
|
-
description: '
|
|
59
|
+
description: 'Summarise source reliability from saved Lookup or Bulk evidence, or merge source-reliability reports.',
|
|
58
60
|
example: 'whoisleuth source-report lookup.json --json',
|
|
59
|
-
boundary: '
|
|
61
|
+
boundary: 'Accepts supported Lookup, Bulk or Bulk-item documents, or an unmixed set of source-reliability reports. Retains source states and timings but excludes targets, queries, endpoints and raw evidence.',
|
|
60
62
|
},
|
|
61
63
|
collection: { scope: 'Reads bounded saved evidence and emits target-free source reliability data.' },
|
|
62
64
|
summary: 'Build a target-free source report',
|
|
@@ -154,23 +156,23 @@ export const REVIEW_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
154
156
|
}),
|
|
155
157
|
intake: commandSeed({
|
|
156
158
|
reference: {
|
|
157
|
-
description: 'Review
|
|
159
|
+
description: 'Review selected messages, indicators, document links, QR destinations, HTTP archives or identity events locally. Plain text offers phone candidates; --intake-context adds explicit selections and source declarations.',
|
|
158
160
|
example: 'whoisleuth intake email message.eml --json',
|
|
159
|
-
boundary: 'Offline only: no destination, embedded command or QR payload is opened or executed. Output omits original bodies, subjects, address local parts, URL paths, queries and fragments. Authentication headers remain attributed claims. Selected PDF and DOCX review retains part identities and extraction coverage; embedded raster QR support does not imply complete rendered-page review. HAR retains minimised request order and timings, never replay authority, headers, cookies or bodies. Use --strict-exit to return 4 for partial coverage.',
|
|
161
|
+
boundary: 'Offline only: no destination, embedded command, phone number or QR payload is opened, contacted or executed. Output omits original bodies, subjects, address local parts, URL paths, queries and fragments. Literal IPs and labelled hashes are bounded text observations, not lookup targets or verdicts; headers, URL contents, QR, HAR and identity-event fields are not scanned for them. Plain-text phone discovery writes only counts, source digest, ranges and status to stderr; --quiet suppresses it. Optional --intake-context reads one separate file up to 8 KiB, not stdin. Version 1 carries distribution declarations; version 2 can also bind explicit phone selections and a supplied destination pair to the exact input digest. Selected phone strings and source declarations enter output; unselected candidates and exact private URLs do not. Authentication headers remain attributed claims. Selected PDF and DOCX review retains part identities and extraction coverage; embedded raster QR support does not imply complete rendered-page review. HAR retains minimised request order and timings, never replay authority, headers, cookies or bodies. Use --strict-exit to return 4 for partial coverage.',
|
|
160
162
|
},
|
|
161
|
-
collection: { scope: 'Reads one selected text, MIME email, calendar, PNG, PDF, DOCX, HAR or identity-event JSON file. No collection or automatic
|
|
163
|
+
collection: { scope: 'Reads one selected text, MIME email, calendar, PNG, PDF, DOCX, HAR or identity-event JSON file and, only with --intake-context, one bounded declaration/selection file. No collection or automatic case write.' },
|
|
162
164
|
summary: 'Review selected files and identity events offline',
|
|
163
|
-
options: ['--json', '--reported-action', '--trusted-auth-header', '--strict-exit', '--quiet', '--no-color'],
|
|
165
|
+
options: ['--json', '--reported-action', '--trusted-auth-header', '--intake-context', '--strict-exit', '--quiet', '--no-color'],
|
|
164
166
|
positionals: Object.freeze([positional('kind', 'enum', 1, 1, MESSAGE_INTAKE_KINDS), positional('source', 'file', 0, 1, [], 'argv_or_stdin')]),
|
|
165
167
|
constraints: EMPTY_CONSTRAINTS, handlerOwner: 'review', networkEffect: 'offline', common: true,
|
|
166
|
-
schemaIdentifiers: Object.freeze([MESSAGE_INTAKE_SCHEMA, IDENTITY_EVENTS_INPUT_SCHEMA]), primaryArtefacts: Object.freeze([]),
|
|
168
|
+
schemaIdentifiers: Object.freeze([MESSAGE_INTAKE_SCHEMA, INTAKE_CONTEXT_SCHEMA, IDENTITY_EVENTS_INPUT_SCHEMA]), primaryArtefacts: Object.freeze([]),
|
|
167
169
|
planSupport: false, additionalOutputFormats: Object.freeze([]), bootstrapProfile: 'allowed',
|
|
168
170
|
}),
|
|
169
171
|
"review-evidence": commandSeed({
|
|
170
172
|
reference: {
|
|
171
|
-
description: 'Review versioned protocol evidence, incident sequences, domain history, platform objects, storefronts
|
|
173
|
+
description: 'Review versioned protocol evidence, source-qualified infrastructure snapshots/comparisons, incident sequences, domain history, platform objects, storefronts, connector configuration or an explicitly selected internal containment handoff offline.',
|
|
172
174
|
example: 'whoisleuth review-evidence domain-change.json --json --strict-exit',
|
|
173
|
-
boundary: 'The command reads only the supplied document and, with --mmdb, one explicitly selected local database up to 512 MiB. MMDB parsing and lookup run in one disposable worker with a ten-second processing deadline and an 8 KiB result limit. Version-2 MMDB queries require a justified age policy; unavailable or partial current reviews return 4 under --strict-exit. Historical version-1 MMDB queries preserve their output without freshness admission. No database is bundled, downloaded, updated or transmitted. It performs no DNS, RDAP, BGP, GeoIP-provider, TLS, HTTP, certificate-authority, or SMTP request.',
|
|
175
|
+
boundary: 'The command reads only the supplied document and, with --mmdb, one explicitly selected local database up to 512 MiB. MMDB parsing and lookup run in one disposable worker with a ten-second processing deadline and an 8 KiB result limit. Version-2 MMDB queries require a justified age policy; unavailable or partial current reviews return 4 under --strict-exit. Historical version-1 MMDB queries preserve their output without freshness admission. Domain-history v2 accepts explicit source-linked registration boundaries; v1 retains its original semantics. Internal containment selects retained next-step assertions and supporting pins after explicit audience-disclosure review; it does not change their state or execute controls. No database is bundled, downloaded, updated or transmitted. It performs no DNS, RDAP, BGP, GeoIP-provider, TLS, HTTP, certificate-authority, or SMTP request.',
|
|
174
176
|
},
|
|
175
177
|
collection: { scope: 'Reads one bounded versioned evidence or request-planning document and, only with --mmdb, one selected bounded local database. Performs no collection.' },
|
|
176
178
|
summary: 'Review supplied evidence offline',
|
|
@@ -182,6 +184,8 @@ export const REVIEW_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
182
184
|
common: true,
|
|
183
185
|
schemaIdentifiers: Object.freeze([
|
|
184
186
|
CONTEXT_REVIEW_SCHEMA,
|
|
187
|
+
CASE_CONTAINMENT_INPUT_SCHEMA,
|
|
188
|
+
CASE_CONTAINMENT_HANDOFF_SCHEMA,
|
|
185
189
|
...CONTEXT_INPUT_SCHEMAS,
|
|
186
190
|
'whoisleuth.cli.offline-evidence-review',
|
|
187
191
|
'whoisleuth.rdap-search-input',
|
|
@@ -192,6 +196,8 @@ export const REVIEW_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
192
196
|
'whoisleuth.local-mmdb-query',
|
|
193
197
|
'whoisleuth.local-mmdb-review',
|
|
194
198
|
'whoisleuth.encrypted-dns-plan-input',
|
|
199
|
+
'whoisleuth.infrastructure-observation',
|
|
200
|
+
'whoisleuth.infrastructure-comparison.input',
|
|
195
201
|
]),
|
|
196
202
|
primaryArtefacts: Object.freeze([]),
|
|
197
203
|
planSupport: false,
|
|
@@ -220,12 +226,12 @@ export const REVIEW_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
220
226
|
}),
|
|
221
227
|
case: commandSeed({
|
|
222
228
|
reference: {
|
|
223
|
-
description: 'Show or open a local
|
|
229
|
+
description: 'Show or open a local case, append a note or evidence pin, link evidence, record an assessment or offline recheck, retain an incident-link, create an action, append an action-event, save a recheck-question, or close-object. JSON mutations use --input; notes use --text or --note-file. Link input contains fromPinId, toPinId, kind and basis. Action-event input contains id and transition. Explicit responseObjects bind an action; a typed objectOutcome requires an explicit affected subset on its event. Object closure never closes other objects or the case. Mutations require --output and write the complete current case export.',
|
|
224
230
|
example: 'whoisleuth case open --domain example.test --output cases.json\n whoisleuth case show cases.json\n whoisleuth case note cases.json --text "Review the retained observation" --output cases.json --force',
|
|
225
|
-
boundary: 'No database, browser launch, request or external report is created. Select --case-id when a file contains multiple
|
|
231
|
+
boundary: 'No database, browser launch, request or external report is created. Select --case-id when a file contains multiple cases. Existing files require --force; --expect-file-digest sha256:<digest> additionally checks the exact file reviewed earlier. Source and output leases reject concurrent changes. Interrupted .workflow.lock files require deliberate inspection. Recheck records supplied observations; it does not collect them. Not reproduced requires an existing saved question, a complete observation and comparable conditions. Working exports include private analyst content and file references, not attached file bytes.',
|
|
226
232
|
},
|
|
227
|
-
collection: { scope: `Reads exact
|
|
228
|
-
summary: 'Review and update ordinary local
|
|
233
|
+
collection: { scope: `Reads exact case schemas ${CLI_CASE_PACK_INPUT_CASE_VERSIONS.join(' or ')}. Input is bounded to ${MAX_EDITABLE_CASE_INPUT_BYTES / 1024 / 1024} MiB including formatting; the complete canonical case store must fit ${MAX_CASE_STORE_BYTES / 1024 / 1024} MiB without pruning. Writes current schema ${CASE_SCHEMA_VERSION}.` },
|
|
234
|
+
summary: 'Review and update ordinary local case files',
|
|
229
235
|
options: ['--case-id', '--domain', '--title', '--new-incident', '--text', '--note-file', '--input', '--expect-file-digest', '--json', '--no-color'],
|
|
230
236
|
positionals: Object.freeze([positional('operation', 'enum', 1, 1, CLI_CASE_OPERATIONS), positional('source', 'file', 0, 1)]),
|
|
231
237
|
constraints: Object.freeze([constraint({ kind: 'mutually_exclusive', options: ['--text', '--note-file'] })]),
|
|
@@ -250,11 +256,11 @@ export const REVIEW_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
250
256
|
}),
|
|
251
257
|
"case-pack": commandSeed({
|
|
252
258
|
reference: {
|
|
253
|
-
description: `Package browser-created
|
|
259
|
+
description: `Package browser-created case records from schemas ${CLI_CASE_PACK_INPUT_CASE_VERSIONS.join(' or ')} as a reviewed, audience-specific Case-pack v2 with current schema ${CASE_SCHEMA_VERSION}.`,
|
|
254
260
|
example: 'whoisleuth case-pack cases.json --audience trusted --reviewed --json',
|
|
255
|
-
boundary: 'The command creates a separate offline package without changing the source or durable
|
|
261
|
+
boundary: 'The command creates a separate offline package without changing the source or durable cases and requires an explicit review acknowledgement. Every audience, including public, retains domain evidence, case, pin and decision identifiers, tags, decision summaries and rationale. Review that analyst-authored content as well as the exclusions before sharing.',
|
|
256
262
|
},
|
|
257
|
-
collection: { scope: `Reads one bounded
|
|
263
|
+
collection: { scope: `Reads one bounded case export from schemas ${CLI_CASE_PACK_INPUT_CASE_VERSIONS.join(' or ')} and writes a separate audience-specific Case-pack v2.` },
|
|
258
264
|
summary: 'Build a reviewed case package',
|
|
259
265
|
options: ['--audience', '--reviewed', '--json', '--quiet', '--no-color'],
|
|
260
266
|
positionals: OPTIONAL_FILE_POSITIONAL,
|
|
@@ -192,8 +192,9 @@ async function runOfflineEvidenceReviewCommand(args, dependencies, context) {
|
|
|
192
192
|
? await dependencies.readArtifactInput(args.source)
|
|
193
193
|
: await context.readInput(args.source, MAX_OFFLINE_EVIDENCE_INPUT_BYTES, 'Offline evidence input'), 'offline evidence input', 'review-evidence requires one JSON file or a document on stdin.');
|
|
194
194
|
const document = args.mmdbSource
|
|
195
|
-
? await buildOfflineEvidenceReviewWithLocalResources(input, context.now(), { mmdbPath: args.mmdbSource })
|
|
195
|
+
? await buildOfflineEvidenceReviewWithLocalResources(input, context.now(), { mmdbPath: args.mmdbSource, ...(dependencies.signal ? { signal: dependencies.signal } : {}) })
|
|
196
196
|
: buildOfflineEvidenceReview(input, context.now());
|
|
197
|
+
dependencies.signal?.throwIfAborted();
|
|
197
198
|
writeReviewReport(context, args, document, formatOfflineEvidenceReview);
|
|
198
199
|
if (args.strictExit) {
|
|
199
200
|
const result = document.result && typeof document.result === 'object' && !Array.isArray(document.result)
|
|
@@ -205,9 +206,11 @@ async function runOfflineEvidenceReviewCommand(args, dependencies, context) {
|
|
|
205
206
|
const zoneMismatch = document.kind === 'zone_intent' && (result.complete !== true
|
|
206
207
|
|| (result.counts && typeof result.counts === 'object' && !Array.isArray(result.counts)
|
|
207
208
|
&& ['different', 'missing', 'unexpected', 'incomplete'].some((key) => Number(result.counts[key]) > 0)));
|
|
208
|
-
const contextPartial = CONTEXT_REVIEW_KINDS.includes(document.kind) && result.state === 'partial';
|
|
209
|
+
const contextPartial = (CONTEXT_REVIEW_KINDS.includes(document.kind) || document.kind === 'internal_containment') && result.state === 'partial';
|
|
209
210
|
const mmdbIncomplete = result.schema === LOCAL_MMDB_REVIEW_SCHEMA && result.version === LOCAL_MMDB_REVIEW_VERSION && result.completeness !== 'complete';
|
|
210
|
-
|
|
211
|
+
const infrastructureIncomplete = document.kind === 'infrastructure' ? result.state !== 'complete'
|
|
212
|
+
: document.kind === 'infrastructure_comparison' && result.state !== 'compared';
|
|
213
|
+
if (gate?.pass === false || zoneMismatch || contextPartial || mmdbIncomplete || infrastructureIncomplete)
|
|
211
214
|
return EXIT_CODES.PARTIAL_FAILURE;
|
|
212
215
|
}
|
|
213
216
|
return EXIT_CODES.SUCCESS;
|
package/cli/risk-calibration.mjs
CHANGED
|
@@ -5,7 +5,7 @@ import { scanBoundedJson } from "../packages/analysis/bounded-json.mjs";
|
|
|
5
5
|
import { canonicalRegistrableDomain } from "../packages/analysis/registrable-domain.mjs";
|
|
6
6
|
import { ANALYST_REVIEW_REASON_VALUES, analystInteroperabilityTags } from "../packages/analysis/analyst-taxonomy.mjs";
|
|
7
7
|
import { buildRiskCalibrationSummaryReport, parseRiskCalibrationSummaryReport, RISK_CALIBRATION_SUMMARY_INTERPRETATION, RISK_CALIBRATION_SUMMARY_THRESHOLDS, } from "../packages/analysis/risk-calibration-summary.mjs";
|
|
8
|
-
import { MAX_RISK_CALIBRATION_ACTIVITY_LENGTH, MAX_RISK_CALIBRATION_AVAILABILITY_LENGTH, MAX_RISK_CALIBRATION_DISPOSITION_LENGTH, MAX_RISK_CALIBRATION_DOMAIN_AGE_DAYS, MAX_RISK_CALIBRATION_DOMAIN_LENGTH, MAX_RISK_CALIBRATION_FINDING_CATEGORY_LENGTH, MAX_RISK_CALIBRATION_FINDINGS_PER_PROVIDER, MAX_RISK_CALIBRATION_INPUT_BYTES, MAX_RISK_CALIBRATION_JSON_CONTAINER_ITEMS, MAX_RISK_CALIBRATION_JSON_DEPTH, MAX_RISK_CALIBRATION_JSON_KEYS, MAX_RISK_CALIBRATION_JSON_VALUES, MAX_RISK_CALIBRATION_MUTATIONS, MAX_RISK_CALIBRATION_PROVIDER_ID_LENGTH, MAX_RISK_CALIBRATION_PROVIDER_STATE_LENGTH, MAX_RISK_CALIBRATION_PROVIDERS, MAX_RISK_CALIBRATION_RECORD_ID_LENGTH, MAX_RISK_CALIBRATION_RECORDS, MAX_RISK_CALIBRATION_REVIEW_REASON_LENGTH, MAX_RISK_CALIBRATION_SCAN_DEPTH_LENGTH, MAX_RISK_CALIBRATION_STRING_LENGTH, MAX_RISK_CALIBRATION_TIMESTAMP_LENGTH, RISK_CALIBRATION_DATASET_SCHEMA,
|
|
8
|
+
import { MAX_RISK_CALIBRATION_ACTIVITY_LENGTH, MAX_RISK_CALIBRATION_AVAILABILITY_LENGTH, MAX_RISK_CALIBRATION_DISPOSITION_LENGTH, MAX_RISK_CALIBRATION_DOMAIN_AGE_DAYS, MAX_RISK_CALIBRATION_DOMAIN_LENGTH, MAX_RISK_CALIBRATION_FINDING_CATEGORY_LENGTH, MAX_RISK_CALIBRATION_FINDINGS_PER_PROVIDER, MAX_RISK_CALIBRATION_INPUT_BYTES, MAX_RISK_CALIBRATION_JSON_CONTAINER_ITEMS, MAX_RISK_CALIBRATION_JSON_DEPTH, MAX_RISK_CALIBRATION_JSON_KEYS, MAX_RISK_CALIBRATION_JSON_VALUES, MAX_RISK_CALIBRATION_MUTATIONS, MAX_RISK_CALIBRATION_PROVIDER_ID_LENGTH, MAX_RISK_CALIBRATION_PROVIDER_STATE_LENGTH, MAX_RISK_CALIBRATION_PROVIDERS, MAX_RISK_CALIBRATION_RECORD_ID_LENGTH, MAX_RISK_CALIBRATION_RECORDS, MAX_RISK_CALIBRATION_REVIEW_REASON_LENGTH, MAX_RISK_CALIBRATION_SCAN_DEPTH_LENGTH, MAX_RISK_CALIBRATION_STRING_LENGTH, MAX_RISK_CALIBRATION_TIMESTAMP_LENGTH, RISK_CALIBRATION_DATASET_SCHEMA, RISK_CALIBRATION_REPORT_SCHEMA, RISK_CALIBRATION_REPORT_VERSION, RISK_CALIBRATION_DISPOSITIONS, serializeRiskCalibrationSnapshot, snapshotRiskCalibrationReportForSerialization, SUPPORTED_RISK_CALIBRATION_DATASET_VERSIONS, } from "../packages/contracts/risk-calibration.mjs";
|
|
9
9
|
import { CliUsageError } from "./errors.mjs";
|
|
10
10
|
import { RISK_MUTATION_TYPES } from "../packages/analysis/risk-scoring.mjs";
|
|
11
11
|
import { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_ENVELOPE_VERSION, THREAT_INTELLIGENCE_SCHEMA, } from "../packages/analysis/threat-intelligence-types.mjs";
|
|
@@ -17,7 +17,7 @@ const AVAILABILITY_STATES = new Set(['registered', 'for_sale', 'expiring', 'avai
|
|
|
17
17
|
const ACTIVITY_STATES = new Set(['active', 'parked', 'unreachable', 'no_site']);
|
|
18
18
|
const BOOLEAN_FIELDS = [
|
|
19
19
|
'faviconMatch', 'faviconNearMatch', 'reusesOfficialAssets', 'hasPasswordField',
|
|
20
|
-
'hasMx', 'hasSpf', 'hasDmarc', 'privacyProtected', 'hasExternalFormAction',
|
|
20
|
+
'hasMx', 'hasSpf', 'hasDmarc', 'privacyProtected', 'hasExternalFormAction', 'hasExternalPasswordForm',
|
|
21
21
|
'idnReferenceMatch', 'pageBaselineMatch', 'hasActiveBrandProfile',
|
|
22
22
|
];
|
|
23
23
|
const REVIEW_REASON_CODES = ANALYST_REVIEW_REASON_VALUES;
|
|
@@ -95,8 +95,10 @@ function projectThreatIntelligence(value, field) {
|
|
|
95
95
|
}),
|
|
96
96
|
};
|
|
97
97
|
}
|
|
98
|
-
function projectEvidence(value, field) {
|
|
98
|
+
function projectEvidence(value, field, version) {
|
|
99
99
|
const source = object(value, field);
|
|
100
|
+
if (version < 3 && Object.hasOwn(source, 'hasExternalPasswordForm'))
|
|
101
|
+
throw new CliUsageError(`${field}.hasExternalPasswordForm requires dataset version 3; historical inputs were not reinterpreted.`);
|
|
100
102
|
const availability = boundedString(source.availability ?? source.state, `${field}.availability`, MAX_RISK_CALIBRATION_AVAILABILITY_LENGTH);
|
|
101
103
|
if (!AVAILABILITY_STATES.has(availability))
|
|
102
104
|
throw new CliUsageError(`${field}.availability is unsupported.`);
|
|
@@ -234,7 +236,7 @@ export function parseRiskCalibrationDataset(text) {
|
|
|
234
236
|
domain,
|
|
235
237
|
analystDisposition,
|
|
236
238
|
...(reviewReasonCode ? { reviewReasonCode } : {}),
|
|
237
|
-
evidence: projectEvidence(record.evidence, `${prefix}.evidence
|
|
239
|
+
evidence: projectEvidence(record.evidence, `${prefix}.evidence`, documentVersion),
|
|
238
240
|
};
|
|
239
241
|
});
|
|
240
242
|
return {
|
package/cli/saved-lookup.mjs
CHANGED
|
@@ -7,7 +7,8 @@ import { CliUsageError } from "./errors.mjs";
|
|
|
7
7
|
import { validHttpDeliveryMetadata, validPagePublicationMetadata, } from "../lib/homepage-metadata-contract.mjs";
|
|
8
8
|
import { registrarStandingObservedBy, resolveRegistrarIanaId, validRegistrarStanding, } from "../lib/registrar-standing-contract.mjs";
|
|
9
9
|
import { normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
|
|
10
|
-
import {
|
|
10
|
+
import { parseLookupCollectionReceipt } from "../packages/evidence/lookup-collection-receipt.mjs";
|
|
11
|
+
import { CLI_LOOKUP_SCHEMA as SAVED_LOOKUP_SCHEMA, CLI_LOOKUP_VERSION as SAVED_LOOKUP_SCHEMA_VERSION, PUBLIC_CLI_LOOKUP_VERSION as PUBLIC_SAVED_LOOKUP_SCHEMA_VERSION, HOMEPAGE_CLI_LOOKUP_VERSION, COLLECTION_RECEIPT_CLI_LOOKUP_VERSION, MAX_CLI_LOOKUP_BYTES as MAX_SAVED_LOOKUP_INPUT_BYTES, MAX_CLI_LOOKUP_JSON_CONTAINER_ITEMS as MAX_BOUNDED_JSON_CONTAINER_ITEMS, MAX_CLI_LOOKUP_JSON_DEPTH as MAX_BOUNDED_JSON_DEPTH, MAX_CLI_LOOKUP_JSON_KEYS as MAX_BOUNDED_JSON_KEYS, MAX_CLI_LOOKUP_JSON_VALUES as MAX_BOUNDED_JSON_VALUES, MAX_CLI_LOOKUP_STRING_LENGTH as MAX_SAVED_LOOKUP_STRING_LENGTH, SUPPORTED_CLI_LOOKUP_VERSIONS as SUPPORTED_SAVED_LOOKUP_SCHEMA_VERSIONS, } from "../packages/contracts/cli-lookup.mjs";
|
|
11
12
|
const RDAP_STATUSES = new Set([
|
|
12
13
|
'success', 'complete', 'partial', 'error', 'unsupported', 'not_found', 'skipped', 'disabled',
|
|
13
14
|
'unavailable', 'rate_limited', 'not_applicable', 'stale',
|
|
@@ -317,23 +318,31 @@ function assertLookupEnvelope(document, label) {
|
|
|
317
318
|
const publicationMetadata = pageIdentity?.publicationMetadata;
|
|
318
319
|
const deliveryMetadata = httpResponse?.deliveryMetadata;
|
|
319
320
|
const registrarStanding = document.registrarStanding;
|
|
321
|
+
if (document.collectionReceipt !== undefined) {
|
|
322
|
+
if (Number(document.version) < COLLECTION_RECEIPT_CLI_LOOKUP_VERSION)
|
|
323
|
+
throw new TypeError(`${label} collection receipt requires saved Lookup version ${COLLECTION_RECEIPT_CLI_LOOKUP_VERSION}.`);
|
|
324
|
+
const receipt = parseLookupCollectionReceipt(document.collectionReceipt);
|
|
325
|
+
if (receipt.targetClass !== document.type || receipt.requestedDepth !== document.mode) {
|
|
326
|
+
throw new TypeError(`${label} collection receipt does not match the Lookup.`);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
320
329
|
if (document.version === PUBLIC_SAVED_LOOKUP_SCHEMA_VERSION
|
|
321
330
|
&& (publicationMetadata !== undefined || deliveryMetadata !== undefined || registrarStanding !== undefined)) {
|
|
322
|
-
throw new TypeError(`${label} version ${PUBLIC_SAVED_LOOKUP_SCHEMA_VERSION} cannot contain version ${
|
|
331
|
+
throw new TypeError(`${label} version ${PUBLIC_SAVED_LOOKUP_SCHEMA_VERSION} cannot contain version ${HOMEPAGE_CLI_LOOKUP_VERSION} evidence fields.`);
|
|
323
332
|
}
|
|
324
|
-
if (document.version
|
|
333
|
+
if (document.version >= HOMEPAGE_CLI_LOOKUP_VERSION
|
|
325
334
|
&& (publicationMetadata !== undefined && (!['success', 'partial'].includes(String(pageIdentity?.status))
|
|
326
335
|
|| !validPagePublicationMetadata(publicationMetadata))
|
|
327
336
|
|| deliveryMetadata !== undefined && (!['success', 'partial'].includes(String(http?.status))
|
|
328
337
|
|| !validHttpDeliveryMetadata(deliveryMetadata)))) {
|
|
329
338
|
throw new TypeError(`${label} contains invalid homepage metadata.`);
|
|
330
339
|
}
|
|
331
|
-
if (document.version
|
|
340
|
+
if (document.version >= HOMEPAGE_CLI_LOOKUP_VERSION
|
|
332
341
|
&& registrarStanding !== undefined
|
|
333
342
|
&& !validRegistrarStanding(registrarStanding)) {
|
|
334
343
|
throw new TypeError(`${label} contains invalid registrar standing.`);
|
|
335
344
|
}
|
|
336
|
-
if (document.version
|
|
345
|
+
if (document.version >= HOMEPAGE_CLI_LOOKUP_VERSION
|
|
337
346
|
&& registrarStanding !== undefined
|
|
338
347
|
&& !registrarStandingObservedBy(registrarStanding, generatedAt)) {
|
|
339
348
|
throw new TypeError(`${label} registrar standing was observed after the saved Lookup time.`);
|
|
@@ -346,7 +355,7 @@ function assertLookupEnvelope(document, label) {
|
|
|
346
355
|
if ((whoisStatus === 'complete' || whoisStatus === 'partial') && !whoisParsed) {
|
|
347
356
|
throw new TypeError('Successful WHOIS input is missing normalised parsed data.');
|
|
348
357
|
}
|
|
349
|
-
if (document.version
|
|
358
|
+
if (document.version >= HOMEPAGE_CLI_LOOKUP_VERSION
|
|
350
359
|
&& validRegistrarStanding(registrarStanding)
|
|
351
360
|
&& registrarStanding.ianaId !== resolveRegistrarIanaId(rdapParsed, whoisParsed)) {
|
|
352
361
|
throw new TypeError(`${label} registrar standing does not match its retained registration sources.`);
|
|
@@ -430,7 +439,7 @@ function buildCliLookupDocument(query, classified, result, generatedAt = new Dat
|
|
|
430
439
|
throw new TypeError('Lookup document generation time must be valid and include an explicit timezone.');
|
|
431
440
|
}
|
|
432
441
|
const classifiedSnapshot = snapshotKnownDataFields(classified, 'Lookup classification', new Set(['type', 'value', 'inputHostname', 'registrableDomain', 'isSubdomain']));
|
|
433
|
-
const contextSnapshot = snapshotKnownDataFields(collectionContext, 'Lookup collection context', new Set(['observerLabel', 'vantageLabel']));
|
|
442
|
+
const contextSnapshot = snapshotKnownDataFields(collectionContext, 'Lookup collection context', new Set(['observerLabel', 'vantageLabel', 'collectionReceipt']));
|
|
434
443
|
const observerLabel = typeof contextSnapshot.observerLabel === 'string'
|
|
435
444
|
? contextSnapshot.observerLabel
|
|
436
445
|
: null;
|
|
@@ -439,6 +448,12 @@ function buildCliLookupDocument(query, classified, result, generatedAt = new Dat
|
|
|
439
448
|
: null;
|
|
440
449
|
const resultSnapshot = snapshotLookupResultRoot(result);
|
|
441
450
|
delete resultSnapshot.collectionContext;
|
|
451
|
+
if (Object.hasOwn(contextSnapshot, 'collectionReceipt')) {
|
|
452
|
+
delete resultSnapshot.collectionReceipt;
|
|
453
|
+
const receipt = parseLookupCollectionReceipt(contextSnapshot.collectionReceipt);
|
|
454
|
+
if (receipt)
|
|
455
|
+
resultSnapshot.collectionReceipt = receipt;
|
|
456
|
+
}
|
|
442
457
|
const classifiedType = classifiedSnapshot.type;
|
|
443
458
|
if (typeof classifiedType !== 'string' || !QUERY_TYPES.has(classifiedType)) {
|
|
444
459
|
throw new TypeError('Lookup classification has an unsupported query type.');
|
|
@@ -3,6 +3,8 @@ import { reviewSelectedInput } from "../packages/investigation/selected-input-re
|
|
|
3
3
|
import { assertMessageBytes } from "../packages/investigation/intake-report.mjs";
|
|
4
4
|
import { DOCUMENT_REVIEW_DEADLINE_MS } from "../packages/contracts/document-intake.mjs";
|
|
5
5
|
import { MESSAGE_INTAKE_SCHEMA, MESSAGE_INTAKE_VERSION } from "../packages/contracts/message-intake.mjs";
|
|
6
|
+
import { validateIntakeExtensions } from "../packages/investigation/intake-context.mjs";
|
|
7
|
+
import { validatePhoneReview } from "../packages/investigation/intake-phones.mjs";
|
|
6
8
|
if (!isMainThread && parentPort) {
|
|
7
9
|
const port = parentPort, request = workerData;
|
|
8
10
|
try {
|
|
@@ -45,8 +47,16 @@ export async function reviewSelectedInputInWorker(bytes, kind, reviewedAt, signa
|
|
|
45
47
|
const abort = () => finish(undefined, new Error('Selected input review was cancelled. Nothing was saved.'));
|
|
46
48
|
const timer = setTimeout(() => finish(undefined, new Error('Selected input review exceeded its processing deadline. Nothing was saved.')), DOCUMENT_REVIEW_DEADLINE_MS);
|
|
47
49
|
worker.once('message', (reply) => {
|
|
48
|
-
if (reply && typeof reply === 'object' && 'result' in reply && reply.result?.report.schema === MESSAGE_INTAKE_SCHEMA && reply.result.report.schemaVersion === MESSAGE_INTAKE_VERSION)
|
|
49
|
-
|
|
50
|
+
if (reply && typeof reply === 'object' && 'result' in reply && reply.result?.report.schema === MESSAGE_INTAKE_SCHEMA && reply.result.report.schemaVersion === MESSAGE_INTAKE_VERSION) {
|
|
51
|
+
try {
|
|
52
|
+
validateIntakeExtensions(reply.result.report);
|
|
53
|
+
validatePhoneReview(reply.result);
|
|
54
|
+
finish(reply.result);
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
finish(undefined, new Error('Selected input returned an unsupported review. Nothing was retained.'));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
50
60
|
else
|
|
51
61
|
finish();
|
|
52
62
|
});
|
|
@@ -730,6 +730,8 @@ export function buildSourceReliabilityReport(raw, generatedAt = new Date().toISO
|
|
|
730
730
|
});
|
|
731
731
|
}
|
|
732
732
|
export function formatSourceReliabilityReport(report) {
|
|
733
|
+
const duration = (value) => value
|
|
734
|
+
? `median ${value.median} ms / p95 ${value.p95} ms (range ${value.minimum}-${value.maximum} ms)` : 'unmeasured';
|
|
733
735
|
const lines = [
|
|
734
736
|
'WHOISleuth source reliability report',
|
|
735
737
|
`Documents: ${report.documentsReviewed}`,
|
|
@@ -741,16 +743,16 @@ export function formatSourceReliabilityReport(report) {
|
|
|
741
743
|
`Exceptions: ${report.totals.truncations} truncated · ${report.totals.rateLimits} rate-limited`,
|
|
742
744
|
];
|
|
743
745
|
for (const source of report.sources) {
|
|
744
|
-
const states = Object.entries(source.states).map(([state, count]) => `${state} ${count}`).join(', ') || '
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
}
|
|
753
|
-
lines.push('Privacy: targets 0 · queries 0 · raw evidence 0');
|
|
746
|
+
const states = Object.entries(source.states).map(([state, count]) => `${state} ${count}`).join(', ') || 'no states retained';
|
|
747
|
+
lines.push('', source.source, ` Collection states: ${states}`, ` Samples: states ${source.samples.states} · observations ${source.samples.observations} · Lookup timings ${source.samples.timing}`, ` Exceptions: ${source.truncationCount} truncated · ${source.rateLimitedCount} rate-limited`, ` Rates (0–1): failure ${source.rates.failure ?? 'unmeasured'} · partial ${source.rates.partial ?? 'unmeasured'}`, ` Observation duration: ${duration(source.durationMs.observation)}`, ` Lookup timing: ${duration(source.durationMs.lookupTiming)}`);
|
|
748
|
+
if (source.durationTrend.reportObservationMedian)
|
|
749
|
+
lines.push(` Merged observation-median distribution: ${duration(source.durationTrend.reportObservationMedian)}`);
|
|
750
|
+
if (source.durationTrend.reportTimingMedian)
|
|
751
|
+
lines.push(` Merged Lookup-median distribution: ${duration(source.durationTrend.reportTimingMedian)}`);
|
|
752
|
+
if (source.durationTimeline.length)
|
|
753
|
+
lines.push(` Retained report timeline: ${source.durationTimeline.length} points`);
|
|
754
|
+
}
|
|
755
|
+
lines.push('', 'Privacy: targets 0 · queries 0 · raw evidence 0');
|
|
754
756
|
for (const limitation of report.limitations)
|
|
755
757
|
lines.push(`Limitation: ${limitation}`);
|
|
756
758
|
return `${lines.join('\n')}\n`;
|
|
@@ -9,7 +9,7 @@ export const SUPPORT_ARGUMENT_PARSERS = Object.freeze({
|
|
|
9
9
|
doctor: (parsed) => ({
|
|
10
10
|
action: 'doctor',
|
|
11
11
|
network: parsed.hasOption('--network'),
|
|
12
|
-
output: jsonOutput(parsed),
|
|
12
|
+
output: parsed.hasOption('--summary-json') ? 'summary_json' : jsonOutput(parsed),
|
|
13
13
|
...terminalOptions(parsed),
|
|
14
14
|
}),
|
|
15
15
|
commands: (parsed) => ({
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { RISK_CALIBRATION_DATASET_SCHEMA, RISK_CALIBRATION_REPORT_SCHEMA } from "../packages/contracts/risk-calibration.mjs";
|
|
2
|
+
import { DOCTOR_SCHEMA, DOCTOR_SUMMARY_SCHEMA } from "../packages/contracts/cli-doctor.mjs";
|
|
2
3
|
import { positional, NO_POSITIONALS, OPTIONAL_FILE_POSITIONAL, constraint, EMPTY_CONSTRAINTS, commandSeed, text } from "./command-definition.mjs";
|
|
3
4
|
export const SUPPORT_COMMAND_DEFINITIONS = Object.freeze({
|
|
4
5
|
completion: commandSeed({
|
|
@@ -25,17 +26,17 @@ export const SUPPORT_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
25
26
|
reference: {
|
|
26
27
|
description: 'Check the supported runtime and local terminal capabilities.',
|
|
27
28
|
example: 'whoisleuth doctor --json',
|
|
28
|
-
boundary: 'The default check is offline. Public DNS and port 43 checks run only
|
|
29
|
+
boundary: 'The default check is offline. Public DNS, HTTPS and port 43 checks run only with --network. --summary-json omits private context and free-form diagnostic details.',
|
|
29
30
|
},
|
|
30
31
|
collection: { scope: 'Network access is opt-in with --network and is limited to fixed public DNS, HTTPS, and WHOIS diagnostics.' },
|
|
31
32
|
summary: 'Check the local CLI runtime',
|
|
32
|
-
options: ['--network', '--json', '--quiet', '--no-color'],
|
|
33
|
+
options: ['--network', '--json', '--summary-json', '--quiet', '--no-color'],
|
|
33
34
|
positionals: NO_POSITIONALS,
|
|
34
|
-
constraints:
|
|
35
|
+
constraints: Object.freeze([constraint({ kind: 'mutually_exclusive', options: ['--json', '--summary-json'] })]),
|
|
35
36
|
handlerOwner: 'support',
|
|
36
37
|
networkEffect: 'conditional_network',
|
|
37
38
|
common: true,
|
|
38
|
-
schemaIdentifiers: Object.freeze([
|
|
39
|
+
schemaIdentifiers: Object.freeze([DOCTOR_SCHEMA, DOCTOR_SUMMARY_SCHEMA]),
|
|
39
40
|
primaryArtefacts: Object.freeze([]),
|
|
40
41
|
planSupport: false,
|
|
41
42
|
additionalOutputFormats: Object.freeze([]),
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { REGISTRY_CAPABILITIES_VERSION, registryCapabilityFor } from "../lib/registry-capabilities.mjs";
|
|
2
2
|
import { buildRiskCalibrationSummaryReport } from "../packages/analysis/risk-calibration-summary.mjs";
|
|
3
|
-
import { explainRiskScore,
|
|
3
|
+
import { explainRiskScore, explainRiskScoreV8, RISK_MODEL_VERSION, RISK_REVIEW_THRESHOLD } from "../packages/analysis/risk-scoring.mjs";
|
|
4
4
|
import { buildCliCommandCatalogue, formatCliCommandCatalogue, selectCliCommands } from "./command-catalogue.mjs";
|
|
5
5
|
import { CLI_COMMAND_REGISTRY, CLI_COMMANDS, COMMAND_COLLECTION, COMMAND_DETAILS, COMMAND_USAGE, } from "./command-reference.mjs";
|
|
6
6
|
import { buildShellCompletion } from "./completion.mjs";
|
|
7
|
-
import { buildDoctorReport, formatDoctorReport } from "./doctor.mjs";
|
|
7
|
+
import { buildDoctorReport, buildDoctorSummary, formatDoctorReport } from "./doctor.mjs";
|
|
8
8
|
import { boundedCliErrorMessage, CliUsageError } from "./errors.mjs";
|
|
9
9
|
import EXIT_CODES from "./exit-codes.mjs";
|
|
10
10
|
import { formatJsonDocument } from "./formatters/json.mjs";
|
|
@@ -68,9 +68,11 @@ async function runDoctorCommand(args, dependencies, context) {
|
|
|
68
68
|
? await context.withProgress('Checking public DNS, HTTPS, and WHOIS connectivity', buildReport)
|
|
69
69
|
: await buildReport();
|
|
70
70
|
if (!args.quiet) {
|
|
71
|
-
context.writeStdout(args.output === '
|
|
72
|
-
? formatJsonDocument(report)
|
|
73
|
-
:
|
|
71
|
+
context.writeStdout(args.output === 'summary_json'
|
|
72
|
+
? formatJsonDocument(buildDoctorSummary(report))
|
|
73
|
+
: args.output === 'json'
|
|
74
|
+
? formatJsonDocument(report)
|
|
75
|
+
: context.terminal(formatDoctorReport(report), args.color));
|
|
74
76
|
}
|
|
75
77
|
return report.state === 'partial' ? EXIT_CODES.PARTIAL_FAILURE : EXIT_CODES.SUCCESS;
|
|
76
78
|
}
|
|
@@ -169,8 +171,8 @@ async function runRiskCalibrationCommand(args, dependencies, context) {
|
|
|
169
171
|
modelVersion: dependencies.riskModelVersion || RISK_MODEL_VERSION,
|
|
170
172
|
reviewThreshold: dependencies.riskReviewThreshold || RISK_REVIEW_THRESHOLD,
|
|
171
173
|
...(!dependencies.explainRiskScore ? {
|
|
172
|
-
previousModelVersion:
|
|
173
|
-
explainPreviousRiskScore:
|
|
174
|
+
previousModelVersion: 8,
|
|
175
|
+
explainPreviousRiskScore: explainRiskScoreV8,
|
|
174
176
|
} : {}),
|
|
175
177
|
});
|
|
176
178
|
if (!args.quiet) {
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { parseBoundedJson } from "../packages/analysis/bounded-json.mjs";
|
|
2
|
+
import { applyCandidateWatchHandoff, planCandidateWatchHandoff, } from "../packages/workspace/candidate-watch-handoff.mjs";
|
|
3
|
+
import { buildWatchlistExport, mergeWatchlistStores, } from "../packages/workspace/watchlist-store.mjs";
|
|
4
|
+
import { MAX_WATCHLIST_IMPORT_BYTES } from "../packages/contracts/workspace-portability.mjs";
|
|
5
|
+
import { ordinaryWorkspaceRecord } from "../packages/workspace/hostile-input.mjs";
|
|
6
|
+
import { CANDIDATE_WATCH_INPUT_SCHEMA, CANDIDATE_WATCH_INPUT_VERSION, } from "../packages/contracts/candidate-watch-review.mjs";
|
|
7
|
+
export function parseCandidateWatchInput(input) {
|
|
8
|
+
const value = typeof input === 'string'
|
|
9
|
+
? parseBoundedJson(input, {
|
|
10
|
+
maximumBytes: MAX_WATCHLIST_IMPORT_BYTES,
|
|
11
|
+
label: 'Candidate watch selection',
|
|
12
|
+
})
|
|
13
|
+
: input;
|
|
14
|
+
const raw = ordinaryWorkspaceRecord(value, 'Candidate watch selection');
|
|
15
|
+
if (!raw ||
|
|
16
|
+
raw.schema !== CANDIDATE_WATCH_INPUT_SCHEMA ||
|
|
17
|
+
raw.version !== CANDIDATE_WATCH_INPUT_VERSION ||
|
|
18
|
+
Object.keys(raw).some((key) => !['schema', 'version', 'watchlists', 'selection'].includes(key)))
|
|
19
|
+
throw new TypeError('Use candidate-watch-input schema 1 with watchlists and an explicit selection.');
|
|
20
|
+
const selection = ordinaryWorkspaceRecord(raw.selection, 'Candidate watch selection');
|
|
21
|
+
if (!selection ||
|
|
22
|
+
Object.keys(selection).some((key) => ![
|
|
23
|
+
'name',
|
|
24
|
+
'candidates',
|
|
25
|
+
'brandProfileId',
|
|
26
|
+
'priority',
|
|
27
|
+
'reason',
|
|
28
|
+
'reviewDueAt',
|
|
29
|
+
'replaceExistingContext',
|
|
30
|
+
].includes(key)))
|
|
31
|
+
throw new TypeError('The candidate selection contains undeclared fields.');
|
|
32
|
+
const local = raw.watchlists === null ? {} : mergeWatchlistStores({}, raw.watchlists).watchlists;
|
|
33
|
+
planCandidateWatchHandoff(local, selection);
|
|
34
|
+
return raw;
|
|
35
|
+
}
|
|
36
|
+
export function reviewCandidateWatchInput(input, operation, now) {
|
|
37
|
+
const raw = parseCandidateWatchInput(input), selection = raw.selection;
|
|
38
|
+
const local = raw.watchlists === null ? {} : mergeWatchlistStores({}, raw.watchlists).watchlists;
|
|
39
|
+
if (operation === 'plan')
|
|
40
|
+
return planCandidateWatchHandoff(local, selection);
|
|
41
|
+
const result = applyCandidateWatchHandoff(local, selection, now);
|
|
42
|
+
return buildWatchlistExport(result.watchlists, now);
|
|
43
|
+
}
|