@slicedearth/whoisleuth-cli 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/bulk-command-runner.mjs +5 -4
- package/cli/collection-arguments.mjs +6 -0
- package/cli/collection-command-definitions.mjs +13 -3
- package/cli/command-definition.mjs +2 -0
- package/cli/discovery-scan-command-runner.mjs +13 -7
- package/cli/domain-control-monitor.mjs +25 -2
- package/cli/domain-control-observations.mjs +52 -15
- package/cli/fail-policy.mjs +31 -22
- package/cli/lookup-command-runner.mjs +6 -5
- package/cli/lookup-diff.mjs +1 -0
- package/cli/risk-policy.mjs +83 -0
- package/cli/workflow-command-runner.mjs +5 -1
- package/docs/cli-reference.md +7 -0
- package/docs/cli.md +53 -1
- package/lib/generated/cisa-kev-catalog.mjs +5 -3
- package/lib/sslbl-certificates.generated.mjs +139 -139
- package/package.json +1 -1
- package/packages/analysis/brand-profile-signals.mjs +42 -0
- package/packages/investigation/bulk-domain-comparison.mjs +17 -8
- package/third-party-notices.txt +1 -1
|
@@ -8,7 +8,8 @@ import EXIT_CODES from "./exit-codes.mjs";
|
|
|
8
8
|
import { buildCliBulkDocument, formatJsonDocument, formatJsonLines } from "./formatters/json.mjs";
|
|
9
9
|
import { formatTerminalBulk } from "./formatters/terminal.mjs";
|
|
10
10
|
import { buildCollectionPreflight, formatCollectionPreflight } from "./collection-preflight.mjs";
|
|
11
|
-
import { evaluateCliFailPolicies,
|
|
11
|
+
import { evaluateCliFailPolicies, reportCliFailPolicies } from "./fail-policy.mjs";
|
|
12
|
+
import { readCliRiskProfile } from "./risk-policy.mjs";
|
|
12
13
|
import { formatCliJunit } from "./ci-report.mjs";
|
|
13
14
|
import { createCliProgressEvents } from "./progress-events.mjs";
|
|
14
15
|
async function runBulkCommand(args, dependencies, context) {
|
|
@@ -39,6 +40,7 @@ async function runBulkCommand(args, dependencies, context) {
|
|
|
39
40
|
context.writeStdout(args.output === 'json' ? formatJsonDocument(document) : context.terminal(formatCollectionPreflight(document), args.color));
|
|
40
41
|
return EXIT_CODES.SUCCESS;
|
|
41
42
|
}
|
|
43
|
+
const riskProfile = await readCliRiskProfile(args.brandProfileSource, context.readInput, args.brandProfileId);
|
|
42
44
|
const classify = dependencies.classifyQuery || classifyQuery;
|
|
43
45
|
const checkpointWriter = dependencies.createBulkCheckpointWriter || createBulkCheckpointWriter;
|
|
44
46
|
const checkpoint = args.checkpoint
|
|
@@ -106,6 +108,7 @@ async function runBulkCommand(args, dependencies, context) {
|
|
|
106
108
|
collectedTotal: items.length,
|
|
107
109
|
filter: args.filter,
|
|
108
110
|
};
|
|
111
|
+
const policyFindings = evaluateCliFailPolicies(buildCliBulkDocument(items, { ...metadata, filter: 'all' }), args.failOn || [], riskProfile);
|
|
109
112
|
const selectedItems = selectBulkItems(items, args.filter);
|
|
110
113
|
const selectedDocument = buildCliBulkDocument(selectedItems, metadata);
|
|
111
114
|
if (!args.quiet) {
|
|
@@ -126,9 +129,7 @@ async function runBulkCommand(args, dependencies, context) {
|
|
|
126
129
|
else
|
|
127
130
|
context.writeStdout(context.terminal(formatTerminalBulk(selectedItems, metadata), args.color));
|
|
128
131
|
}
|
|
129
|
-
|
|
130
|
-
if (policyFindings.length && !eventProgress.enabled)
|
|
131
|
-
context.writeStderr(formatFailPolicyNotice(policyFindings));
|
|
132
|
+
reportCliFailPolicies(policyFindings, context.writeStderr, eventProgress);
|
|
132
133
|
const exitCode = checkpointFailure || items.some((item) => !item.ok) || policyFindings.length
|
|
133
134
|
? EXIT_CODES.PARTIAL_FAILURE
|
|
134
135
|
: EXIT_CODES.SUCCESS;
|
|
@@ -29,6 +29,8 @@ function parseLookupArguments(parsed) {
|
|
|
29
29
|
? { saveLookup: parsed.optionValue('--save-lookup') }
|
|
30
30
|
: {}),
|
|
31
31
|
...(selectedFailPolicies ? { failOn: selectedFailPolicies } : {}),
|
|
32
|
+
...(parsed.optionValue('--brand-profile') ? { brandProfileSource: parsed.optionValue('--brand-profile') } : {}),
|
|
33
|
+
...(parsed.optionValue('--brand-id') ? { brandProfileId: parsed.optionValue('--brand-id') } : {}),
|
|
32
34
|
};
|
|
33
35
|
}
|
|
34
36
|
function parseBulkArguments(parsed) {
|
|
@@ -55,6 +57,8 @@ function parseBulkArguments(parsed) {
|
|
|
55
57
|
filter,
|
|
56
58
|
...terminalOptions(parsed),
|
|
57
59
|
...(selectedFailPolicies ? { failOn: selectedFailPolicies } : {}),
|
|
60
|
+
...(parsed.optionValue('--brand-profile') ? { brandProfileSource: parsed.optionValue('--brand-profile') } : {}),
|
|
61
|
+
...(parsed.optionValue('--brand-id') ? { brandProfileId: parsed.optionValue('--brand-id') } : {}),
|
|
58
62
|
};
|
|
59
63
|
}
|
|
60
64
|
function parseDiscoverArguments(parsed) {
|
|
@@ -113,6 +117,8 @@ function parseDiscoverScanArguments(parsed) {
|
|
|
113
117
|
plan: parsed.hasOption('--plan'),
|
|
114
118
|
...terminalOptions(parsed),
|
|
115
119
|
...(selectedFailPolicies ? { failOn: selectedFailPolicies } : {}),
|
|
120
|
+
...(parsed.optionValue('--brand-profile') ? { brandProfileSource: parsed.optionValue('--brand-profile') } : {}),
|
|
121
|
+
...(parsed.optionValue('--brand-id') ? { brandProfileId: parsed.optionValue('--brand-id') } : {}),
|
|
116
122
|
};
|
|
117
123
|
}
|
|
118
124
|
export const COLLECTION_ARGUMENT_PARSERS = Object.freeze({
|
|
@@ -2,12 +2,13 @@ import { positional, OPTIONAL_FILE_POSITIONAL, OPTIONAL_TEXT_POSITIONAL, constra
|
|
|
2
2
|
import { bulkQueryLimit, bulkConcurrencyLimit } from "../lib/bulk-limits.mjs";
|
|
3
3
|
const COLLECTION_CEILINGS = `${bulkQueryLimit('fast')} fast or ${bulkQueryLimit('deep')} deep`;
|
|
4
4
|
const CONCURRENCY_CEILINGS = `${bulkConcurrencyLimit('fast')} fast or ${bulkConcurrencyLimit('deep')} deep`;
|
|
5
|
+
const FAILURE_POLICY_BOUNDARY = 'Failure policies evaluate all collected and restored results before output filtering. For danger review, explicitly select --deep and --brand-profile; use --brand-id to select one profile from a multi-profile export. The current canonical Risk model compares collected evidence with that local reference; Fast, missing or ineligible evidence cannot establish a threshold match. The heuristic review threshold is not a maliciousness or safety verdict. The profile is not sent to collectors, retained in checkpoints or copied into output.';
|
|
5
6
|
export const COLLECTION_COMMAND_DEFINITIONS = Object.freeze({
|
|
6
7
|
lookup: commandSeed({
|
|
7
8
|
reference: {
|
|
8
9
|
description: 'Collect registration evidence for one domain, IP, or ASN.',
|
|
9
10
|
example: 'whoisleuth lookup example.test --deep --browse',
|
|
10
|
-
boundary: 'Fast is the default. An ICANN-recognised public domain, reserved documentation domain, IP, or ASN may occupy command position as shorthand; it delegates to this same parser and URL-like input requires the explicit lookup command. Deep mode adds bounded WHOIS, DNS, HTTP, TLS, technology, posture, and network context where applicable. --deep --exact-url explicitly sends the input URL path and query to the website, without its fragment; ordinary URL input sends only the hostname. --plan remains offline and omits the selected URL. Retained paths and page-derived text require review before sharing. A full Deep homepage observation can derive fixed publication and delivery/cache summaries from the same response without retaining raw metadata values or making another request. --browse opens before collection, shows aggregate Fast progress or independently settled planned Deep sources, and then navigates allowlisted retained fields in the completed document. Press ? for help and / to search rendered panel text only. Closing during collection cancels without a partial document. --save-lookup writes the exact completed private JSON only after a normal browser close; it can contain normalised evidence omitted from panels and refuses an existing path.',
|
|
11
|
+
boundary: 'Fast is the default. An ICANN-recognised public domain, reserved documentation domain, IP, or ASN may occupy command position as shorthand; it delegates to this same parser and URL-like input requires the explicit lookup command. Deep mode adds bounded WHOIS, DNS, HTTP, TLS, technology, posture, and network context where applicable. --deep --exact-url explicitly sends the input URL path and query to the website, without its fragment; ordinary URL input sends only the hostname. --plan remains offline and omits the selected URL. Retained paths and page-derived text require review before sharing. A full Deep homepage observation can derive fixed publication and delivery/cache summaries from the same response without retaining raw metadata values or making another request. --browse opens before collection, shows aggregate Fast progress or independently settled planned Deep sources, and then navigates allowlisted retained fields in the completed document. Press ? for help and / to search rendered panel text only. Closing during collection cancels without a partial document. --save-lookup writes the exact completed private JSON only after a normal browser close; it can contain normalised evidence omitted from panels and refuses an existing path. ' + FAILURE_POLICY_BOUNDARY,
|
|
11
12
|
},
|
|
12
13
|
collection: {
|
|
13
14
|
scope: 'Accepts one target. Fast is the default; deep collection must be selected explicitly.',
|
|
@@ -31,6 +32,8 @@ export const COLLECTION_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
31
32
|
'--save-lookup',
|
|
32
33
|
'--strict-exit',
|
|
33
34
|
'--fail-on',
|
|
35
|
+
'--brand-profile',
|
|
36
|
+
'--brand-id',
|
|
34
37
|
'--events',
|
|
35
38
|
'--quiet',
|
|
36
39
|
'--no-color',
|
|
@@ -39,6 +42,7 @@ export const COLLECTION_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
39
42
|
positional('target', 'text', 0, 1, [], 'argv_or_stdin', ['--browse']),
|
|
40
43
|
]),
|
|
41
44
|
constraints: Object.freeze([
|
|
45
|
+
constraint({ kind: 'requires_all', option: '--brand-id', requiredOptions: ['--brand-profile'] }),
|
|
42
46
|
constraint({
|
|
43
47
|
kind: 'mutually_exclusive',
|
|
44
48
|
options: ['--json', '--junit', '--markdown', '--html'],
|
|
@@ -106,7 +110,7 @@ export const COLLECTION_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
106
110
|
reference: {
|
|
107
111
|
description: 'Triage newline-delimited domains, IPs, or ASNs with bounded concurrency.',
|
|
108
112
|
example: 'cat domains.txt | whoisleuth bulk --jsonl',
|
|
109
|
-
boundary: 'Fast and deep jobs use separate concurrency ceilings. Filters affect output only; collection failures and inconclusive authority states remain explicit in JSON, JSONL, and CSV. --csv-with-metadata adds source versions, separate observation and report times, collection origin and diagnostic states; --csv retains the compact columns.',
|
|
113
|
+
boundary: 'Fast and deep jobs use separate concurrency ceilings. Filters affect output only; collection failures and inconclusive authority states remain explicit in JSON, JSONL, and CSV. --csv-with-metadata adds source versions, separate observation and report times, collection origin and diagnostic states; --csv retains the compact columns. ' + FAILURE_POLICY_BOUNDARY,
|
|
110
114
|
},
|
|
111
115
|
collection: {
|
|
112
116
|
scope: `Accepts at most ${COLLECTION_CEILINGS} targets, with concurrency capped at ${CONCURRENCY_CEILINGS}.`,
|
|
@@ -131,11 +135,14 @@ export const COLLECTION_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
131
135
|
'--events',
|
|
132
136
|
'--plan',
|
|
133
137
|
'--fail-on',
|
|
138
|
+
'--brand-profile',
|
|
139
|
+
'--brand-id',
|
|
134
140
|
'--quiet',
|
|
135
141
|
'--no-color',
|
|
136
142
|
],
|
|
137
143
|
positionals: OPTIONAL_FILE_POSITIONAL,
|
|
138
144
|
constraints: Object.freeze([
|
|
145
|
+
constraint({ kind: 'requires_all', option: '--brand-id', requiredOptions: ['--brand-profile'] }),
|
|
139
146
|
constraint({
|
|
140
147
|
kind: 'mutually_exclusive',
|
|
141
148
|
options: [
|
|
@@ -236,7 +243,7 @@ export const COLLECTION_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
236
243
|
reference: {
|
|
237
244
|
description: 'Generate a bounded candidate set, collect a selected subset, and produce a supervised review queue.',
|
|
238
245
|
example: 'whoisleuth discover-scan example.test --scan-limit 50 --checkpoint scan.json --json',
|
|
239
|
-
boundary: `This command performs network collection. Fast compact lookup is the default; deep mode is capped at ${bulkQueryLimit('deep')} candidates. Allowlisting changes review priority only and shared infrastructure remains a lead, not attribution. --csv-with-metadata adds source versions, separate observation and report times, collection origin and diagnostic states; --csv retains the compact columns
|
|
246
|
+
boundary: `This command performs network collection. Fast compact lookup is the default; deep mode is capped at ${bulkQueryLimit('deep')} candidates. Allowlisting changes review priority only and shared infrastructure remains a lead, not attribution. --csv-with-metadata adds source versions, separate observation and report times, collection origin and diagnostic states; --csv retains the compact columns. ${FAILURE_POLICY_BOUNDARY}`,
|
|
240
247
|
},
|
|
241
248
|
collection: {
|
|
242
249
|
scope: `Scans at most ${COLLECTION_CEILINGS} candidates, with concurrency capped at ${CONCURRENCY_CEILINGS}.`,
|
|
@@ -265,6 +272,8 @@ export const COLLECTION_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
265
272
|
'--events',
|
|
266
273
|
'--plan',
|
|
267
274
|
'--fail-on',
|
|
275
|
+
'--brand-profile',
|
|
276
|
+
'--brand-id',
|
|
268
277
|
'--json',
|
|
269
278
|
'--jsonl',
|
|
270
279
|
'--csv',
|
|
@@ -275,6 +284,7 @@ export const COLLECTION_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
275
284
|
],
|
|
276
285
|
positionals: OPTIONAL_TEXT_POSITIONAL,
|
|
277
286
|
constraints: Object.freeze([
|
|
287
|
+
constraint({ kind: 'requires_all', option: '--brand-id', requiredOptions: ['--brand-profile'] }),
|
|
278
288
|
constraint({
|
|
279
289
|
kind: 'mutually_exclusive',
|
|
280
290
|
options: ['--json', '--jsonl', '--csv', '--csv-with-metadata', '--domains'],
|
|
@@ -156,6 +156,8 @@ const CLI_OPTION_DEFINITIONS = Object.freeze({
|
|
|
156
156
|
'--interactive': flag('Prompt for missing supported inputs on an interactive terminal.'),
|
|
157
157
|
'--save-lookup': file('After a normal evidence-browser close, save the completed private Lookup JSON to a new file.'),
|
|
158
158
|
'--strict-exit': flag('Use the command’s strict outcome policy when deciding the exit status.'),
|
|
159
|
+
'--brand-profile': file('Read a local Brand Profile export for danger-policy comparisons; no additional requests or profile retention.'),
|
|
160
|
+
'--brand-id': text('Select an exact profile ID when --brand-profile contains multiple profiles.'),
|
|
159
161
|
'--fail-on': optionDefinition('policy_list', 'Return a failure-policy exit status for the selected comma-separated outcomes.', {
|
|
160
162
|
values: (command) => CLI_FAIL_POLICIES_BY_COMMAND[command] ?? [],
|
|
161
163
|
}),
|
|
@@ -10,7 +10,8 @@ import EXIT_CODES from "./exit-codes.mjs";
|
|
|
10
10
|
import { formatJsonDocument } from "./formatters/json.mjs";
|
|
11
11
|
import { createCliProgressEvents } from "./progress-events.mjs";
|
|
12
12
|
import { buildCollectionPreflight, formatCollectionPreflight } from "./collection-preflight.mjs";
|
|
13
|
-
import { evaluateCliFailPolicies,
|
|
13
|
+
import { evaluateCliFailPolicies, reportCliFailPolicies } from "./fail-policy.mjs";
|
|
14
|
+
import { readCliRiskProfile } from "./risk-policy.mjs";
|
|
14
15
|
async function readAllowlist(source, dependencies, context, classify) {
|
|
15
16
|
if (!source)
|
|
16
17
|
return new Set();
|
|
@@ -59,6 +60,7 @@ async function runDiscoveryScanCommand(args, dependencies, context) {
|
|
|
59
60
|
context.writeStdout(args.output === 'json' ? formatJsonDocument(document) : context.terminal(formatCollectionPreflight(document), args.color));
|
|
60
61
|
return EXIT_CODES.SUCCESS;
|
|
61
62
|
}
|
|
63
|
+
const riskProfile = await readCliRiskProfile(args.brandProfileSource, context.readInput, args.brandProfileId);
|
|
62
64
|
const classify = dependencies.classifyQuery || classifyQuery;
|
|
63
65
|
const allowlist = await readAllowlist(args.allowlistSource, dependencies, context, classify);
|
|
64
66
|
const checkpointWriter = dependencies.createBulkCheckpointWriter || createBulkCheckpointWriter;
|
|
@@ -144,7 +146,7 @@ async function runDiscoveryScanCommand(args, dependencies, context) {
|
|
|
144
146
|
const snapshot = args.observationSnapshot
|
|
145
147
|
? await updateDiscoveryObservationSnapshot(args.observationSnapshot, candidates, items, { deep: args.deep, resolverServers }, generationMetadata.generatedAt)
|
|
146
148
|
: null;
|
|
147
|
-
const
|
|
149
|
+
const metadata = {
|
|
148
150
|
generatedAt: generationMetadata.generatedAt,
|
|
149
151
|
seed: generationMetadata.seed,
|
|
150
152
|
preset: generationMetadata.preset,
|
|
@@ -157,9 +159,15 @@ async function runDiscoveryScanCommand(args, dependencies, context) {
|
|
|
157
159
|
chunkSize: args.chunkSize,
|
|
158
160
|
concurrency: args.concurrency,
|
|
159
161
|
deep: args.deep,
|
|
160
|
-
filter:
|
|
162
|
+
filter: 'all',
|
|
161
163
|
resolverServers,
|
|
162
|
-
}
|
|
164
|
+
};
|
|
165
|
+
// Policies describe the complete collected run, including restored results,
|
|
166
|
+
// source diagnostics and candidate context, before presentation selection.
|
|
167
|
+
const collectedDocument = buildDiscoveryScanDocument(candidates, items, metadata, allowlist, snapshot);
|
|
168
|
+
const policyFindings = evaluateCliFailPolicies(collectedDocument, args.failOn || [], riskProfile);
|
|
169
|
+
const document = args.filter === 'all' ? collectedDocument
|
|
170
|
+
: buildDiscoveryScanDocument(candidates, items, { ...metadata, filter: args.filter }, allowlist, snapshot);
|
|
163
171
|
if (!args.quiet) {
|
|
164
172
|
if (args.output === 'json')
|
|
165
173
|
context.writeStdout(formatJsonDocument(document));
|
|
@@ -174,9 +182,7 @@ async function runDiscoveryScanCommand(args, dependencies, context) {
|
|
|
174
182
|
else
|
|
175
183
|
context.writeStdout(context.terminal(formatTerminalDiscoveryScan(document), args.color));
|
|
176
184
|
}
|
|
177
|
-
|
|
178
|
-
if (policyFindings.length && !eventProgress.enabled)
|
|
179
|
-
context.writeStderr(formatFailPolicyNotice(policyFindings));
|
|
185
|
+
reportCliFailPolicies(policyFindings, context.writeStderr, eventProgress);
|
|
180
186
|
const exitCode = checkpointFailure || items.some((item) => !item.ok) || policyFindings.length
|
|
181
187
|
? EXIT_CODES.PARTIAL_FAILURE
|
|
182
188
|
: EXIT_CODES.SUCCESS;
|
|
@@ -9,6 +9,8 @@ import { requireDomainName, requireIsoTimestamp } from "../lib/bounded-contract-
|
|
|
9
9
|
import { buildCliLookupDocument } from "./formatters/json.mjs";
|
|
10
10
|
import { buildCliDomainControlReview, DOMAIN_CONTROL_REVIEW_SOURCE_FIELDS, } from "./domain-control-observations.mjs";
|
|
11
11
|
import { CliUsageError } from "./errors.mjs";
|
|
12
|
+
import { lookupStrictExitFindings } from "./strict-exit.mjs";
|
|
13
|
+
import { lookupObservationHostname } from "../packages/evidence/lookup-target.mjs";
|
|
12
14
|
export { CLI_DOMAIN_CONTROL_MONITOR_SCHEMA, CLI_DOMAIN_CONTROL_MONITOR_VERSION };
|
|
13
15
|
export const MAX_DOMAIN_CONTROL_MONITOR_DOMAINS = CONTRACT_MAX_DOMAIN_CONTROL_MONITOR_DOMAINS;
|
|
14
16
|
function parseMonitorJson(input, label) {
|
|
@@ -232,6 +234,19 @@ export async function runDomainControlMonitor(manifestInput, previousInput, opti
|
|
|
232
234
|
}
|
|
233
235
|
const previous = previousSnapshot(previousInput, startedAt);
|
|
234
236
|
const entries = manifest.entries.slice(0, Math.min(options.limit, MAX_DOMAIN_CONTROL_MONITOR_DOMAINS));
|
|
237
|
+
const targets = entries.map(entry => {
|
|
238
|
+
let classified;
|
|
239
|
+
try {
|
|
240
|
+
classified = classifyQuery(entry.domain);
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
throw new CliUsageError('monitor-once requires registrable-domain targets.');
|
|
244
|
+
}
|
|
245
|
+
if (classified.type !== 'domain' || classified.isSubdomain) {
|
|
246
|
+
throw new CliUsageError('monitor-once supports registrable-domain targets only; exact-host monitoring is not supported.');
|
|
247
|
+
}
|
|
248
|
+
return classified;
|
|
249
|
+
});
|
|
235
250
|
const lookups = new Array(entries.length);
|
|
236
251
|
const failuresByIndex = new Array(entries.length);
|
|
237
252
|
let cursor = 0;
|
|
@@ -244,13 +259,18 @@ export async function runDomainControlMonitor(manifestInput, previousInput, opti
|
|
|
244
259
|
if (!entry)
|
|
245
260
|
break;
|
|
246
261
|
try {
|
|
247
|
-
const classified =
|
|
262
|
+
const classified = targets[index];
|
|
248
263
|
const result = await options.executeLookup(classified, { fast: false, compact: false, ...(options.signal ? { signal: options.signal } : {}) });
|
|
249
264
|
options.signal?.throwIfAborted();
|
|
250
265
|
const capturedAt = requireIsoTimestamp(options.now(), 'Domain-control capture time');
|
|
251
266
|
if (Date.parse(capturedAt) < checkedAt)
|
|
252
267
|
throw new CliUsageError('Domain-control capture time must not precede the run.');
|
|
253
|
-
|
|
268
|
+
const lookup = buildCliLookupDocument(entry.domain, classified, result, capturedAt, 'deep');
|
|
269
|
+
const observedHostname = lookupObservationHostname({ domain: entry.domain, ...lookup.availability });
|
|
270
|
+
if (observedHostname !== null && observedHostname !== entry.domain) {
|
|
271
|
+
throw new TypeError('Lookup observation does not match the monitor target.');
|
|
272
|
+
}
|
|
273
|
+
lookups[index] = lookup;
|
|
254
274
|
}
|
|
255
275
|
catch {
|
|
256
276
|
if (options.signal?.aborted) {
|
|
@@ -270,6 +290,9 @@ export async function runDomainControlMonitor(manifestInput, previousInput, opti
|
|
|
270
290
|
const successful = lookups.filter(Boolean);
|
|
271
291
|
if (!successful.length)
|
|
272
292
|
throw new Error('No domain-control monitor lookup completed successfully.');
|
|
293
|
+
// Only current Lookup diagnostics contribute; checkpoints retain observations,
|
|
294
|
+
// not policy metadata or raw diagnostics.
|
|
295
|
+
options.onSourceFailures?.(successful.reduce((count, lookup) => count + lookupStrictExitFindings(lookup).length, 0));
|
|
273
296
|
const generatedAt = requireIsoTimestamp(options.now(), 'Domain-control completion time');
|
|
274
297
|
if (Date.parse(generatedAt) < checkedAt || successful.some((lookup) => Date.parse(String(lookup.generatedAt)) > Date.parse(generatedAt))) {
|
|
275
298
|
throw new CliUsageError('Domain-control completion time must not precede its captures.');
|
|
@@ -3,10 +3,11 @@ import { latestObservationCohort } from "../packages/evidence/latest-observation
|
|
|
3
3
|
import { canonicalCaaRecord, canonicalDsRecord, canonicalMxRecord, canonicalDomainControlRecords, canonicalPostureRecords } from "../packages/evidence/domain-control-runtime.mjs";
|
|
4
4
|
import { normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
|
|
5
5
|
import { postureTransferRestriction } from "../packages/evidence/domain-posture-context.mjs";
|
|
6
|
+
import { lookupObservationHostname } from "../packages/evidence/lookup-target.mjs";
|
|
6
7
|
import { MAX_DOMAIN_CONTROL_INPUT_RECORDS, PUBLIC_DOMAIN_CONTROL_MANIFEST_VERSION } from "../packages/contracts/domain-control-manifest.mjs";
|
|
7
8
|
import { scanBoundedJson, } from "../packages/analysis/bounded-json.mjs";
|
|
8
9
|
import { CLI_DOMAIN_CONTROL_REVIEW_INPUT_KEYS, CLI_DOMAIN_CONTROL_REVIEW_INPUT_SCHEMA, CLI_DOMAIN_CONTROL_REVIEW_LIMITATIONS, CLI_DOMAIN_CONTROL_REVIEW_SCHEMA, CLI_DOMAIN_CONTROL_REVIEW_VERSION, PUBLIC_CLI_DOMAIN_CONTROL_REVIEW_VERSION, SUPPORTED_CLI_DOMAIN_CONTROL_REVIEW_VERSIONS, DOMAIN_CONTROL_REVIEW_VERSION, MAX_DOMAIN_CONTROL_REVIEW_INPUT_BYTES, MAX_DOMAIN_CONTROL_REVIEW_JSON_DEPTH, MAX_DOMAIN_CONTROL_REVIEW_JSON_KEYS, MAX_DOMAIN_CONTROL_REVIEW_JSON_VALUES, MAX_DOMAIN_CONTROL_REVIEW_LOOKUPS, MIN_DOMAIN_CONTROL_REVIEW_LOOKUPS, } from "../packages/contracts/domain-control-review.mjs";
|
|
9
|
-
import { MAX_FLIGHT_RECORDER_VALUES, mergeConcurrentDomainControlFields, } from "../lib/domain-control-flight-recorder.mjs";
|
|
10
|
+
import { MAX_FLIGHT_RECORDER_VALUES, DOMAIN_CONTROL_FLIGHT_RECORDER_FIELDS, mergeConcurrentDomainControlFields, } from "../lib/domain-control-flight-recorder.mjs";
|
|
10
11
|
import { DOMAIN_CONTROL_REVIEW_INPUT_SCHEMA, reviewDomainControlManifest, verifyDomainControlManifest, } from "../lib/domain-control-manifest.mjs";
|
|
11
12
|
import { CliUsageError } from "./errors.mjs";
|
|
12
13
|
import { parseSavedLookupDocument } from "./saved-lookup.mjs";
|
|
@@ -113,12 +114,21 @@ function pageIdentityValue(availability) {
|
|
|
113
114
|
].filter((item) => Boolean(item));
|
|
114
115
|
return components.length ? [components.join(' | ')] : [];
|
|
115
116
|
}
|
|
116
|
-
|
|
117
|
+
function serviceHostname(document) {
|
|
118
|
+
// Historical saved Lookups collected supporting evidence at the registration
|
|
119
|
+
// domain unless availability retained a more specific scope. Never use query.
|
|
120
|
+
return lookupObservationHostname({ domain: document.registrableDomain, ...record(document.availability) });
|
|
121
|
+
}
|
|
122
|
+
export function domainControlObservationFromSavedLookup(document, domain = document.registrableDomain) {
|
|
117
123
|
const availability = record(document.availability);
|
|
124
|
+
const observedHostname = serviceHostname(document);
|
|
118
125
|
const dns = record(availability.dns);
|
|
119
126
|
const dnsRecords = record(dns.records);
|
|
120
127
|
const dnsAt = sourceClock(dns.observedAt, dns.checkedAt);
|
|
121
128
|
const delegation = record(dns.delegation);
|
|
129
|
+
const delegationHostname = delegation.domain === undefined
|
|
130
|
+
? observedHostname === document.registrableDomain ? observedHostname : null
|
|
131
|
+
: lookupObservationHostname({ domain: delegation.domain });
|
|
122
132
|
const delegationRecords = record(delegation.records);
|
|
123
133
|
const tls = record(availability.tls);
|
|
124
134
|
const tlsState = sourceState(tls);
|
|
@@ -134,14 +144,15 @@ export function domainControlObservationFromSavedLookup(document) {
|
|
|
134
144
|
const hostname = (value) => canonicalDomainControlRecords([value], 'nameservers')[0] ?? '';
|
|
135
145
|
const identity = record(availability.pageIdentity);
|
|
136
146
|
const identityValues = pageIdentityValue(availability);
|
|
137
|
-
const
|
|
147
|
+
const registrationFields = [
|
|
138
148
|
scalar('registrar', registry.source, registry.sourceState, record(registry.parsed.registrar).name ?? registry.parsed.registrar, registry.observedAt, 300),
|
|
139
149
|
scalar('registrar_lock', registry.source, registry.sourceState, lockValue(registry.parsed), registry.observedAt, 20),
|
|
140
150
|
scalar('registry_dnssec', 'Registry RDAP', rdapState, rdapParsed.dnssec, rdapAt, 80),
|
|
141
151
|
recordField('registry_nameservers', 'Registry RDAP', rdapState, rdapParsed.nameservers, hostname, rdapAt),
|
|
142
152
|
recordField('whois_nameservers', 'WHOIS', diagnostic(document, 'whois'), record(record(document.whois).parsed).nameservers, hostname, registryClock(document, 'whois')),
|
|
153
|
+
];
|
|
154
|
+
const serviceFields = [
|
|
143
155
|
recordField('delegated_nameservers', 'DNS NS', dnsRecordState(dns, 'ns'), dnsRecords.ns, hostname, dnsAt),
|
|
144
|
-
recordField('delegation_ds', 'DNS delegation', Object.hasOwn(delegationRecords, 'ds') ? sourceState(delegation) : 'unsupported', delegationRecords.ds, canonicalDsRecord, sourceClock(delegation.observedAt)),
|
|
145
156
|
recordField('mail_exchangers', 'DNS MX', dnsRecordState(dns, 'mx'), dnsRecords.mx, canonicalMxRecord, dnsAt),
|
|
146
157
|
recordField('caa_policy', 'DNS CAA', dnsRecordState(dns, 'caa'), dnsRecords.caa, canonicalCaaRecord, dnsAt),
|
|
147
158
|
scalar('tls_certificate', 'TLS', tlsState, certificate.fingerprintSha256 ?? tls.fingerprintSha256, tlsAt, 128),
|
|
@@ -149,8 +160,13 @@ export function domainControlObservationFromSavedLookup(document) {
|
|
|
149
160
|
scalar('http_origin', 'HTTP', sourceState(http), http.finalOrigin, sourceClock(http.observedAt, http.checkedAt)),
|
|
150
161
|
field('page_identity', 'Static page identity', sourceState(identity) === 'observed' && !identityValues.length ? 'partial' : sourceState(identity), identityValues, sourceClock(identity.observedAt)),
|
|
151
162
|
];
|
|
163
|
+
const fields = mergeConcurrentDomainControlFields([
|
|
164
|
+
domain === document.registrableDomain ? registrationFields : [],
|
|
165
|
+
domain === observedHostname ? serviceFields : [],
|
|
166
|
+
domain === delegationHostname ? [recordField('delegation_ds', 'DNS delegation', Object.hasOwn(delegationRecords, 'ds') ? sourceState(delegation) : 'unsupported', delegationRecords.ds, canonicalDsRecord, sourceClock(delegation.observedAt))] : [],
|
|
167
|
+
]);
|
|
152
168
|
return Object.freeze({
|
|
153
|
-
domain
|
|
169
|
+
domain, capturedAt: document.generatedAt, collectionDepth: document.mode,
|
|
154
170
|
fields: Object.freeze(fields.map((item) => item.state === 'observed' && item.observedAt !== null
|
|
155
171
|
&& Date.parse(item.observedAt) > Date.parse(document.generatedAt) ? Object.freeze({ ...item, state: 'partial' }) : item)),
|
|
156
172
|
});
|
|
@@ -219,19 +235,40 @@ export function buildCliDomainControlReview(inputText, generatedAt = new Date().
|
|
|
219
235
|
const lookups = input.lookups.map((item, index) => parseSavedLookupDocument(JSON.stringify(item), { label: `Lookup ${index + 1}` }));
|
|
220
236
|
const byDomain = new Map();
|
|
221
237
|
for (const lookup of lookups) {
|
|
222
|
-
const
|
|
223
|
-
|
|
224
|
-
|
|
238
|
+
const domains = new Set([lookup.registrableDomain, serviceHostname(lookup)]);
|
|
239
|
+
for (const domain of domains) {
|
|
240
|
+
if (domain === null)
|
|
241
|
+
continue;
|
|
242
|
+
const group = byDomain.get(domain) ?? [];
|
|
243
|
+
group.push(lookup);
|
|
244
|
+
byDomain.set(domain, group);
|
|
245
|
+
}
|
|
225
246
|
}
|
|
226
|
-
|
|
247
|
+
const retainedLookups = new Set();
|
|
227
248
|
const selected = [...byDomain.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([domain, documents]) => {
|
|
228
|
-
const
|
|
229
|
-
|
|
230
|
-
|
|
249
|
+
const candidates = documents.map((document) => {
|
|
250
|
+
const observation = domainControlObservationFromSavedLookup(document, domain);
|
|
251
|
+
return { document, observation, reviewFields: reviewFields(document, observation) };
|
|
252
|
+
});
|
|
253
|
+
const currentCaptures = new Set();
|
|
254
|
+
const currentFields = [];
|
|
255
|
+
const currentReviewFields = [];
|
|
256
|
+
for (const id of DOMAIN_CONTROL_FLIGHT_RECORDER_FIELDS) {
|
|
257
|
+
const eligible = candidates.filter(candidate => candidate.observation.fields.some(item => item.id === id));
|
|
258
|
+
const latest = latestObservationCohort(eligible, candidate => candidate.document.generatedAt);
|
|
259
|
+
for (const capture of latest.latest) {
|
|
260
|
+
currentCaptures.add(capture);
|
|
261
|
+
retainedLookups.add(capture.document);
|
|
262
|
+
currentFields.push(capture.observation.fields.filter(item => item.id === id));
|
|
263
|
+
currentReviewFields.push(capture.reviewFields.filter(item => item.id === id));
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
const captures = [...currentCaptures];
|
|
267
|
+
const cohort = latestObservationCohort(captures, capture => capture.document.generatedAt);
|
|
231
268
|
const first = captures[0];
|
|
232
269
|
if (!first || !cohort.observedAt || cohort.undated.length)
|
|
233
270
|
throw new CliUsageError('Saved Lookup observation times could not be ordered.');
|
|
234
|
-
const mergedFields = mergeConcurrentDomainControlFields(
|
|
271
|
+
const mergedFields = mergeConcurrentDomainControlFields(currentFields);
|
|
235
272
|
const futureCapture = Date.parse(cohort.observedAt) > Date.parse(generatedAt);
|
|
236
273
|
const observation = Object.freeze({
|
|
237
274
|
domain,
|
|
@@ -240,7 +277,7 @@ export function buildCliDomainControlReview(inputText, generatedAt = new Date().
|
|
|
240
277
|
? first.observation.collectionDepth : 'unknown',
|
|
241
278
|
fields: futureCapture ? mergedFields.map((field) => ({ ...field, state: 'partial' })) : mergedFields,
|
|
242
279
|
});
|
|
243
|
-
const fields = mergeConcurrentDomainControlFields(
|
|
280
|
+
const fields = mergeConcurrentDomainControlFields(currentReviewFields);
|
|
244
281
|
return { observation, reviewFields: futureCapture || observation.collectionDepth === 'unknown'
|
|
245
282
|
? fields.map((field) => ({ ...field, state: 'partial' })) : fields };
|
|
246
283
|
});
|
|
@@ -268,7 +305,7 @@ export function buildCliDomainControlReview(inputText, generatedAt = new Date().
|
|
|
268
305
|
input: Object.freeze({
|
|
269
306
|
lookupsReceived: lookups.length,
|
|
270
307
|
latestDomainObservations: observations.length,
|
|
271
|
-
ignoredHistoricalLookups,
|
|
308
|
+
ignoredHistoricalLookups: lookups.length - retainedLookups.size,
|
|
272
309
|
}),
|
|
273
310
|
limitations: CLI_DOMAIN_CONTROL_REVIEW_LIMITATIONS,
|
|
274
311
|
});
|
package/cli/fail-policy.mjs
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { lookupStrictExitFindings } from "./strict-exit.mjs";
|
|
2
2
|
import { CliUsageError } from "./errors.mjs";
|
|
3
|
+
import { RISK_REVIEW_THRESHOLD } from "../packages/analysis/risk-scoring.mjs";
|
|
4
|
+
import { explainCliLookupRisk } from "./risk-policy.mjs";
|
|
5
|
+
import { DOMAIN_CONTROL_REVIEW_SCHEMA } from "../packages/contracts/domain-control-review.mjs";
|
|
3
6
|
export const CLI_FAIL_POLICIES = ['source-failure', 'inconclusive', 'danger', 'material-drift'];
|
|
4
7
|
export const CLI_FAIL_POLICIES_BY_COMMAND = Object.freeze({
|
|
5
8
|
lookup: Object.freeze(['source-failure', 'inconclusive', 'danger']),
|
|
@@ -31,14 +34,7 @@ function availabilityStates(document) {
|
|
|
31
34
|
});
|
|
32
35
|
return [direct, review, ...resultStates].filter(Boolean);
|
|
33
36
|
}
|
|
34
|
-
function
|
|
35
|
-
const raw = record(value.risk).score ?? value.riskScore;
|
|
36
|
-
if (raw === null || raw === undefined || raw === '')
|
|
37
|
-
return null;
|
|
38
|
-
const parsed = Number(raw);
|
|
39
|
-
return Number.isFinite(parsed) ? parsed : null;
|
|
40
|
-
}
|
|
41
|
-
export function evaluateCliFailPolicies(document, policies) {
|
|
37
|
+
export function evaluateCliFailPolicies(document, policies, profile = null, currentSourceFailureCount = 0) {
|
|
42
38
|
const root = record(document);
|
|
43
39
|
const findings = [];
|
|
44
40
|
for (const policy of policies) {
|
|
@@ -50,28 +46,31 @@ export function evaluateCliFailPolicies(document, policies) {
|
|
|
50
46
|
const collection = record(root.collection);
|
|
51
47
|
const summary = record(root.summary);
|
|
52
48
|
const failed = Number(collection.failed ?? summary.failed ?? 0);
|
|
53
|
-
|
|
54
|
-
|
|
49
|
+
const failures = sourceFindings.length + failed + currentSourceFailureCount;
|
|
50
|
+
if (failures > 0)
|
|
51
|
+
findings.push({ policy, reason: `${failures} failed or incomplete collection state(s)` });
|
|
55
52
|
}
|
|
56
53
|
else if (policy === 'inconclusive') {
|
|
57
|
-
const
|
|
54
|
+
const review = record(root.review);
|
|
55
|
+
const comparisons = review.schema === DOMAIN_CONTROL_REVIEW_SCHEMA;
|
|
56
|
+
const counts = record(review.counts);
|
|
57
|
+
const count = comparisons
|
|
58
|
+
? Number(counts.partial ?? 0) + Number(counts.unavailable ?? 0) + Number(counts.unsupported ?? 0)
|
|
59
|
+
: availabilityStates(root).filter((state) => ['unknown', 'inconclusive', 'partial', 'unavailable'].includes(state)).length;
|
|
58
60
|
if (count)
|
|
59
|
-
findings.push({ policy, reason: `${count} authority or availability state(s) remained inconclusive` });
|
|
61
|
+
findings.push({ policy, reason: `${count} ${comparisons ? 'requested comparison(s)' : 'authority or availability state(s)'} remained inconclusive` });
|
|
60
62
|
}
|
|
61
63
|
else if (policy === 'danger') {
|
|
62
64
|
const documents = [root, ...(Array.isArray(root.results) ? root.results.map(record) : [])];
|
|
63
|
-
const risks = documents.
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
if ((highestRisk !== null && highestRisk >= 70) || severities.length) {
|
|
65
|
+
const risks = documents.flatMap(item => {
|
|
66
|
+
const risk = explainCliLookupRisk(item, profile, item.mode ?? root.mode);
|
|
67
|
+
return risk ? [risk] : [];
|
|
68
|
+
});
|
|
69
|
+
const highestRisk = risks.reduce((highest, risk) => (!highest || risk.score > highest.score ? risk : highest), null);
|
|
70
|
+
if (highestRisk && highestRisk.score >= RISK_REVIEW_THRESHOLD) {
|
|
70
71
|
findings.push({
|
|
71
72
|
policy,
|
|
72
|
-
reason: highestRisk
|
|
73
|
-
? `risk score ${highestRisk} met the 70 review threshold`
|
|
74
|
-
: `explicit ${severities[0]} severity was present`,
|
|
73
|
+
reason: `risk score ${highestRisk.score} met the ${RISK_REVIEW_THRESHOLD} review threshold; model v${highestRisk.modelVersion}; evidence ${highestRisk.evidenceQuality.state} (heuristic, not a verdict)`,
|
|
75
74
|
});
|
|
76
75
|
}
|
|
77
76
|
}
|
|
@@ -87,3 +86,13 @@ export function evaluateCliFailPolicies(document, policies) {
|
|
|
87
86
|
export function formatFailPolicyNotice(findings) {
|
|
88
87
|
return `Failure policy matched: ${findings.map((item) => `${item.policy} (${item.reason})`).join('; ')}.\n`;
|
|
89
88
|
}
|
|
89
|
+
export function reportCliFailPolicies(findings, writeStderr, progress) {
|
|
90
|
+
if (!findings.length)
|
|
91
|
+
return;
|
|
92
|
+
if (!progress.enabled)
|
|
93
|
+
writeStderr(formatFailPolicyNotice(findings));
|
|
94
|
+
else
|
|
95
|
+
for (const finding of findings) {
|
|
96
|
+
progress.emit({ event: 'warning', state: 'failure_policy_matched', source: finding.policy, reason: finding.reason });
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -13,7 +13,8 @@ import { buildCliLookupPlan, formatCliLookupPlan } from "./lookup-plan.mjs";
|
|
|
13
13
|
import { createCliProgressEvents } from "./progress-events.mjs";
|
|
14
14
|
import { serializeCliLookupDocument } from "./saved-lookup.mjs";
|
|
15
15
|
import { lookupStrictExitFindings } from "./strict-exit.mjs";
|
|
16
|
-
import { evaluateCliFailPolicies,
|
|
16
|
+
import { evaluateCliFailPolicies, reportCliFailPolicies } from "./fail-policy.mjs";
|
|
17
|
+
import { readCliRiskProfile } from "./risk-policy.mjs";
|
|
17
18
|
import { formatCliJunit } from "./ci-report.mjs";
|
|
18
19
|
import { browseLookupOperation, canBrowseLookup } from "./lookup-browser.mjs";
|
|
19
20
|
import { writePrivateFile } from "./output-file.mjs";
|
|
@@ -66,6 +67,7 @@ async function runLookupCommand(args, dependencies, context) {
|
|
|
66
67
|
: context.terminal(formatCliLookupPlan(plan), args.color));
|
|
67
68
|
return EXIT_CODES.SUCCESS;
|
|
68
69
|
}
|
|
70
|
+
const riskProfile = await readCliRiskProfile(args.brandProfileSource, context.readInput, args.brandProfileId);
|
|
69
71
|
let documentGeneratedAt = '';
|
|
70
72
|
const buildDocument = ({ result, collectionReceipt }) => {
|
|
71
73
|
documentGeneratedAt = context.now();
|
|
@@ -139,6 +141,8 @@ async function runLookupCommand(args, dependencies, context) {
|
|
|
139
141
|
}
|
|
140
142
|
document = buildDocument(result);
|
|
141
143
|
}
|
|
144
|
+
const strictFindings = args.strictExit ? lookupStrictExitFindings(document) : [];
|
|
145
|
+
const policyFindings = evaluateCliFailPolicies(document, args.failOn || [], riskProfile);
|
|
142
146
|
if (!args.quiet) {
|
|
143
147
|
if (args.output === 'json')
|
|
144
148
|
context.writeStdout(serializeCliLookupDocument(document));
|
|
@@ -156,14 +160,11 @@ async function runLookupCommand(args, dependencies, context) {
|
|
|
156
160
|
context.writeStdout(context.terminal(formatTerminalLookup(document, { detail: args.detail }), args.color));
|
|
157
161
|
}
|
|
158
162
|
}
|
|
159
|
-
const strictFindings = args.strictExit ? lookupStrictExitFindings(document) : [];
|
|
160
|
-
const policyFindings = evaluateCliFailPolicies(document, args.failOn || []);
|
|
161
163
|
const exitCode = strictFindings.length || policyFindings.length ? EXIT_CODES.PARTIAL_FAILURE : EXIT_CODES.SUCCESS;
|
|
162
164
|
if (strictFindings.length && !args.events) {
|
|
163
165
|
context.writeStderr(`Strict exit: ${strictFindings.length} requested source state${strictFindings.length === 1 ? '' : 's'} were incomplete.\n`);
|
|
164
166
|
}
|
|
165
|
-
|
|
166
|
-
context.writeStderr(formatFailPolicyNotice(policyFindings));
|
|
167
|
+
reportCliFailPolicies(policyFindings, context.writeStderr, eventProgress);
|
|
167
168
|
eventProgress.emit({ event: 'completed', exitCode });
|
|
168
169
|
return exitCode;
|
|
169
170
|
}
|
package/cli/lookup-diff.mjs
CHANGED
|
@@ -83,6 +83,7 @@ function lookupComparisonInput(document) {
|
|
|
83
83
|
trusted: null,
|
|
84
84
|
error: '',
|
|
85
85
|
scanDepth: document.mode,
|
|
86
|
+
webCollectionQuality: availability.webCollectionQuality,
|
|
86
87
|
createdDate: lifecycleValue(preferredRegistry, 'createdDate'),
|
|
87
88
|
expiryDate: lifecycleValue(preferredRegistry, 'expiryDate'),
|
|
88
89
|
nameservers: relationship.nameservers.length ? relationship.nameservers : stringList(preferredRegistry.nameservers, 20),
|