@slicedearth/whoisleuth-cli 1.39.1 → 1.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/bin/whoisleuth.mjs +13 -2
- package/cli/archive-inspect.mjs +37 -2
- package/cli/arguments.mjs +361 -256
- package/cli/artifact-verify.mjs +35 -1
- package/cli/bulk-checkpoint.mjs +7 -11
- package/cli/bulk-command-runner.mjs +19 -2
- package/cli/case-pack.mjs +136 -0
- package/cli/ci-report.mjs +73 -0
- package/cli/collection-preflight.mjs +59 -0
- package/cli/command-reference.mjs +395 -0
- package/cli/completion.mjs +33 -10
- package/cli/config-profile.mjs +180 -0
- package/cli/ct-event-intake.mjs +159 -0
- package/cli/discovery-scan-command-runner.mjs +15 -1
- package/cli/domain-control-monitor.mjs +104 -0
- package/cli/domain-control-observations.mjs +235 -0
- package/cli/evidence-command-arguments.mjs +11 -0
- package/cli/evidence-command-runner.mjs +1 -0
- package/cli/external-observation-mapping.mjs +162 -0
- package/cli/fail-policy.mjs +64 -0
- package/cli/formatters/html.mjs +1 -1
- package/cli/investigation-manifest.mjs +113 -0
- package/cli/investigation-run.mjs +112 -0
- package/cli/lookup-brief.mjs +116 -0
- package/cli/lookup-command-runner.mjs +8 -1
- package/cli/lookup-diff.mjs +17 -19
- package/cli/network-command-runner.mjs +4 -1
- package/cli/offline-evidence-review.mjs +105 -8
- package/cli/open-asset-model-bridge.mjs +169 -0
- package/cli/registry-cohort.mjs +121 -0
- package/cli/registry-doctor.mjs +106 -11
- package/cli/registry-fixture-scaffold.mjs +73 -0
- package/cli/runner.mjs +368 -306
- package/docs/cli-reference.md +240 -14
- package/docs/cli.md +42 -7
- package/frontend/src/lib/analysis/acquisition-due-diligence.js +1 -1
- package/frontend/src/lib/analysis/brand-profile-model.js +46 -4
- package/frontend/src/lib/analysis/case-report.js +573 -0
- package/frontend/src/lib/analysis/case-response-model.js +30 -1
- package/frontend/src/lib/analysis/detection-rule-model.js +36 -0
- package/frontend/src/lib/analysis/external-findings-import.js +320 -0
- package/frontend/src/lib/analysis/lookup-decision-support.js +7 -0
- package/frontend/src/lib/analysis/lookup-presentation.js +10 -20
- package/frontend/src/lib/analysis/scoring.js +1 -1
- package/frontend/src/lib/analysis/visualization-models.js +2 -3
- package/frontend/src/lib/analysis/workspace-archive.js +2 -0
- package/lib/abusech-auth.mjs +1 -1
- package/lib/availability.mjs +1 -1
- package/lib/bounded-contract-normalizers.mjs +33 -0
- package/lib/dns-convergence-review.mjs +183 -0
- package/lib/dns-delegation-health.mjs +196 -3
- package/lib/domain-change-packet.mjs +133 -0
- package/lib/domain-change-review.mjs +307 -0
- package/lib/domain-control-flight-recorder.mjs +268 -0
- package/lib/domain-portfolio-review.mjs +151 -0
- package/lib/evidence-export.mjs +1 -1
- package/lib/html-signals.mjs +5 -11
- package/lib/nameserver-preflight-review.mjs +159 -0
- package/lib/page-language-signals.mjs +124 -0
- package/lib/risk-scoring.mjs +42 -1
- package/lib/static-html-analysis.mjs +29 -1
- package/lib/trust-store-comparison.mjs +146 -0
- package/lib/website-technology.mjs +24 -10
- package/lib/zone-intent-review.mjs +534 -0
- package/package.json +1 -1
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import { open } from 'node:fs/promises';
|
|
3
|
+
import { homedir } from 'node:os';
|
|
4
|
+
import { join, resolve } from 'node:path';
|
|
5
|
+
import { CliUsageError } from "./errors.mjs";
|
|
6
|
+
export const CLI_CONFIG_SCHEMA = 'whoisleuth.cli.config';
|
|
7
|
+
export const CLI_CONFIG_VERSION = 1;
|
|
8
|
+
export const MAX_CLI_CONFIG_BYTES = 64 * 1024;
|
|
9
|
+
const SAFE_BOOLEAN_DEFAULTS = new Set(['--no-color', '--summary', '--verbose', '--fast']);
|
|
10
|
+
const SAFE_VALUE_DEFAULTS = new Set(['--concurrency', '--observer', '--vantage']);
|
|
11
|
+
const OPTION_GROUPS = Object.freeze({
|
|
12
|
+
'--no-color': 'colour',
|
|
13
|
+
'--summary': 'detail',
|
|
14
|
+
'--verbose': 'detail',
|
|
15
|
+
'--fast': 'scan',
|
|
16
|
+
'--concurrency': 'concurrency',
|
|
17
|
+
'--observer': 'observer',
|
|
18
|
+
'--vantage': 'vantage',
|
|
19
|
+
});
|
|
20
|
+
function validProfileName(value) {
|
|
21
|
+
return typeof value === 'string' && /^[a-z0-9][a-z0-9_-]{0,39}$/u.test(value);
|
|
22
|
+
}
|
|
23
|
+
function safeArgumentText(value) {
|
|
24
|
+
if (typeof value !== 'string' || !value || value.length > 253 || /[\u0000-\u001f\u007f]/u.test(value)) {
|
|
25
|
+
throw new CliUsageError('CLI profile arguments must be bounded text without control characters.');
|
|
26
|
+
}
|
|
27
|
+
return value;
|
|
28
|
+
}
|
|
29
|
+
function validateProfileArguments(value) {
|
|
30
|
+
if (!Array.isArray(value) || value.length > 16)
|
|
31
|
+
throw new CliUsageError('Each CLI profile is limited to 16 safe default arguments.');
|
|
32
|
+
const argumentsList = value.map(safeArgumentText);
|
|
33
|
+
const validated = [];
|
|
34
|
+
const groups = new Set();
|
|
35
|
+
for (let index = 0; index < argumentsList.length; index += 1) {
|
|
36
|
+
const argument = argumentsList[index];
|
|
37
|
+
if (SAFE_BOOLEAN_DEFAULTS.has(argument)) {
|
|
38
|
+
const group = OPTION_GROUPS[argument];
|
|
39
|
+
if (groups.has(group))
|
|
40
|
+
throw new CliUsageError(`CLI profile contains conflicting ${group} defaults.`);
|
|
41
|
+
groups.add(group);
|
|
42
|
+
validated.push(argument);
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
if (!SAFE_VALUE_DEFAULTS.has(argument)) {
|
|
46
|
+
throw new CliUsageError(`CLI profiles cannot set ${argument}. Only presentation labels, fast mode, and bounded concurrency are supported.`);
|
|
47
|
+
}
|
|
48
|
+
const next = argumentsList[++index];
|
|
49
|
+
if (!next || next.startsWith('-'))
|
|
50
|
+
throw new CliUsageError(`${argument} in a CLI profile requires one value.`);
|
|
51
|
+
const group = OPTION_GROUPS[argument];
|
|
52
|
+
if (groups.has(group))
|
|
53
|
+
throw new CliUsageError(`CLI profile contains duplicate ${group} defaults.`);
|
|
54
|
+
if (argument === '--concurrency' && (!/^\d+$/u.test(next) || Number(next) < 1 || Number(next) > 8)) {
|
|
55
|
+
throw new CliUsageError('CLI profile concurrency must be an integer from 1 to 8.');
|
|
56
|
+
}
|
|
57
|
+
groups.add(group);
|
|
58
|
+
validated.push(argument, next);
|
|
59
|
+
}
|
|
60
|
+
return validated;
|
|
61
|
+
}
|
|
62
|
+
function parseProfileDocument(input) {
|
|
63
|
+
if (Buffer.byteLength(input, 'utf8') > MAX_CLI_CONFIG_BYTES)
|
|
64
|
+
throw new CliUsageError('CLI configuration is limited to 64 KiB.');
|
|
65
|
+
let parsed;
|
|
66
|
+
try {
|
|
67
|
+
parsed = JSON.parse(input.replace(/^\uFEFF/u, ''));
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
throw new CliUsageError('CLI configuration must be valid JSON.');
|
|
71
|
+
}
|
|
72
|
+
const root = parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {};
|
|
73
|
+
if (root.schema !== CLI_CONFIG_SCHEMA || root.version !== CLI_CONFIG_VERSION || !root.profiles || typeof root.profiles !== 'object' || Array.isArray(root.profiles)) {
|
|
74
|
+
throw new CliUsageError(`CLI configuration must use ${CLI_CONFIG_SCHEMA} version ${CLI_CONFIG_VERSION}.`);
|
|
75
|
+
}
|
|
76
|
+
if (Object.keys(root).some((key) => !['schema', 'version', 'defaultProfile', 'profiles'].includes(key)))
|
|
77
|
+
throw new CliUsageError('CLI configuration contains an unsupported root field.');
|
|
78
|
+
if (root.defaultProfile !== undefined && !validProfileName(root.defaultProfile))
|
|
79
|
+
throw new CliUsageError('CLI configuration defaultProfile is invalid.');
|
|
80
|
+
const profileEntries = Object.entries(root.profiles);
|
|
81
|
+
if (profileEntries.length < 1 || profileEntries.length > 20)
|
|
82
|
+
throw new CliUsageError('CLI configuration requires from 1 to 20 profiles.');
|
|
83
|
+
const profiles = Object.fromEntries(profileEntries.map(([name, value]) => {
|
|
84
|
+
if (!validProfileName(name))
|
|
85
|
+
throw new CliUsageError(`CLI profile name "${name}" is invalid.`);
|
|
86
|
+
const profile = value && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
87
|
+
if (Object.keys(profile).some((key) => key !== 'arguments'))
|
|
88
|
+
throw new CliUsageError(`CLI profile "${name}" contains an unsupported field.`);
|
|
89
|
+
return [name, Object.freeze({ arguments: Object.freeze(validateProfileArguments(profile.arguments)) })];
|
|
90
|
+
}));
|
|
91
|
+
return Object.freeze({ schema: CLI_CONFIG_SCHEMA, version: CLI_CONFIG_VERSION, ...(root.defaultProfile ? { defaultProfile: root.defaultProfile } : {}), profiles: Object.freeze(profiles) });
|
|
92
|
+
}
|
|
93
|
+
async function readConfigFile(path) {
|
|
94
|
+
const handle = await open(path, 'r');
|
|
95
|
+
try {
|
|
96
|
+
const metadata = await handle.stat();
|
|
97
|
+
if (metadata.size > MAX_CLI_CONFIG_BYTES)
|
|
98
|
+
throw new CliUsageError('CLI configuration is limited to 64 KiB.');
|
|
99
|
+
return await handle.readFile({ encoding: 'utf8' });
|
|
100
|
+
}
|
|
101
|
+
finally {
|
|
102
|
+
await handle.close();
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
function suppliedGroups(argumentsList) {
|
|
106
|
+
const groups = new Set();
|
|
107
|
+
for (const argument of argumentsList) {
|
|
108
|
+
const group = OPTION_GROUPS[argument];
|
|
109
|
+
if (group)
|
|
110
|
+
groups.add(group);
|
|
111
|
+
if (argument === '--deep')
|
|
112
|
+
groups.add('scan');
|
|
113
|
+
}
|
|
114
|
+
return groups;
|
|
115
|
+
}
|
|
116
|
+
export async function resolveCliProfileArguments(argv, options = {}) {
|
|
117
|
+
const retained = [];
|
|
118
|
+
let configPath = null;
|
|
119
|
+
let profileName = null;
|
|
120
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
121
|
+
const argument = argv[index];
|
|
122
|
+
if (argument === undefined)
|
|
123
|
+
break;
|
|
124
|
+
if (argument === '--config' || argument === '--profile') {
|
|
125
|
+
const next = argv[++index];
|
|
126
|
+
if (!next || next.startsWith('-'))
|
|
127
|
+
throw new CliUsageError(`${argument} requires one value.`);
|
|
128
|
+
if (argument === '--config') {
|
|
129
|
+
if (configPath !== null)
|
|
130
|
+
throw new CliUsageError('--config may be supplied only once.');
|
|
131
|
+
configPath = next;
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
if (profileName !== null)
|
|
135
|
+
throw new CliUsageError('--profile may be supplied only once.');
|
|
136
|
+
if (!validProfileName(next))
|
|
137
|
+
throw new CliUsageError('--profile requires a lower-case name of at most 40 characters.');
|
|
138
|
+
profileName = next;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
else
|
|
142
|
+
retained.push(argument);
|
|
143
|
+
}
|
|
144
|
+
if (!configPath && !profileName)
|
|
145
|
+
return retained;
|
|
146
|
+
const environment = options.environment || process.env;
|
|
147
|
+
const baseDirectory = environment.XDG_CONFIG_HOME || join(options.homeDirectory || homedir(), '.config');
|
|
148
|
+
const selectedPath = resolve(configPath || join(baseDirectory, 'whoisleuth', 'config.json'));
|
|
149
|
+
let input;
|
|
150
|
+
try {
|
|
151
|
+
input = await (options.readConfig || readConfigFile)(selectedPath);
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
if (error instanceof CliUsageError)
|
|
155
|
+
throw error;
|
|
156
|
+
throw new CliUsageError(`Could not read CLI configuration from ${selectedPath}.`);
|
|
157
|
+
}
|
|
158
|
+
const document = parseProfileDocument(input);
|
|
159
|
+
const selectedName = profileName || document.defaultProfile;
|
|
160
|
+
if (!selectedName)
|
|
161
|
+
throw new CliUsageError('CLI configuration requires --profile or a defaultProfile.');
|
|
162
|
+
const selected = document.profiles[selectedName];
|
|
163
|
+
if (!selected)
|
|
164
|
+
throw new CliUsageError(`CLI profile "${selectedName}" was not found.`);
|
|
165
|
+
if (!retained.length)
|
|
166
|
+
throw new CliUsageError('A CLI profile must accompany a command.');
|
|
167
|
+
const explicitGroups = suppliedGroups(retained.slice(1));
|
|
168
|
+
const defaults = [];
|
|
169
|
+
for (let index = 0; index < selected.arguments.length; index += 1) {
|
|
170
|
+
const argument = selected.arguments[index];
|
|
171
|
+
const group = OPTION_GROUPS[argument];
|
|
172
|
+
const hasValue = SAFE_VALUE_DEFAULTS.has(argument);
|
|
173
|
+
const value = hasValue ? selected.arguments[++index] : undefined;
|
|
174
|
+
if (explicitGroups.has(group))
|
|
175
|
+
continue;
|
|
176
|
+
defaults.push(argument, ...(value ? [value] : []));
|
|
177
|
+
}
|
|
178
|
+
return [retained[0], ...defaults, ...retained.slice(1)];
|
|
179
|
+
}
|
|
180
|
+
export { parseProfileDocument };
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { domainToASCII } from 'node:url';
|
|
2
|
+
import { exactKeys, requireBoundedString, requireIsoTimestamp, requireRecord, } from "../lib/bounded-contract-normalizers.mjs";
|
|
3
|
+
export const CT_EVENT_BATCH_SCHEMA = 'whoisleuth.ct-event-batch';
|
|
4
|
+
export const CT_EVENT_BATCH_VERSION = 1;
|
|
5
|
+
export const MAX_CT_EVENT_INPUT_BYTES = 4 * 1024 * 1024;
|
|
6
|
+
export const MAX_CT_EVENTS = 500;
|
|
7
|
+
export const MAX_CT_EVENT_NAMES = 100;
|
|
8
|
+
export const MAX_CT_EXTERNAL_FINDINGS = 100;
|
|
9
|
+
export const MAX_CT_EXTERNAL_DOMAINS = 25;
|
|
10
|
+
export const MAX_CT_EXTERNAL_FINDINGS_PER_DOMAIN = 20;
|
|
11
|
+
const ROOT_KEYS = new Set(['schema', 'version', 'source', 'events']);
|
|
12
|
+
const SOURCE_KEYS = new Set(['name', 'reference', 'collectedAt']);
|
|
13
|
+
const EVENT_KEYS = new Set([
|
|
14
|
+
'logId',
|
|
15
|
+
'observedAt',
|
|
16
|
+
'certificateSha256',
|
|
17
|
+
'dnsNames',
|
|
18
|
+
'issuer',
|
|
19
|
+
'notAfter',
|
|
20
|
+
'completeness',
|
|
21
|
+
'limitations',
|
|
22
|
+
]);
|
|
23
|
+
const SHA256_RE = /^[a-f0-9]{64}$/u;
|
|
24
|
+
function optionalText(value, label, maximum) {
|
|
25
|
+
if (value === undefined || value === null || value === '')
|
|
26
|
+
return null;
|
|
27
|
+
return requireBoundedString(value, label, maximum);
|
|
28
|
+
}
|
|
29
|
+
function domain(value, label) {
|
|
30
|
+
const supplied = requireBoundedString(value, label, 255).toLowerCase().replace(/^\*\./u, '').replace(/\.$/u, '');
|
|
31
|
+
const ascii = domainToASCII(supplied).toLowerCase();
|
|
32
|
+
if (!ascii || ascii.length > 253 || !ascii.includes('.') || ascii.split('.').some((part) => !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/u.test(part))) {
|
|
33
|
+
throw new TypeError(`${label} must be a valid DNS name.`);
|
|
34
|
+
}
|
|
35
|
+
return ascii;
|
|
36
|
+
}
|
|
37
|
+
function boundedLimitations(value, label) {
|
|
38
|
+
if (value === undefined)
|
|
39
|
+
return Object.freeze([]);
|
|
40
|
+
if (!Array.isArray(value) || value.length > 8)
|
|
41
|
+
throw new TypeError(`${label} must contain no more than 8 entries.`);
|
|
42
|
+
return Object.freeze([...new Set(value.map((item, index) => requireBoundedString(item, `${label}[${index}]`, 240)))]);
|
|
43
|
+
}
|
|
44
|
+
export function buildCtEventFindings(inputRaw) {
|
|
45
|
+
const input = requireRecord(inputRaw, 'Certificate event batch');
|
|
46
|
+
if (input.schema !== CT_EVENT_BATCH_SCHEMA || input.version !== CT_EVENT_BATCH_VERSION) {
|
|
47
|
+
throw new TypeError(`Certificate event input must use ${CT_EVENT_BATCH_SCHEMA} version ${CT_EVENT_BATCH_VERSION}.`);
|
|
48
|
+
}
|
|
49
|
+
exactKeys(input, ROOT_KEYS, 'Certificate event batch');
|
|
50
|
+
const sourceInput = requireRecord(input.source, 'source');
|
|
51
|
+
exactKeys(sourceInput, SOURCE_KEYS, 'source');
|
|
52
|
+
const source = Object.freeze({
|
|
53
|
+
name: requireBoundedString(sourceInput.name, 'source.name', 80),
|
|
54
|
+
reference: optionalText(sourceInput.reference, 'source.reference', 500),
|
|
55
|
+
collectedAt: sourceInput.collectedAt === null || sourceInput.collectedAt === undefined
|
|
56
|
+
? null
|
|
57
|
+
: requireIsoTimestamp(sourceInput.collectedAt, 'source.collectedAt'),
|
|
58
|
+
});
|
|
59
|
+
if (!Array.isArray(input.events) || input.events.length < 1 || input.events.length > MAX_CT_EVENTS) {
|
|
60
|
+
throw new TypeError(`events must contain between 1 and ${MAX_CT_EVENTS} entries.`);
|
|
61
|
+
}
|
|
62
|
+
const candidates = input.events.flatMap((raw, eventIndex) => {
|
|
63
|
+
const event = requireRecord(raw, `events[${eventIndex}]`);
|
|
64
|
+
exactKeys(event, EVENT_KEYS, `events[${eventIndex}]`);
|
|
65
|
+
const logId = requireBoundedString(event.logId, `events[${eventIndex}].logId`, 200);
|
|
66
|
+
const observedAt = requireIsoTimestamp(event.observedAt, `events[${eventIndex}].observedAt`);
|
|
67
|
+
const certificateSha256 = requireBoundedString(event.certificateSha256, `events[${eventIndex}].certificateSha256`, 64).toLowerCase();
|
|
68
|
+
if (!SHA256_RE.test(certificateSha256))
|
|
69
|
+
throw new TypeError(`events[${eventIndex}].certificateSha256 must be a SHA-256 hexadecimal digest.`);
|
|
70
|
+
if (!Array.isArray(event.dnsNames) || event.dnsNames.length < 1 || event.dnsNames.length > MAX_CT_EVENT_NAMES) {
|
|
71
|
+
throw new TypeError(`events[${eventIndex}].dnsNames must contain between 1 and ${MAX_CT_EVENT_NAMES} entries.`);
|
|
72
|
+
}
|
|
73
|
+
const names = [...new Set(event.dnsNames.map((name, nameIndex) => domain(name, `events[${eventIndex}].dnsNames[${nameIndex}]`)))].sort();
|
|
74
|
+
const issuer = optionalText(event.issuer, `events[${eventIndex}].issuer`, 160);
|
|
75
|
+
const notAfter = event.notAfter === null || event.notAfter === undefined
|
|
76
|
+
? null
|
|
77
|
+
: requireIsoTimestamp(event.notAfter, `events[${eventIndex}].notAfter`);
|
|
78
|
+
const completeness = event.completeness;
|
|
79
|
+
if (completeness !== 'complete' && completeness !== 'partial') {
|
|
80
|
+
throw new TypeError(`events[${eventIndex}].completeness must be complete or partial.`);
|
|
81
|
+
}
|
|
82
|
+
const limitations = boundedLimitations(event.limitations, `events[${eventIndex}].limitations`);
|
|
83
|
+
return names.map((name) => ({
|
|
84
|
+
domain: name,
|
|
85
|
+
category: 'certificate',
|
|
86
|
+
evidenceClass: 'deployment_observation',
|
|
87
|
+
summary: `Supplied certificate …${certificateSha256.slice(-12)} included ${name} among ${names.length} bounded DNS name${names.length === 1 ? '' : 's'}.`,
|
|
88
|
+
observedAt,
|
|
89
|
+
completeness,
|
|
90
|
+
limitations,
|
|
91
|
+
reference: `Certificate event ${logId}`,
|
|
92
|
+
structuredObservation: {
|
|
93
|
+
sourceSchema: 'whoisleuth.certificate-observation-rows',
|
|
94
|
+
sourceVersion: 1,
|
|
95
|
+
field: 'certificateSha256',
|
|
96
|
+
value: certificateSha256,
|
|
97
|
+
issuer,
|
|
98
|
+
notAfter,
|
|
99
|
+
},
|
|
100
|
+
}));
|
|
101
|
+
});
|
|
102
|
+
const seen = new Set();
|
|
103
|
+
const unique = candidates
|
|
104
|
+
.sort((left, right) => left.domain.localeCompare(right.domain)
|
|
105
|
+
|| left.observedAt.localeCompare(right.observedAt)
|
|
106
|
+
|| left.structuredObservation.value.localeCompare(right.structuredObservation.value))
|
|
107
|
+
.filter((item) => {
|
|
108
|
+
const key = `${item.domain}\u0000${item.observedAt}\u0000${item.structuredObservation.value}`;
|
|
109
|
+
if (seen.has(key))
|
|
110
|
+
return false;
|
|
111
|
+
seen.add(key);
|
|
112
|
+
return true;
|
|
113
|
+
});
|
|
114
|
+
const domainCounts = new Map();
|
|
115
|
+
const acceptedDomains = new Set();
|
|
116
|
+
const selected = [];
|
|
117
|
+
for (const finding of unique) {
|
|
118
|
+
if (!acceptedDomains.has(finding.domain) && acceptedDomains.size >= MAX_CT_EXTERNAL_DOMAINS)
|
|
119
|
+
continue;
|
|
120
|
+
const domainCount = domainCounts.get(finding.domain) ?? 0;
|
|
121
|
+
if (domainCount >= MAX_CT_EXTERNAL_FINDINGS_PER_DOMAIN)
|
|
122
|
+
continue;
|
|
123
|
+
acceptedDomains.add(finding.domain);
|
|
124
|
+
domainCounts.set(finding.domain, domainCount + 1);
|
|
125
|
+
selected.push(finding);
|
|
126
|
+
if (selected.length >= MAX_CT_EXTERNAL_FINDINGS)
|
|
127
|
+
break;
|
|
128
|
+
}
|
|
129
|
+
const truncated = selected.length < unique.length;
|
|
130
|
+
const findings = selected.map((finding) => Object.freeze({
|
|
131
|
+
...finding,
|
|
132
|
+
limitations: Object.freeze([
|
|
133
|
+
...finding.limitations,
|
|
134
|
+
'The supplied event is an observation, not proof that the certificate was served by the named domain or that the domain operator requested it.',
|
|
135
|
+
...(truncated ? [`The batch contained ${unique.length} unique domain observations; this import preserves the first ${MAX_CT_EXTERNAL_FINDINGS} in deterministic order.`] : []),
|
|
136
|
+
].slice(0, 8)),
|
|
137
|
+
}));
|
|
138
|
+
return Object.freeze({
|
|
139
|
+
schema: 'whoisleuth.external-findings',
|
|
140
|
+
schemaVersion: 3,
|
|
141
|
+
source,
|
|
142
|
+
findings: Object.freeze(findings),
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
export function formatCtEventFindings(document) {
|
|
146
|
+
const domains = new Set(document.findings.map((finding) => finding.domain));
|
|
147
|
+
const certificates = new Set(document.findings.map((finding) => finding.structuredObservation.value));
|
|
148
|
+
return [
|
|
149
|
+
'Certificate event intake',
|
|
150
|
+
`Source ${document.source.name}`,
|
|
151
|
+
`Findings ${document.findings.length}`,
|
|
152
|
+
`Domains ${domains.size}`,
|
|
153
|
+
`Certificates ${certificates.size}`,
|
|
154
|
+
'',
|
|
155
|
+
'The JSON form can be imported through the Console external-findings workflow.',
|
|
156
|
+
'A certificate event is a review lead, not proof that the certificate was served or requested by the domain operator.',
|
|
157
|
+
'',
|
|
158
|
+
].join('\n');
|
|
159
|
+
}
|
|
@@ -10,6 +10,8 @@ import { boundedCliErrorMessage, CliUsageError } from "./errors.mjs";
|
|
|
10
10
|
import EXIT_CODES from "./exit-codes.mjs";
|
|
11
11
|
import { formatJsonDocument } from "./formatters/json.mjs";
|
|
12
12
|
import { createCliProgressEvents } from "./progress-events.mjs";
|
|
13
|
+
import { buildCollectionPreflight, formatCollectionPreflight } from "./collection-preflight.mjs";
|
|
14
|
+
import { evaluateCliFailPolicies, formatFailPolicyNotice } from "./fail-policy.mjs";
|
|
13
15
|
async function readAllowlist(source, dependencies, classify) {
|
|
14
16
|
if (!source)
|
|
15
17
|
return new Set();
|
|
@@ -40,6 +42,15 @@ async function runDiscoveryScanCommand(args, dependencies, context) {
|
|
|
40
42
|
if (!candidates.length)
|
|
41
43
|
throw new CliUsageError('discover-scan did not generate any valid candidates to collect.');
|
|
42
44
|
const queries = candidates.map((candidate) => String(candidate.domain));
|
|
45
|
+
if (args.plan) {
|
|
46
|
+
const document = buildCollectionPreflight({
|
|
47
|
+
command: 'discover-scan', targetCount: queries.length, targetLimit: args.deep ? 50 : 500,
|
|
48
|
+
deep: args.deep, concurrency: args.concurrency, output: args.output, checkpoint: false,
|
|
49
|
+
customResolvers: Boolean(args.resolverText), allowlist: Boolean(args.allowlistSource),
|
|
50
|
+
});
|
|
51
|
+
context.writeStdout(args.output === 'json' ? formatJsonDocument(document) : context.terminal(formatCollectionPreflight(document), args.color));
|
|
52
|
+
return EXIT_CODES.SUCCESS;
|
|
53
|
+
}
|
|
43
54
|
const classify = dependencies.classifyQuery || classifyQuery;
|
|
44
55
|
let resolverServers = [];
|
|
45
56
|
if (args.resolverText) {
|
|
@@ -132,7 +143,10 @@ async function runDiscoveryScanCommand(args, dependencies, context) {
|
|
|
132
143
|
context.writeStderr(`Checkpoint warning: ${boundedCliErrorMessage(checkpointFailure, 'Checkpoint could not be written')}. Completed output is still available.\n`);
|
|
133
144
|
}
|
|
134
145
|
}
|
|
135
|
-
const
|
|
146
|
+
const policyFindings = evaluateCliFailPolicies(document, args.failOn || []);
|
|
147
|
+
if (policyFindings.length && !eventProgress.enabled)
|
|
148
|
+
context.writeStderr(formatFailPolicyNotice(policyFindings));
|
|
149
|
+
const exitCode = checkpointFailure || items.some((item) => !item.ok) || policyFindings.length
|
|
136
150
|
? EXIT_CODES.PARTIAL_FAILURE
|
|
137
151
|
: EXIT_CODES.SUCCESS;
|
|
138
152
|
eventProgress.emit({ event: 'completed', exitCode });
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { classifyQuery } from "../lib/classify.mjs";
|
|
2
|
+
import { buildDomainControlFlightRecorder, } from "../lib/domain-control-flight-recorder.mjs";
|
|
3
|
+
import { verifyDomainControlManifest } from "../lib/domain-control-manifest.mjs";
|
|
4
|
+
import { buildCliLookupDocument } from "./formatters/json.mjs";
|
|
5
|
+
import { buildCliDomainControlReview, } from "./domain-control-observations.mjs";
|
|
6
|
+
import { boundedCliErrorMessage, CliUsageError } from "./errors.mjs";
|
|
7
|
+
export const CLI_DOMAIN_CONTROL_MONITOR_SCHEMA = 'whoisleuth.cli.domain-control-monitor';
|
|
8
|
+
export const CLI_DOMAIN_CONTROL_MONITOR_VERSION = 1;
|
|
9
|
+
export const MAX_DOMAIN_CONTROL_MONITOR_DOMAINS = 20;
|
|
10
|
+
function previousSnapshot(input) {
|
|
11
|
+
if (!input)
|
|
12
|
+
return { observations: [] };
|
|
13
|
+
let parsed;
|
|
14
|
+
try {
|
|
15
|
+
parsed = JSON.parse(input.replace(/^\uFEFF/u, ''));
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
throw new CliUsageError('Previous monitor snapshot must be valid JSON.');
|
|
19
|
+
}
|
|
20
|
+
const root = parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {};
|
|
21
|
+
if (root.schema !== CLI_DOMAIN_CONTROL_MONITOR_SCHEMA || root.version !== CLI_DOMAIN_CONTROL_MONITOR_VERSION || !Array.isArray(root.observations)) {
|
|
22
|
+
throw new CliUsageError(`Previous monitor snapshot must use ${CLI_DOMAIN_CONTROL_MONITOR_SCHEMA} version ${CLI_DOMAIN_CONTROL_MONITOR_VERSION}.`);
|
|
23
|
+
}
|
|
24
|
+
const validated = buildDomainControlFlightRecorder({
|
|
25
|
+
schema: 'whoisleuth.domain-control-flight-recorder.input', version: 1, observations: root.observations, approvedWindows: [],
|
|
26
|
+
});
|
|
27
|
+
void validated;
|
|
28
|
+
return { observations: root.observations };
|
|
29
|
+
}
|
|
30
|
+
export async function runDomainControlMonitor(manifestInput, previousInput, options) {
|
|
31
|
+
const generatedAt = options.now();
|
|
32
|
+
let rawManifest;
|
|
33
|
+
try {
|
|
34
|
+
rawManifest = JSON.parse(manifestInput.replace(/^\uFEFF/u, ''));
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
throw new CliUsageError('Domain-control manifest must be valid JSON.');
|
|
38
|
+
}
|
|
39
|
+
const manifest = verifyDomainControlManifest(rawManifest);
|
|
40
|
+
const entries = manifest.entries.slice(0, Math.min(options.limit, MAX_DOMAIN_CONTROL_MONITOR_DOMAINS));
|
|
41
|
+
const lookups = new Array(entries.length);
|
|
42
|
+
const failures = [];
|
|
43
|
+
let cursor = 0;
|
|
44
|
+
let completed = 0;
|
|
45
|
+
const worker = async () => {
|
|
46
|
+
while (cursor < entries.length) {
|
|
47
|
+
const index = cursor++;
|
|
48
|
+
const entry = entries[index];
|
|
49
|
+
if (!entry)
|
|
50
|
+
break;
|
|
51
|
+
try {
|
|
52
|
+
const classified = classifyQuery(entry.domain);
|
|
53
|
+
const result = await options.executeLookup(classified, { fast: false, compact: false, ...(options.signal ? { signal: options.signal } : {}) });
|
|
54
|
+
lookups[index] = buildCliLookupDocument(entry.domain, classified, result, generatedAt, 'deep');
|
|
55
|
+
}
|
|
56
|
+
catch (cause) {
|
|
57
|
+
failures.push({ domain: entry.domain, error: boundedCliErrorMessage(cause, 'Lookup failed') });
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
completed += 1;
|
|
61
|
+
options.onSettled?.(completed, entries.length);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
await Promise.all(Array.from({ length: Math.min(options.concurrency, Math.max(entries.length, 1)) }, worker));
|
|
66
|
+
const successful = lookups.filter(Boolean);
|
|
67
|
+
if (!successful.length)
|
|
68
|
+
throw new Error('No domain-control monitor lookup completed successfully.');
|
|
69
|
+
const review = buildCliDomainControlReview(JSON.stringify({
|
|
70
|
+
schema: 'whoisleuth.cli.domain-control-review-input', version: 1, manifest, lookups: successful,
|
|
71
|
+
}), generatedAt);
|
|
72
|
+
const previous = previousSnapshot(previousInput);
|
|
73
|
+
const observations = review.observations;
|
|
74
|
+
const flightRecorder = buildDomainControlFlightRecorder({
|
|
75
|
+
schema: 'whoisleuth.domain-control-flight-recorder.input', version: 1,
|
|
76
|
+
observations: [...previous.observations, ...observations], approvedWindows: [],
|
|
77
|
+
}, generatedAt);
|
|
78
|
+
return Object.freeze({
|
|
79
|
+
schema: CLI_DOMAIN_CONTROL_MONITOR_SCHEMA,
|
|
80
|
+
version: CLI_DOMAIN_CONTROL_MONITOR_VERSION,
|
|
81
|
+
generatedAt,
|
|
82
|
+
manifest: Object.freeze({ digestSha256: manifest.integrity.digestSha256, expiresAt: manifest.expiresAt }),
|
|
83
|
+
collection: Object.freeze({ requested: entries.length, succeeded: observations.length, failed: failures.length, failures: Object.freeze(failures) }),
|
|
84
|
+
observations,
|
|
85
|
+
review: review.review,
|
|
86
|
+
flightRecorder,
|
|
87
|
+
limitations: Object.freeze([
|
|
88
|
+
'This is one bounded run, not a daemon. Scheduling and secure checkpoint retention remain operator-controlled.',
|
|
89
|
+
'The output retains compact normalised observations and errors, not raw RDAP, WHOIS, DNS, HTTP, TLS or page payloads.',
|
|
90
|
+
'A failed source or lookup remains incomplete and does not establish that a previously observed value disappeared.',
|
|
91
|
+
]),
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
export function formatDomainControlMonitor(document) {
|
|
95
|
+
return [
|
|
96
|
+
'One-shot domain control review',
|
|
97
|
+
`Requested ${document.collection.requested}`,
|
|
98
|
+
`Succeeded ${document.collection.succeeded}`,
|
|
99
|
+
`Failed ${document.collection.failed}`,
|
|
100
|
+
`Review ${document.review.state}`,
|
|
101
|
+
`Unexpected ${document.flightRecorder.summary.unexpectedChanges}`,
|
|
102
|
+
'',
|
|
103
|
+
].join('\n');
|
|
104
|
+
}
|