@slicedearth/whoisleuth-cli 1.34.0 → 1.36.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/cli/arguments.mjs +92 -0
- package/cli/artifact-verify.mjs +4 -0
- package/cli/bulk.mjs +1 -2
- package/cli/completion.mjs +19 -0
- package/cli/export-evidence.mjs +9 -2
- package/cli/formatters/terminal.mjs +7 -1
- package/cli/investigation-plan.mjs +112 -0
- package/cli/mail-review.mjs +47 -2
- package/cli/offline-evidence-review.mjs +141 -0
- package/cli/registry-support.mjs +1 -4
- package/cli/runner.mjs +109 -1
- package/docs/cli.md +102 -9
- package/frontend/src/lib/analysis/bulk-retry-plan.js +3 -10
- package/frontend/src/lib/analysis/bulk-source-coverage.js +76 -0
- package/frontend/src/lib/analysis/http-summary.js +2 -1
- package/frontend/src/lib/analysis/utils.js +7 -40
- package/frontend/src/lib/analysis/website-snapshot-model.js +57 -3
- package/lib/availability.mjs +30 -10
- package/lib/bulk-limits.mjs +13 -0
- package/lib/ct-response-bounds.mjs +9 -0
- package/lib/ct-search.mjs +66 -12
- package/lib/dns-delegation-health.mjs +56 -5
- package/lib/dnssec-evidence-validation.mjs +235 -0
- package/lib/domain-control-manifest.mjs +339 -0
- package/lib/domain-posture.mjs +2 -1
- package/lib/encrypted-dns-contract.mjs +81 -0
- package/lib/evidence-export.mjs +17 -3
- package/lib/evidence-report.mjs +3 -0
- package/lib/favicon.mjs +3 -8
- package/lib/html-signals.mjs +4 -0
- package/lib/http-evidence-bounds.mjs +4 -0
- package/lib/http-intelligence.mjs +2 -1
- package/lib/ip-prefix.mjs +54 -0
- package/lib/local-geoip-evidence.mjs +89 -0
- package/lib/outbound-identity.mjs +9 -0
- package/lib/outbound-request-bounds.mjs +9 -0
- package/lib/perceptual-hash-comparison.mjs +34 -0
- package/lib/perceptual-hash.mjs +3 -31
- package/lib/project-metadata.mjs +8 -0
- package/lib/provider-policy-admission.mjs +49 -0
- package/lib/rdap-search-workbench.mjs +157 -0
- package/lib/rdap.mjs +7 -2
- package/lib/registry-capabilities.mjs +31 -1
- package/lib/registry-capability-catalogue.mjs +11 -10
- package/lib/response-policy.mjs +81 -6
- package/lib/rpki-evidence.mjs +105 -0
- package/lib/safe-fetch.mjs +2 -1
- package/lib/semantic-version.mjs +54 -0
- package/lib/static-html-analysis.mjs +27 -0
- package/lib/threatfox-intelligence.mjs +10 -5
- package/lib/tlsa-evidence.mjs +194 -0
- package/lib/urlhaus-intelligence.mjs +10 -5
- package/lib/urlscan-intelligence.mjs +10 -5
- package/lib/website-security-posture.mjs +11 -4
- package/lib/whois-chain.mjs +14 -152
- package/lib/whois.mjs +2 -2
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -58,6 +58,8 @@ node bin/whoisleuth.mts discover example.test --dictionary private-terms.txt --s
|
|
|
58
58
|
node bin/whoisleuth.mts discover-scan example.test --scan-limit 50 --checkpoint candidate-scan.json --json
|
|
59
59
|
node bin/whoisleuth.mts page-compare official.json candidate.json --json
|
|
60
60
|
node bin/whoisleuth.mts mail-review bulk.json --json
|
|
61
|
+
node bin/whoisleuth.mts domain-control domain-control-input.json --json
|
|
62
|
+
node bin/whoisleuth.mts workflow-plan domain-triage example.test --json
|
|
61
63
|
node bin/whoisleuth.mts diff first-lookup.json second-lookup.json --json
|
|
62
64
|
node bin/whoisleuth.mts timeline first-observation.json second-observation.json latest-observation.json --json
|
|
63
65
|
node bin/whoisleuth.mts registry-support example.test --json
|
package/cli/arguments.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { parseInspectArchiveArguments, parseSignArtifactArguments, parseVerifySignatureArguments, } from "./evidence-command-arguments.mjs";
|
|
2
2
|
import { CliUsageError } from "./errors.mjs";
|
|
3
|
+
import { INVESTIGATION_PLAN_RECIPES, } from "./investigation-plan.mjs";
|
|
3
4
|
const MAX_CLI_ARGUMENTS = 32;
|
|
4
5
|
const MAX_CLI_ARGUMENT_LENGTH = 1024;
|
|
5
6
|
const CLI_COMMANDS = [
|
|
@@ -25,6 +26,9 @@ const CLI_COMMANDS = [
|
|
|
25
26
|
'compare',
|
|
26
27
|
'page-compare',
|
|
27
28
|
'mail-review',
|
|
29
|
+
'review-evidence',
|
|
30
|
+
'domain-control',
|
|
31
|
+
'workflow-plan',
|
|
28
32
|
'diff',
|
|
29
33
|
'timeline',
|
|
30
34
|
'export',
|
|
@@ -146,6 +150,12 @@ function parseCliArgumentsCore(argv) {
|
|
|
146
150
|
return parsePageCompareArguments(argv.slice(1));
|
|
147
151
|
if (command === 'mail-review')
|
|
148
152
|
return parseMailReviewArguments(argv.slice(1));
|
|
153
|
+
if (command === 'review-evidence')
|
|
154
|
+
return parseReviewEvidenceArguments(argv.slice(1));
|
|
155
|
+
if (command === 'domain-control')
|
|
156
|
+
return parseDomainControlArguments(argv.slice(1));
|
|
157
|
+
if (command === 'workflow-plan')
|
|
158
|
+
return parseWorkflowPlanArguments(argv.slice(1));
|
|
149
159
|
if (command === 'diff')
|
|
150
160
|
return parseDiffArguments(argv.slice(1));
|
|
151
161
|
if (command === 'timeline')
|
|
@@ -893,6 +903,88 @@ function parseMailReviewArguments(argv) {
|
|
|
893
903
|
throw new CliUsageError('--quiet cannot be combined with machine-readable output.');
|
|
894
904
|
return { action: 'mail-review', source, output, quiet, color };
|
|
895
905
|
}
|
|
906
|
+
function parseReviewEvidenceArguments(argv) {
|
|
907
|
+
let source = null;
|
|
908
|
+
let output = 'terminal';
|
|
909
|
+
let quiet = false;
|
|
910
|
+
let color = true;
|
|
911
|
+
for (const argument of argv) {
|
|
912
|
+
if (argument === '--json') {
|
|
913
|
+
if (output !== 'terminal')
|
|
914
|
+
throw new CliUsageError('--json may be supplied only once.');
|
|
915
|
+
output = 'json';
|
|
916
|
+
}
|
|
917
|
+
else if (argument === '--quiet')
|
|
918
|
+
quiet = true;
|
|
919
|
+
else if (argument === '--no-color')
|
|
920
|
+
color = false;
|
|
921
|
+
else if (argument.startsWith('-'))
|
|
922
|
+
throw new CliUsageError(`Unknown option "${argument}".`);
|
|
923
|
+
else if (source === null)
|
|
924
|
+
source = argument;
|
|
925
|
+
else
|
|
926
|
+
throw new CliUsageError('review-evidence accepts one optional versioned JSON input file.');
|
|
927
|
+
}
|
|
928
|
+
if (quiet && output !== 'terminal')
|
|
929
|
+
throw new CliUsageError('--quiet cannot be combined with machine-readable output.');
|
|
930
|
+
return { action: 'review-evidence', source, output, quiet, color };
|
|
931
|
+
}
|
|
932
|
+
function parseDomainControlArguments(argv) {
|
|
933
|
+
let source = null;
|
|
934
|
+
let output = 'terminal';
|
|
935
|
+
let quiet = false;
|
|
936
|
+
let color = true;
|
|
937
|
+
for (const argument of argv) {
|
|
938
|
+
if (argument === '--json') {
|
|
939
|
+
if (output !== 'terminal')
|
|
940
|
+
throw new CliUsageError('--json may be supplied only once.');
|
|
941
|
+
output = 'json';
|
|
942
|
+
}
|
|
943
|
+
else if (argument === '--quiet')
|
|
944
|
+
quiet = true;
|
|
945
|
+
else if (argument === '--no-color')
|
|
946
|
+
color = false;
|
|
947
|
+
else if (argument.startsWith('-'))
|
|
948
|
+
throw new CliUsageError(`Unknown option "${argument}".`);
|
|
949
|
+
else if (source === null)
|
|
950
|
+
source = argument;
|
|
951
|
+
else
|
|
952
|
+
throw new CliUsageError('domain-control accepts one optional versioned JSON input file.');
|
|
953
|
+
}
|
|
954
|
+
if (quiet && output !== 'terminal')
|
|
955
|
+
throw new CliUsageError('--quiet cannot be combined with machine-readable output.');
|
|
956
|
+
return { action: 'domain-control', source, output, quiet, color };
|
|
957
|
+
}
|
|
958
|
+
function parseWorkflowPlanArguments(argv) {
|
|
959
|
+
const positional = [];
|
|
960
|
+
let output = 'terminal';
|
|
961
|
+
let quiet = false;
|
|
962
|
+
let color = true;
|
|
963
|
+
for (const argument of argv) {
|
|
964
|
+
if (argument === '--json') {
|
|
965
|
+
if (output !== 'terminal')
|
|
966
|
+
throw new CliUsageError('--json may be supplied only once.');
|
|
967
|
+
output = 'json';
|
|
968
|
+
}
|
|
969
|
+
else if (argument === '--quiet')
|
|
970
|
+
quiet = true;
|
|
971
|
+
else if (argument === '--no-color')
|
|
972
|
+
color = false;
|
|
973
|
+
else if (argument.startsWith('-'))
|
|
974
|
+
throw new CliUsageError(`Unknown option "${argument}".`);
|
|
975
|
+
else
|
|
976
|
+
positional.push(argument);
|
|
977
|
+
}
|
|
978
|
+
if (positional.length !== 2)
|
|
979
|
+
throw new CliUsageError('workflow-plan requires one fixed recipe and one subject.');
|
|
980
|
+
const recipe = positional[0];
|
|
981
|
+
if (!INVESTIGATION_PLAN_RECIPES.includes(recipe)) {
|
|
982
|
+
throw new CliUsageError(`workflow-plan recipe must be one of: ${INVESTIGATION_PLAN_RECIPES.join(', ')}.`);
|
|
983
|
+
}
|
|
984
|
+
if (quiet && output !== 'terminal')
|
|
985
|
+
throw new CliUsageError('--quiet cannot be combined with machine-readable output.');
|
|
986
|
+
return { action: 'workflow-plan', recipe: recipe, subject: positional[1], output, quiet, color };
|
|
987
|
+
}
|
|
896
988
|
function parseDiffArguments(argv) {
|
|
897
989
|
let leftSource = null;
|
|
898
990
|
let rightSource = null;
|
package/cli/artifact-verify.mjs
CHANGED
|
@@ -8,6 +8,7 @@ import { decryptWorkspaceArchive, inspectEncryptedWorkspaceArchive, isEncryptedW
|
|
|
8
8
|
import { WORKSPACE_ARCHIVE_SCHEMA, readWorkspaceArchive, } from "../frontend/src/lib/analysis/workspace-archive.js";
|
|
9
9
|
import { sha256ArtifactDigest } from "../frontend/src/lib/analysis/artifact-integrity.js";
|
|
10
10
|
import { SAVED_LOOKUP_SCHEMA, SAVED_LOOKUP_SCHEMA_VERSION, parseSavedLookupDocument, } from "./saved-lookup.mjs";
|
|
11
|
+
import { DOMAIN_CONTROL_MANIFEST_SCHEMA, DOMAIN_CONTROL_MANIFEST_VERSION, verifyDomainControlManifest, } from "../lib/domain-control-manifest.mjs";
|
|
11
12
|
export const OFFLINE_ARTIFACT_VERIFICATION_SCHEMA = 'whoisleuth.offline-artifact-verification';
|
|
12
13
|
export const OFFLINE_ARTIFACT_VERIFICATION_VERSION = 1;
|
|
13
14
|
export const MAX_OFFLINE_ARTIFACT_BYTES = 15 * 1024 * 1024;
|
|
@@ -17,6 +18,7 @@ const SIGNED_ARTIFACT_VERSIONS = Object.freeze({
|
|
|
17
18
|
[BULK_DOMAIN_COMPARISON_SCHEMA]: new Set([BULK_DOMAIN_COMPARISON_VERSION]),
|
|
18
19
|
[BULK_MAIL_EXPOSURE_SCHEMA]: new Set([BULK_MAIL_EXPOSURE_VERSION]),
|
|
19
20
|
[BULK_REVIEW_MANIFEST_SCHEMA]: new Set([BULK_REVIEW_MANIFEST_VERSION]),
|
|
21
|
+
[DOMAIN_CONTROL_MANIFEST_SCHEMA]: new Set([DOMAIN_CONTROL_MANIFEST_VERSION]),
|
|
20
22
|
});
|
|
21
23
|
function record(value) {
|
|
22
24
|
return value && typeof value === 'object' && !Array.isArray(value)
|
|
@@ -94,6 +96,8 @@ async function verifySignedArtifact(raw, value, schema, version) {
|
|
|
94
96
|
if (await sha256ArtifactDigest(unsigned) !== integrity.digestSha256) {
|
|
95
97
|
throw new TypeError('The signed review artifact failed its SHA-256 integrity check.');
|
|
96
98
|
}
|
|
99
|
+
if (schema === DOMAIN_CONTROL_MANIFEST_SCHEMA)
|
|
100
|
+
verifyDomainControlManifest(value);
|
|
97
101
|
return Object.freeze({
|
|
98
102
|
schema: OFFLINE_ARTIFACT_VERIFICATION_SCHEMA,
|
|
99
103
|
version: OFFLINE_ARTIFACT_VERIFICATION_VERSION,
|
package/cli/bulk.mjs
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
2
|
import { abortable } from "../lib/abort.mjs";
|
|
3
3
|
import { CliUsageError } from "./arguments.mjs";
|
|
4
|
+
import { MAX_DEEP_BULK_QUERIES, MAX_FAST_BULK_QUERIES, } from "../lib/bulk-limits.mjs";
|
|
4
5
|
const MAX_BULK_INPUT_BYTES = 1024 * 1024;
|
|
5
|
-
const MAX_FAST_BULK_QUERIES = 500;
|
|
6
|
-
const MAX_DEEP_BULK_QUERIES = 50;
|
|
7
6
|
async function readTextStreamBounded(stream, limit = MAX_BULK_INPUT_BYTES) {
|
|
8
7
|
if (!stream || stream.isTTY)
|
|
9
8
|
return '';
|
package/cli/completion.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { CLI_COMMANDS } from "./arguments.mjs";
|
|
2
|
+
import { INVESTIGATION_PLAN_RECIPES } from "./investigation-plan.mjs";
|
|
2
3
|
const COMMON_OPTIONS = Object.freeze(['--help', '--output', '--force']);
|
|
3
4
|
const OPTIONS_BY_COMMAND = Object.freeze({
|
|
4
5
|
lookup: ['--json', '--markdown', '--html', '--fast', '--deep', '--plan', '--summary', '--verbose', '--strict-exit', '--events', '--quiet', '--no-color'],
|
|
@@ -19,6 +20,9 @@ const OPTIONS_BY_COMMAND = Object.freeze({
|
|
|
19
20
|
compare: ['--json', '--quiet', '--no-color'],
|
|
20
21
|
'page-compare': ['--json', '--quiet', '--no-color'],
|
|
21
22
|
'mail-review': ['--json', '--quiet', '--no-color'],
|
|
23
|
+
'review-evidence': ['--json', '--quiet', '--no-color'],
|
|
24
|
+
'domain-control': ['--json', '--quiet', '--no-color'],
|
|
25
|
+
'workflow-plan': ['--json', '--quiet', '--no-color'],
|
|
22
26
|
diff: ['--json', '--quiet', '--no-color'],
|
|
23
27
|
timeline: ['--json', '--quiet', '--no-color'],
|
|
24
28
|
export: ['--markdown', '--html', '--compact'],
|
|
@@ -46,6 +50,9 @@ const COMMAND_DESCRIPTIONS = Object.freeze({
|
|
|
46
50
|
compare: 'Compare registry publications in one lookup',
|
|
47
51
|
'page-compare': 'Compare saved static page evidence',
|
|
48
52
|
'mail-review': 'Review saved passive mail evidence',
|
|
53
|
+
'review-evidence': 'Review supplied evidence offline',
|
|
54
|
+
'domain-control': 'Build or review a domain control manifest',
|
|
55
|
+
'workflow-plan': 'Plan a fixed investigation recipe',
|
|
49
56
|
diff: 'Compare two saved domain lookups',
|
|
50
57
|
timeline: 'Build same-domain history from saved lookups',
|
|
51
58
|
export: 'Convert a lookup to an evidence report',
|
|
@@ -99,6 +106,10 @@ _whoisleuth_completion() {
|
|
|
99
106
|
COMPREPLY=( $(compgen -W "bash zsh fish powershell" -- "\${current}") )
|
|
100
107
|
return
|
|
101
108
|
fi
|
|
109
|
+
if [[ "\${command}" == "workflow-plan" && \${COMP_CWORD} -eq 2 ]]; then
|
|
110
|
+
COMPREPLY=( $(compgen -W "${INVESTIGATION_PLAN_RECIPES.join(' ')}" -- "\${current}") )
|
|
111
|
+
return
|
|
112
|
+
fi
|
|
102
113
|
case "\${previous}" in
|
|
103
114
|
${valueCases}
|
|
104
115
|
${FILE_OPTIONS.join('|')}) COMPREPLY=( $(compgen -f -- "\${current}") ); return ;;
|
|
@@ -132,6 +143,10 @@ _whoisleuth() {
|
|
|
132
143
|
compadd -- bash zsh fish powershell
|
|
133
144
|
return
|
|
134
145
|
fi
|
|
146
|
+
if [[ "\${command}" == "workflow-plan" && CURRENT -eq 3 ]]; then
|
|
147
|
+
compadd -- ${INVESTIGATION_PLAN_RECIPES.join(' ')}
|
|
148
|
+
return
|
|
149
|
+
fi
|
|
135
150
|
case "\${previous}" in
|
|
136
151
|
${valueCases}
|
|
137
152
|
${FILE_OPTIONS.join('|')}) _files; return ;;
|
|
@@ -165,6 +180,7 @@ complete -c whoisleuth -n '__fish_use_subcommand' -l help
|
|
|
165
180
|
complete -c whoisleuth -n '__fish_use_subcommand' -l version
|
|
166
181
|
${commandLines.join('\n')}
|
|
167
182
|
complete -c whoisleuth -n '__fish_seen_subcommand_from completion' -a 'bash zsh fish powershell'
|
|
183
|
+
complete -c whoisleuth -n '__fish_seen_subcommand_from workflow-plan' -a '${INVESTIGATION_PLAN_RECIPES.join(' ')}'
|
|
168
184
|
${optionLines.join('\n')}
|
|
169
185
|
${valueLines.join('\n')}
|
|
170
186
|
`;
|
|
@@ -184,6 +200,7 @@ ${Object.entries(commandOptions).map(([command, options]) => ` '${command}' =
|
|
|
184
200
|
$values = @{
|
|
185
201
|
${Object.entries(VALUE_OPTIONS).map(([option, values]) => ` '${option}' = @(${values.map((value) => `'${value}'`).join(', ')})`).join('\n')}
|
|
186
202
|
'completion' = @('bash', 'zsh', 'fish', 'powershell')
|
|
203
|
+
'workflow-plan' = @(${INVESTIGATION_PLAN_RECIPES.map((value) => `'${value}'`).join(', ')})
|
|
187
204
|
}
|
|
188
205
|
$elements = @($commandAst.CommandElements | ForEach-Object { $_.Extent.Text })
|
|
189
206
|
$command = if ($elements.Count -gt 1) { $elements[1] } else { '' }
|
|
@@ -192,6 +209,8 @@ ${Object.entries(VALUE_OPTIONS).map(([option, values]) => ` '${option}' = @($
|
|
|
192
209
|
@($commands) + @('--help', '--version')
|
|
193
210
|
} elseif ($command -eq 'completion' -and $elements.Count -le 3) {
|
|
194
211
|
$values['completion']
|
|
212
|
+
} elseif ($command -eq 'workflow-plan' -and $elements.Count -le 3) {
|
|
213
|
+
$values['workflow-plan']
|
|
195
214
|
} elseif ($values.ContainsKey($previous)) {
|
|
196
215
|
$values[$previous]
|
|
197
216
|
} elseif ($options.ContainsKey($command)) {
|
package/cli/export-evidence.mjs
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
import { createRequire } from 'node:module';
|
|
1
2
|
import { projectCliLookupComparisonInput } from "./compare.mjs";
|
|
2
3
|
import { parseSavedLookupDocument } from "./saved-lookup.mjs";
|
|
4
|
+
const require = createRequire(import.meta.url);
|
|
5
|
+
const { version: APPLICATION_VERSION } = require('../package.json');
|
|
3
6
|
function objectOrNull(value) {
|
|
4
7
|
return value && typeof value === 'object' && !Array.isArray(value) ? value : null;
|
|
5
8
|
}
|
|
@@ -14,7 +17,11 @@ function buildCliEvidenceExport(text, evidenceModule, generatedAt = new Date().t
|
|
|
14
17
|
throw new TypeError('Lookup evidence export dependency is required.');
|
|
15
18
|
}
|
|
16
19
|
const buildLookupEvidence = dependency.buildLookupEvidence;
|
|
17
|
-
const result = objectOrNull(buildLookupEvidence(source, {
|
|
20
|
+
const result = objectOrNull(buildLookupEvidence(source, {
|
|
21
|
+
generatedAt,
|
|
22
|
+
idnAnalysis: null,
|
|
23
|
+
applicationVersion: APPLICATION_VERSION,
|
|
24
|
+
}));
|
|
18
25
|
if (!result
|
|
19
26
|
|| result.schema !== dependency.LOOKUP_EVIDENCE_SCHEMA
|
|
20
27
|
|| result.schemaVersion !== dependency.LOOKUP_EVIDENCE_SCHEMA_VERSION) {
|
|
@@ -25,4 +32,4 @@ function buildCliEvidenceExport(text, evidenceModule, generatedAt = new Date().t
|
|
|
25
32
|
function formatCliEvidenceExport(document, compact = false) {
|
|
26
33
|
return `${JSON.stringify(document, null, compact ? 0 : 2)}\n`;
|
|
27
34
|
}
|
|
28
|
-
export { buildCliEvidenceExport, formatCliEvidenceExport };
|
|
35
|
+
export { APPLICATION_VERSION, buildCliEvidenceExport, formatCliEvidenceExport };
|
|
@@ -271,7 +271,6 @@ function formatTerminalRegistrySupport(document) {
|
|
|
271
271
|
`WHOIS scope ${supportLabel(whois.queryScope)}`,
|
|
272
272
|
`WHOIS encoding ${supportLabel(whois.encodingProfile)}`,
|
|
273
273
|
`WHOIS parser ${supportLabel(whois.parserProfile)}`,
|
|
274
|
-
`Fallback ${profile.fallbackProfile ? supportLabel(profile.fallbackProfile) : 'None'}`,
|
|
275
274
|
`Fixture states ${fixtures.length ? fixtures.map((value) => supportLabel(value)).join(', ') : 'None documented'}`,
|
|
276
275
|
];
|
|
277
276
|
for (const file of files)
|
|
@@ -302,6 +301,11 @@ function formatTerminalBulk(items, metadata) {
|
|
|
302
301
|
}
|
|
303
302
|
function formatTerminalCtSearch(document) {
|
|
304
303
|
const matches = Array.isArray(document.matches) ? document.matches : [];
|
|
304
|
+
const certificateGroups = Array.isArray(document.certificateGroups) ? document.certificateGroups : [];
|
|
305
|
+
const crossDomainGroups = certificateGroups.filter((value) => {
|
|
306
|
+
const group = terminalRecord(value);
|
|
307
|
+
return Array.isArray(group.domains) && group.domains.length > 1;
|
|
308
|
+
});
|
|
305
309
|
const visible = matches.slice(0, MAX_CT_TERMINAL_MATCHES);
|
|
306
310
|
const observation = terminalRecord(document.observation);
|
|
307
311
|
const lines = [
|
|
@@ -310,7 +314,9 @@ function formatTerminalCtSearch(document) {
|
|
|
310
314
|
`Certificates ${safeTerminalValue(document.certCount, '0')}`,
|
|
311
315
|
`Observed hosts ${safeTerminalValue(Array.isArray(document.domains) ? document.domains.length : 0, '0')}`,
|
|
312
316
|
`Matches ${safeTerminalValue(matches.length, '0')}`,
|
|
317
|
+
`Issuance groups ${safeTerminalValue(certificateGroups.length, '0')} (${safeTerminalValue(crossDomainGroups.length, '0')} cross-domain)`,
|
|
313
318
|
`Truncated ${document.truncated ? 'Yes' : 'No'}`,
|
|
319
|
+
`Group cap ${document.certificateGroupsTruncated ? 'Reached' : 'Not reached'}`,
|
|
314
320
|
'',
|
|
315
321
|
];
|
|
316
322
|
if (!visible.length) {
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { domainToASCII } from 'node:url';
|
|
2
|
+
export const CLI_INVESTIGATION_PLAN_SCHEMA = 'whoisleuth.cli.investigation-plan';
|
|
3
|
+
export const CLI_INVESTIGATION_PLAN_VERSION = 1;
|
|
4
|
+
export const INVESTIGATION_PLAN_RECIPES = [
|
|
5
|
+
'domain-triage',
|
|
6
|
+
'lookalike-review',
|
|
7
|
+
'owned-domain-review',
|
|
8
|
+
'historical-comparison',
|
|
9
|
+
];
|
|
10
|
+
const RECIPES = Object.freeze({
|
|
11
|
+
'domain-triage': Object.freeze({
|
|
12
|
+
label: 'New domain triage',
|
|
13
|
+
requiresDomain: true,
|
|
14
|
+
objective: 'Collect and preserve a source-aware registration, DNS, HTTP, TLS, page, and network-context review.',
|
|
15
|
+
steps: (domain) => Object.freeze([
|
|
16
|
+
step('collect', 'Collect a Deep lookup', 'lookup', [domain, '--deep', '--json'], 'network', 'network_disclosure', 'whoisleuth.cli.lookup', 'Review source health and limitations before using missing fields.'),
|
|
17
|
+
step('export', 'Create a portable evidence report', 'export', ['<saved-lookup.json>'], 'offline', 'analyst_selection', 'whoisleuth.lookup-evidence', 'Select the reviewed lookup file; the plan never guesses a path.'),
|
|
18
|
+
step('verify', 'Verify the exported artifact', 'verify-artifact', ['<evidence.json>', '--json'], 'offline', 'analyst_selection', 'whoisleuth.offline-artifact-verification', 'Keep verification distinct from a claim that the observations are correct or current.'),
|
|
19
|
+
]),
|
|
20
|
+
}),
|
|
21
|
+
'lookalike-review': Object.freeze({
|
|
22
|
+
label: 'Lookalike candidate review',
|
|
23
|
+
requiresDomain: false,
|
|
24
|
+
objective: 'Generate a bounded candidate queue, collect only the selected scope, and retain a reviewed candidate lookup.',
|
|
25
|
+
steps: (subject) => Object.freeze([
|
|
26
|
+
step('generate', 'Generate candidates offline', 'discover', [subject, '--preset', 'all', '--json'], 'offline', 'none', 'whoisleuth.cli.discover', 'Review mutation families and suppressions before collection.'),
|
|
27
|
+
step('scan', 'Collect a bounded candidate queue', 'discover-scan', [subject, '--fast', '--scan-limit', '50', '--json'], 'network', 'network_disclosure', 'whoisleuth.cli.discovery-scan', 'Fast collection is a triage boundary; partial or inconclusive authority evidence remains explicit.'),
|
|
28
|
+
step('inspect', 'Deep-review one selected candidate', 'lookup', ['<selected-domain>', '--deep', '--json'], 'network', 'analyst_selection', 'whoisleuth.cli.lookup', 'Select a candidate deliberately; generation does not prove registration, control, intent, or maliciousness.'),
|
|
29
|
+
]),
|
|
30
|
+
}),
|
|
31
|
+
'owned-domain-review': Object.freeze({
|
|
32
|
+
label: 'Owned domain posture review',
|
|
33
|
+
requiresDomain: true,
|
|
34
|
+
objective: 'Review current passive posture and compare supplied observations with an analyst-authored control manifest.',
|
|
35
|
+
steps: (domain) => Object.freeze([
|
|
36
|
+
step('posture', 'Collect bounded DNS posture', 'posture', [domain, '--json'], 'network', 'network_disclosure', 'whoisleuth.cli.posture', 'Review mail profile and delegation evidence before interpreting missing records.'),
|
|
37
|
+
step('lookup', 'Collect supporting Deep evidence', 'lookup', [domain, '--deep', '--json'], 'network', 'network_disclosure', 'whoisleuth.cli.lookup', 'Retain separately attributed registration, DNS, TLS, and page observations.'),
|
|
38
|
+
step('manifest', 'Review the domain control manifest', 'domain-control', ['<review-input.json>', '--json'], 'offline', 'analyst_selection', 'whoisleuth.domain-control-review', 'Only complete supplied observations may produce drift.'),
|
|
39
|
+
]),
|
|
40
|
+
}),
|
|
41
|
+
'historical-comparison': Object.freeze({
|
|
42
|
+
label: 'Historical observation comparison',
|
|
43
|
+
requiresDomain: true,
|
|
44
|
+
objective: 'Collect a current observation and compare it with analyst-selected saved observations without merging source states.',
|
|
45
|
+
steps: (domain) => Object.freeze([
|
|
46
|
+
step('current', 'Collect the current lookup', 'lookup', [domain, '--deep', '--json'], 'network', 'network_disclosure', 'whoisleuth.cli.lookup', 'A current request does not refresh or validate older provider-reported history.'),
|
|
47
|
+
step('diff', 'Compare two selected observations', 'diff', ['<previous.json>', '<current.json>', '--json'], 'offline', 'analyst_selection', 'whoisleuth.cli.lookup-diff', 'Equal, different, conflicting, and unavailable evidence remain separate.'),
|
|
48
|
+
step('timeline', 'Build a bounded local timeline', 'timeline', ['<oldest.json>', '<newer.json>', '<current.json>', '--json'], 'offline', 'analyst_selection', 'whoisleuth.cli.lookup-timeline', 'Choose two to twenty same-domain files in chronological scope.'),
|
|
49
|
+
]),
|
|
50
|
+
}),
|
|
51
|
+
});
|
|
52
|
+
function step(id, label, command, args, mode, approval, produces, completion) {
|
|
53
|
+
return Object.freeze({ id, label, command, arguments: Object.freeze([...args]), mode, approval, produces, completion });
|
|
54
|
+
}
|
|
55
|
+
function boundedSubject(value) {
|
|
56
|
+
if (typeof value !== 'string' || !value.trim() || value.length > 253 || /[\u0000-\u001f\u007f]/u.test(value)) {
|
|
57
|
+
throw new TypeError('Investigation-plan subject must be bounded text without control characters.');
|
|
58
|
+
}
|
|
59
|
+
return value.replace(/\s+/gu, ' ').trim();
|
|
60
|
+
}
|
|
61
|
+
function normalizedDomain(value) {
|
|
62
|
+
const candidate = domainToASCII(value.toLowerCase().replace(/\.$/u, ''));
|
|
63
|
+
if (!candidate || candidate.length > 253 || !candidate.includes('.'))
|
|
64
|
+
return null;
|
|
65
|
+
return candidate.split('.').every((label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/u.test(label))
|
|
66
|
+
? candidate
|
|
67
|
+
: null;
|
|
68
|
+
}
|
|
69
|
+
export function buildInvestigationPlan(recipeId, subjectValue, generatedAtValue = new Date().toISOString()) {
|
|
70
|
+
const recipe = RECIPES[recipeId];
|
|
71
|
+
if (!recipe)
|
|
72
|
+
throw new TypeError('Investigation-plan recipe is unsupported.');
|
|
73
|
+
const suppliedSubject = boundedSubject(subjectValue);
|
|
74
|
+
const subject = recipe.requiresDomain ? normalizedDomain(suppliedSubject) : suppliedSubject.toLowerCase();
|
|
75
|
+
if (!subject)
|
|
76
|
+
throw new TypeError('This investigation-plan recipe requires one valid domain.');
|
|
77
|
+
const parsedTime = Date.parse(generatedAtValue);
|
|
78
|
+
if (!Number.isFinite(parsedTime))
|
|
79
|
+
throw new TypeError('Investigation-plan generation time is invalid.');
|
|
80
|
+
const steps = recipe.steps(subject);
|
|
81
|
+
return Object.freeze({
|
|
82
|
+
schema: CLI_INVESTIGATION_PLAN_SCHEMA,
|
|
83
|
+
version: CLI_INVESTIGATION_PLAN_VERSION,
|
|
84
|
+
generatedAt: new Date(parsedTime).toISOString(),
|
|
85
|
+
recipe: Object.freeze({ id: recipeId, label: recipe.label, objective: recipe.objective }),
|
|
86
|
+
subject,
|
|
87
|
+
execution: 'plan_only',
|
|
88
|
+
steps,
|
|
89
|
+
limitations: Object.freeze([
|
|
90
|
+
'This document is a fixed plan. It does not execute commands, expand placeholders, make requests, read files, change cases, or submit reports.',
|
|
91
|
+
'Network steps require deliberate execution and disclose the selected target to the sources described by that command.',
|
|
92
|
+
'Analyst-selection steps require reviewed local artifacts; placeholders are never interpreted as file paths by this planner.',
|
|
93
|
+
]),
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
export function formatInvestigationPlan(plan) {
|
|
97
|
+
return [
|
|
98
|
+
`Investigation plan: ${plan.recipe.label}`,
|
|
99
|
+
`Subject ${plan.subject}`,
|
|
100
|
+
`Mode ${plan.execution}`,
|
|
101
|
+
'',
|
|
102
|
+
...plan.steps.flatMap((item, index) => [
|
|
103
|
+
`${index + 1}. ${item.label}`,
|
|
104
|
+
` ${item.command} ${item.arguments.join(' ')}`,
|
|
105
|
+
` ${item.mode}; approval: ${item.approval.replaceAll('_', ' ')}`,
|
|
106
|
+
` ${item.completion}`,
|
|
107
|
+
]),
|
|
108
|
+
'',
|
|
109
|
+
...plan.limitations.map((item) => `Limitation: ${item}`),
|
|
110
|
+
'',
|
|
111
|
+
].join('\n');
|
|
112
|
+
}
|
package/cli/mail-review.mjs
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
2
|
import { getDomain } from 'tldts';
|
|
3
|
+
import { analyzeTlsaEvidence } from "../lib/tlsa-evidence.mjs";
|
|
3
4
|
import { CliUsageError } from "./errors.mjs";
|
|
4
5
|
export const CLI_MAIL_REVIEW_SCHEMA = 'whoisleuth.cli.mail-review';
|
|
5
|
-
export const CLI_MAIL_REVIEW_VERSION =
|
|
6
|
+
export const CLI_MAIL_REVIEW_VERSION = 2;
|
|
6
7
|
export const MAX_MAIL_REVIEW_INPUT_BYTES = 16 * 1024 * 1024;
|
|
7
8
|
export const MAX_MAIL_REVIEW_ROWS = 500;
|
|
8
9
|
function record(value) {
|
|
@@ -53,6 +54,21 @@ function mailState(input) {
|
|
|
53
54
|
return 'no_explicit_mx';
|
|
54
55
|
return 'evidence_incomplete';
|
|
55
56
|
}
|
|
57
|
+
function alignMailDaneEvidence(report, hosts) {
|
|
58
|
+
const aligned = report.service?.transport === 'tcp'
|
|
59
|
+
&& report.service.port === 25
|
|
60
|
+
&& hosts.includes(report.service.hostname);
|
|
61
|
+
if (aligned || report.state === 'invalid')
|
|
62
|
+
return report;
|
|
63
|
+
return Object.freeze({
|
|
64
|
+
...report,
|
|
65
|
+
state: 'invalid',
|
|
66
|
+
limitations: Object.freeze([
|
|
67
|
+
...report.limitations,
|
|
68
|
+
'The TLSA service name did not identify port 25 on an MX hostname retained for this domain.',
|
|
69
|
+
]),
|
|
70
|
+
});
|
|
71
|
+
}
|
|
56
72
|
function normalizeMailRow(value) {
|
|
57
73
|
const item = record(value);
|
|
58
74
|
if (item.ok !== true)
|
|
@@ -70,6 +86,18 @@ function normalizeMailRow(value) {
|
|
|
70
86
|
const hasSpf = boolean(availability.hasSpf) ?? boolean(summary.hasSpf);
|
|
71
87
|
const hasDmarc = boolean(availability.hasDmarc) ?? boolean(summary.hasDmarc);
|
|
72
88
|
const state = mailState({ dnsStatus, hasMx, hasNullMx, hasSpf, hasDmarc });
|
|
89
|
+
const tlsaInput = record(item.tlsaEvidence);
|
|
90
|
+
const dane = Object.keys(tlsaInput).length
|
|
91
|
+
? alignMailDaneEvidence(analyzeTlsaEvidence({
|
|
92
|
+
serviceName: tlsaInput.serviceName,
|
|
93
|
+
dnssecState: tlsaInput.dnssecState,
|
|
94
|
+
pkixValidationState: tlsaInput.pkixValidationState,
|
|
95
|
+
records: tlsaInput.records,
|
|
96
|
+
certificateDerBase64: tlsaInput.certificateDerBase64,
|
|
97
|
+
spkiDerBase64: tlsaInput.spkiDerBase64,
|
|
98
|
+
authorityMaterials: tlsaInput.authorityMaterials,
|
|
99
|
+
}), hosts)
|
|
100
|
+
: null;
|
|
73
101
|
return {
|
|
74
102
|
domain,
|
|
75
103
|
state,
|
|
@@ -80,9 +108,11 @@ function normalizeMailRow(value) {
|
|
|
80
108
|
hasDmarc,
|
|
81
109
|
mxHosts: hosts,
|
|
82
110
|
providerDomains: providerDomains(hosts),
|
|
111
|
+
dane,
|
|
83
112
|
limitations: [
|
|
84
113
|
...(dnsStatus === 'partial' ? ['DNS evidence was partial.'] : []),
|
|
85
114
|
...(state === 'no_explicit_mx' ? ['No explicit MX is not equivalent to a null MX or proof that delivery is impossible.'] : []),
|
|
115
|
+
...(dane?.state === 'matched' ? [] : dane ? ['Supplied TLSA evidence did not establish one complete DNSSEC-validated match.'] : ['No TLSA evidence was supplied for offline DANE comparison.']),
|
|
86
116
|
'Mail configuration does not establish use, control, intent, safety, or maliciousness.',
|
|
87
117
|
],
|
|
88
118
|
};
|
|
@@ -142,6 +172,17 @@ function buildCliMailReview(textValue, generatedAt = new Date().toISOString()) {
|
|
|
142
172
|
};
|
|
143
173
|
for (const row of rows)
|
|
144
174
|
counts[row.state] += 1;
|
|
175
|
+
const daneCounts = {
|
|
176
|
+
matched: 0,
|
|
177
|
+
different: 0,
|
|
178
|
+
partial: 0,
|
|
179
|
+
untrusted: 0,
|
|
180
|
+
unavailable: 0,
|
|
181
|
+
invalid: 0,
|
|
182
|
+
not_supplied: 0,
|
|
183
|
+
};
|
|
184
|
+
for (const row of rows)
|
|
185
|
+
daneCounts[row.dane?.state ?? 'not_supplied'] += 1;
|
|
145
186
|
const providerMap = new Map();
|
|
146
187
|
for (const row of rows) {
|
|
147
188
|
for (const provider of row.providerDomains) {
|
|
@@ -165,11 +206,13 @@ function buildCliMailReview(textValue, generatedAt = new Date().toISOString()) {
|
|
|
165
206
|
version: CLI_MAIL_REVIEW_VERSION,
|
|
166
207
|
generatedAt,
|
|
167
208
|
counts,
|
|
209
|
+
daneCounts,
|
|
168
210
|
rows,
|
|
169
211
|
providerRelationships,
|
|
170
212
|
limitations: [
|
|
171
213
|
'This review is passive and uses DNS evidence already retained in a WHOISleuth Bulk result; it makes no network request.',
|
|
172
214
|
'Null MX, no explicit MX, receiving mail, authentication gaps, and incomplete evidence remain separate states.',
|
|
215
|
+
'Optional TLSA evidence is compared offline. A DANE match requires separately validated DNSSEC evidence from the same observation.',
|
|
173
216
|
'SMTP delivery, mailbox existence, catch-all behavior, banner collection, and message acceptance were not tested.',
|
|
174
217
|
],
|
|
175
218
|
};
|
|
@@ -183,12 +226,14 @@ function formatCliMailReview(document) {
|
|
|
183
226
|
`Null MX ${document.counts.null_mx}`,
|
|
184
227
|
`No explicit MX ${document.counts.no_explicit_mx}`,
|
|
185
228
|
`Incomplete ${document.counts.evidence_incomplete + document.counts.mail_auth_incomplete}`,
|
|
229
|
+
`DANE matched ${document.daneCounts.matched}`,
|
|
186
230
|
'',
|
|
187
231
|
];
|
|
188
232
|
for (const row of document.rows) {
|
|
189
233
|
lines.push(`${row.domain} ${row.state.replaceAll('_', ' ')}`);
|
|
190
234
|
lines.push(` MX providers ${row.providerDomains.join(', ') || 'None observed'}`);
|
|
191
235
|
lines.push(` SPF / DMARC ${row.hasSpf === null ? 'unknown' : row.hasSpf ? 'observed' : 'not observed'} / ${row.hasDmarc === null ? 'unknown' : row.hasDmarc ? 'observed' : 'not observed'}`);
|
|
236
|
+
lines.push(` DANE / TLSA ${row.dane?.state ?? 'not supplied'}`);
|
|
192
237
|
}
|
|
193
238
|
if (document.providerRelationships.length) {
|
|
194
239
|
lines.push('', 'Shared mail-provider relationships');
|
|
@@ -201,4 +246,4 @@ function formatCliMailReview(document) {
|
|
|
201
246
|
lines.push(` - ${limitation}`);
|
|
202
247
|
return `${lines.join('\n')}\n`;
|
|
203
248
|
}
|
|
204
|
-
export { buildCliMailReview, formatCliMailReview, mailState, normalizeMailRow };
|
|
249
|
+
export { alignMailDaneEvidence, buildCliMailReview, formatCliMailReview, mailState, normalizeMailRow };
|