@slicedearth/whoisleuth-cli 1.34.0 → 1.39.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/README.md +19 -2
  2. package/bin/whoisleuth.mjs +41 -24
  3. package/cli/arguments.mjs +326 -3
  4. package/cli/artifact-verify.mjs +60 -14
  5. package/cli/bulk.mjs +1 -2
  6. package/cli/compare.mjs +1 -1
  7. package/cli/completion.mjs +36 -2
  8. package/cli/doctor.mjs +11 -5
  9. package/cli/evidence-command-arguments.mjs +1 -1
  10. package/cli/evidence-command-runner.mjs +2 -2
  11. package/cli/evidence-signing.mjs +4 -4
  12. package/cli/export-evidence.mjs +9 -2
  13. package/cli/formatters/html.mjs +1 -1
  14. package/cli/formatters/json.mjs +13 -1
  15. package/cli/formatters/terminal.mjs +14 -4
  16. package/cli/investigation-plan.mjs +112 -0
  17. package/cli/local-mmdb-review.mjs +76 -0
  18. package/cli/lookalike-calibration.mjs +121 -0
  19. package/cli/lookup-command-runner.mjs +4 -1
  20. package/cli/lookup-plan.mjs +3 -3
  21. package/cli/lookup-reconcile.mjs +141 -0
  22. package/cli/lookup-timeline.mjs +1 -1
  23. package/cli/mail-review.mjs +48 -3
  24. package/cli/offline-evidence-review.mjs +159 -0
  25. package/cli/registry-doctor.mjs +125 -0
  26. package/cli/registry-support.mjs +1 -4
  27. package/cli/risk-calibration.mjs +117 -9
  28. package/cli/runner.mjs +342 -15
  29. package/cli/saved-lookup.mjs +2 -2
  30. package/cli/sharing-review.mjs +202 -0
  31. package/cli/source-reliability.mjs +1 -1
  32. package/docs/cli-reference.md +1110 -0
  33. package/docs/cli.md +224 -847
  34. package/frontend/src/lib/analysis/acquisition-decision-packet.js +1 -1
  35. package/frontend/src/lib/analysis/acquisition-due-diligence.js +6 -6
  36. package/frontend/src/lib/analysis/bulk-domain-comparison.js +12 -12
  37. package/frontend/src/lib/analysis/bulk-mail-exposure.js +1 -1
  38. package/frontend/src/lib/analysis/bulk-retry-plan.js +3 -10
  39. package/frontend/src/lib/analysis/bulk-session-model.js +7 -2
  40. package/frontend/src/lib/analysis/bulk-source-coverage.js +76 -0
  41. package/frontend/src/lib/analysis/case-evidence-model.js +26 -4
  42. package/frontend/src/lib/analysis/case-migration-model.js +6 -1
  43. package/frontend/src/lib/analysis/case-record-contracts.js +4 -2
  44. package/frontend/src/lib/analysis/case-record-core.js +5 -1
  45. package/frontend/src/lib/analysis/case-record-operations.js +6 -1
  46. package/frontend/src/lib/analysis/case-response-packet.js +7 -7
  47. package/frontend/src/lib/analysis/evidence-coverage-ledger.js +328 -0
  48. package/frontend/src/lib/analysis/http-summary.js +2 -1
  49. package/frontend/src/lib/analysis/investigation-capsule.js +105 -0
  50. package/frontend/src/lib/analysis/lookup-asset-graph.js +806 -0
  51. package/frontend/src/lib/analysis/lookup-decision-support.js +480 -0
  52. package/frontend/src/lib/analysis/lookup-investigation-brief.js +153 -0
  53. package/frontend/src/lib/analysis/lookup-presentation.js +74 -0
  54. package/frontend/src/lib/analysis/lookup-response.js +2 -0
  55. package/frontend/src/lib/analysis/lookup-source-refresh.js +304 -0
  56. package/frontend/src/lib/analysis/lookup-summary-model.js +398 -0
  57. package/frontend/src/lib/analysis/page-similarity.js +1 -1
  58. package/frontend/src/lib/analysis/relationship-evidence.js +3 -3
  59. package/frontend/src/lib/analysis/relationship-observation-model.js +4 -4
  60. package/frontend/src/lib/analysis/scoring.js +8 -88
  61. package/frontend/src/lib/analysis/shortlist-model.js +8 -4
  62. package/frontend/src/lib/analysis/utils.js +7 -40
  63. package/frontend/src/lib/analysis/visualization-bounds.js +26 -0
  64. package/frontend/src/lib/analysis/visualization-force-graph.js +352 -0
  65. package/frontend/src/lib/analysis/visualization-matrix.js +88 -0
  66. package/frontend/src/lib/analysis/visualization-models.js +370 -0
  67. package/frontend/src/lib/analysis/visualization-time-series.js +144 -0
  68. package/frontend/src/lib/analysis/website-snapshot-model.js +57 -3
  69. package/frontend/src/lib/analysis/workspace-archive-crypto.js +1 -1
  70. package/frontend/src/lib/analysis/workspace-archive.js +2 -2
  71. package/lib/analyst-taxonomy.mjs +40 -0
  72. package/lib/availability.mjs +30 -10
  73. package/lib/browser-library-profile.mjs +15 -2
  74. package/lib/bulk-limits.mjs +13 -0
  75. package/lib/certificate-extension-profile.mjs +71 -0
  76. package/lib/credential-surface-profile.mjs +1 -1
  77. package/lib/ct-response-bounds.mjs +9 -0
  78. package/lib/ct-search.mjs +66 -12
  79. package/lib/dns-delegation-health.mjs +56 -5
  80. package/lib/dnssec-evidence-validation.mjs +235 -0
  81. package/lib/domain-assurance.mjs +368 -0
  82. package/lib/domain-control-manifest.mjs +382 -0
  83. package/lib/domain-posture.mjs +11 -10
  84. package/lib/encrypted-dns-contract.mjs +81 -0
  85. package/lib/evidence-export.mjs +17 -3
  86. package/lib/evidence-report-markdown.mjs +1 -1
  87. package/lib/evidence-report.mjs +4 -1
  88. package/lib/favicon.mjs +3 -8
  89. package/lib/generated/cisa-kev-catalog.mjs +1668 -0
  90. package/lib/html-signals.mjs +4 -0
  91. package/lib/http-evidence-bounds.mjs +4 -0
  92. package/lib/http-intelligence.mjs +2 -1
  93. package/lib/ip-prefix.mjs +87 -0
  94. package/lib/local-geoip-evidence.mjs +89 -0
  95. package/lib/lookup-response-contract.mjs +346 -0
  96. package/lib/observed-network-context.mjs +1 -1
  97. package/lib/opportunity-scoring.mjs +107 -0
  98. package/lib/outbound-identity.mjs +9 -0
  99. package/lib/outbound-request-bounds.mjs +9 -0
  100. package/lib/page-fingerprints.mjs +1 -1
  101. package/lib/perceptual-hash-comparison.mjs +34 -0
  102. package/lib/perceptual-hash.mjs +3 -31
  103. package/lib/project-metadata.mjs +8 -0
  104. package/lib/provider-policy-admission.mjs +49 -0
  105. package/lib/rdap-capabilities.mjs +6 -6
  106. package/lib/rdap-search-workbench.mjs +247 -0
  107. package/lib/rdap.mjs +7 -2
  108. package/lib/registry-access.mjs +1 -1
  109. package/lib/registry-capabilities.mjs +31 -1
  110. package/lib/registry-capability-catalogue.mjs +15 -14
  111. package/lib/registry-insights.mjs +6 -6
  112. package/lib/response-policy.mjs +81 -6
  113. package/lib/risk-scoring.mjs +206 -83
  114. package/lib/rpki-evidence.mjs +105 -0
  115. package/lib/safe-fetch.mjs +2 -1
  116. package/lib/scoring-evidence-quality.mjs +107 -0
  117. package/lib/security-txt.mjs +3 -3
  118. package/lib/semantic-version.mjs +54 -0
  119. package/lib/static-html-analysis.mjs +27 -0
  120. package/lib/threat-intelligence-manifests.mjs +1 -1
  121. package/lib/threatfox-intelligence.mjs +11 -6
  122. package/lib/tls-intelligence.mjs +13 -3
  123. package/lib/tlsa-evidence.mjs +194 -0
  124. package/lib/urlhaus-intelligence.mjs +11 -6
  125. package/lib/urlscan-intelligence.mjs +10 -5
  126. package/lib/website-security-posture.mjs +16 -9
  127. package/lib/website-technology.mjs +1 -1
  128. package/lib/whois-chain.mjs +14 -152
  129. package/lib/whois.mjs +2 -2
  130. package/package.json +6 -2
  131. package/third-party-notices.txt +911 -19
@@ -0,0 +1,76 @@
1
+ import { stat } from 'node:fs/promises';
2
+ import { isIP } from 'node:net';
3
+ import { open } from 'maxmind';
4
+ import { formatIpPrefix, parseIpPrefix } from "../lib/ip-prefix.mjs";
5
+ import { CliUsageError } from "./errors.mjs";
6
+ export const LOCAL_MMDB_QUERY_SCHEMA = 'whoisleuth.local-mmdb-query';
7
+ export const MAX_LOCAL_MMDB_BYTES = 512 * 1024 * 1024;
8
+ function record(value) {
9
+ return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
10
+ }
11
+ function boundedText(value, maximum) {
12
+ return typeof value === 'string' && value.length <= maximum && !/[\u0000-\u001f\u007f]/u.test(value)
13
+ ? value.replace(/\s+/gu, ' ').trim() || null
14
+ : null;
15
+ }
16
+ function names(value) {
17
+ const source = record(value);
18
+ return boundedText(source.en, 120);
19
+ }
20
+ export async function reviewLocalMmdb(input, databasePath) {
21
+ const address = boundedText(input.address, 64)?.toLowerCase() ?? '';
22
+ const sourceLabel = boundedText(input.sourceLabel, 120);
23
+ const databaseVersion = boundedText(input.databaseVersion, 80);
24
+ const license = boundedText(input.license, 240);
25
+ if (!isIP(address))
26
+ throw new CliUsageError('Local MMDB review requires one valid IP address.');
27
+ if (!sourceLabel || !databaseVersion || !license) {
28
+ throw new CliUsageError('Local MMDB review requires bounded sourceLabel, databaseVersion, and licence metadata.');
29
+ }
30
+ let info;
31
+ try {
32
+ info = await stat(databasePath);
33
+ }
34
+ catch {
35
+ throw new CliUsageError('The supplied MMDB file could not be read.');
36
+ }
37
+ if (!info.isFile() || info.size <= 0 || info.size > MAX_LOCAL_MMDB_BYTES) {
38
+ throw new CliUsageError(`The supplied MMDB must be a file no larger than ${MAX_LOCAL_MMDB_BYTES} bytes.`);
39
+ }
40
+ let reader;
41
+ try {
42
+ reader = await open(databasePath, { cache: { max: 1_000 } });
43
+ }
44
+ catch {
45
+ throw new CliUsageError('The supplied file is not a readable MaxMind DB database.');
46
+ }
47
+ const [rawMatch, prefixLength] = reader.getWithPrefixLength(address);
48
+ const match = record(rawMatch);
49
+ const traits = record(match.traits);
50
+ const country = record(match.country);
51
+ const registeredCountry = record(match.registered_country);
52
+ const subdivisions = Array.isArray(match.subdivisions) ? match.subdivisions : [];
53
+ const subdivision = record(subdivisions[0]);
54
+ const city = record(match.city);
55
+ const asnValue = match.autonomous_system_number ?? traits.autonomous_system_number;
56
+ const asn = typeof asnValue === 'number' && Number.isSafeInteger(asnValue) && asnValue >= 0 ? asnValue : null;
57
+ const prefix = parseIpPrefix(`${address}/${prefixLength}`);
58
+ return Object.freeze({
59
+ state: rawMatch ? 'matched' : 'not_found',
60
+ address,
61
+ match: rawMatch ? Object.freeze({
62
+ network: prefix ? formatIpPrefix(prefix) : null,
63
+ countryCode: boundedText(country.iso_code ?? registeredCountry.iso_code, 2)?.toUpperCase() ?? null,
64
+ region: names(subdivision.names) ?? boundedText(subdivision.iso_code, 120),
65
+ city: names(city.names),
66
+ asn,
67
+ asName: boundedText(match.autonomous_system_organization ?? traits.autonomous_system_organization, 240),
68
+ }) : null,
69
+ source: Object.freeze({ label: sourceLabel, version: databaseVersion, license }),
70
+ limitations: Object.freeze([
71
+ 'This CLI-only result comes from an analyst-supplied local MMDB file. WHOISleuth does not bundle, download, update, license, or transmit the database.',
72
+ 'Geolocation and network attribution are provider estimates and may be stale, coarse, shared, or incorrect.',
73
+ 'A location or ASN association does not establish the operator, owner, user, intent, safety, or maliciousness of an address.',
74
+ ]),
75
+ });
76
+ }
@@ -0,0 +1,121 @@
1
+ import { Buffer } from 'node:buffer';
2
+ const LOOKALIKE_CALIBRATION_INPUT_SCHEMA = 'whoisleuth.lookalike-calibration-input';
3
+ const LOOKALIKE_CALIBRATION_SCHEMA = 'whoisleuth.lookalike-calibration';
4
+ const LOOKALIKE_CALIBRATION_VERSION = 1;
5
+ const MAX_LOOKALIKE_CALIBRATION_BYTES = 2 * 1024 * 1024;
6
+ const MAX_LOOKALIKE_CALIBRATION_RECORDS = 5_000;
7
+ const MIN_LOOKALIKE_CALIBRATION_SAMPLE = 20;
8
+ const DISPOSITIONS = ['suspicious', 'confirmed_abuse', 'false_positive', 'expected', 'closed_no_action'];
9
+ function record(value) {
10
+ return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
11
+ }
12
+ function token(value, field) {
13
+ if (typeof value !== 'string')
14
+ throw new TypeError(`${field} must be text.`);
15
+ const normalized = value.trim().toLowerCase();
16
+ if (!/^[a-z0-9][a-z0-9_-]{0,79}$/u.test(normalized))
17
+ throw new TypeError(`${field} must be a bounded identifier.`);
18
+ return normalized;
19
+ }
20
+ function ratio(value, total) {
21
+ return total ? Number((value / total).toFixed(4)) : 0;
22
+ }
23
+ function emptyCounts() {
24
+ return { suspicious: 0, confirmed_abuse: 0, false_positive: 0, expected: 0, closed_no_action: 0 };
25
+ }
26
+ function buildLookalikeCalibration(raw, generatedAt = new Date().toISOString()) {
27
+ if (Buffer.byteLength(raw, 'utf8') > MAX_LOOKALIKE_CALIBRATION_BYTES) {
28
+ throw new TypeError(`Lookalike calibration input is limited to ${MAX_LOOKALIKE_CALIBRATION_BYTES} bytes.`);
29
+ }
30
+ let parsed;
31
+ try {
32
+ parsed = JSON.parse(raw);
33
+ }
34
+ catch {
35
+ throw new TypeError('Lookalike calibration input must be valid JSON.');
36
+ }
37
+ const input = record(parsed);
38
+ if (input.schema !== LOOKALIKE_CALIBRATION_INPUT_SCHEMA || input.version !== LOOKALIKE_CALIBRATION_VERSION) {
39
+ throw new TypeError(`Lookalike calibration input must use ${LOOKALIKE_CALIBRATION_INPUT_SCHEMA} version ${LOOKALIKE_CALIBRATION_VERSION}.`);
40
+ }
41
+ if (!Array.isArray(input.records) || input.records.length < 1 || input.records.length > MAX_LOOKALIKE_CALIBRATION_RECORDS) {
42
+ throw new TypeError(`Lookalike calibration input must contain from 1 to ${MAX_LOOKALIKE_CALIBRATION_RECORDS} reviewed records.`);
43
+ }
44
+ const ids = new Set();
45
+ const familyCounts = new Map();
46
+ const totalCounts = emptyCounts();
47
+ for (let index = 0; index < input.records.length; index++) {
48
+ const item = record(input.records[index]);
49
+ const id = token(item.id, `records[${index}].id`);
50
+ if (ids.has(id))
51
+ throw new TypeError(`Lookalike calibration record id "${id}" is duplicated.`);
52
+ ids.add(id);
53
+ if (typeof item.disposition !== 'string' || !DISPOSITIONS.includes(item.disposition)) {
54
+ throw new TypeError(`records[${index}].disposition is not a supported reviewed disposition.`);
55
+ }
56
+ const disposition = item.disposition;
57
+ if (!Array.isArray(item.mutationFamilies) || item.mutationFamilies.length < 1 || item.mutationFamilies.length > 12) {
58
+ throw new TypeError(`records[${index}].mutationFamilies must contain from 1 to 12 identifiers.`);
59
+ }
60
+ const families = [...new Set(item.mutationFamilies.map((value) => token(value, `records[${index}].mutationFamilies`)))];
61
+ totalCounts[disposition] += 1;
62
+ for (const family of families) {
63
+ const counts = familyCounts.get(family) || emptyCounts();
64
+ counts[disposition] += 1;
65
+ familyCounts.set(family, counts);
66
+ }
67
+ }
68
+ const families = [...familyCounts.entries()].map(([id, dispositions]) => {
69
+ const reviewedCandidates = Object.values(dispositions).reduce((sum, count) => sum + count, 0);
70
+ return {
71
+ id,
72
+ reviewedCandidates,
73
+ sampleState: reviewedCandidates >= MIN_LOOKALIKE_CALIBRATION_SAMPLE ? 'reviewed' : 'insufficient',
74
+ dispositions,
75
+ reviewLeadRate: ratio(dispositions.suspicious + dispositions.confirmed_abuse, reviewedCandidates),
76
+ falsePositiveRate: ratio(dispositions.false_positive, reviewedCandidates),
77
+ };
78
+ }).sort((left, right) => right.reviewedCandidates - left.reviewedCandidates || left.id.localeCompare(right.id));
79
+ const reviewedCandidates = input.records.length;
80
+ return {
81
+ schema: LOOKALIKE_CALIBRATION_SCHEMA,
82
+ version: LOOKALIKE_CALIBRATION_VERSION,
83
+ generatedAt,
84
+ summary: {
85
+ reviewedCandidates,
86
+ mutationFamilies: families.length,
87
+ sampleState: reviewedCandidates >= MIN_LOOKALIKE_CALIBRATION_SAMPLE ? 'reviewed' : 'insufficient',
88
+ reviewLeadRate: ratio(totalCounts.suspicious + totalCounts.confirmed_abuse, reviewedCandidates),
89
+ falsePositiveRate: ratio(totalCounts.false_positive, reviewedCandidates),
90
+ expectedRate: ratio(totalCounts.expected, reviewedCandidates),
91
+ },
92
+ families,
93
+ privacy: { candidateIdsRetained: 0, domainsRetained: 0, notesRetained: 0 },
94
+ limitations: [
95
+ 'This offline report summarises analyst dispositions for the supplied reviewed sample and never changes mutation generation, ordering, Risk, or filtering automatically.',
96
+ 'Suspicious and confirmed-abuse dispositions are counted as review leads, not proof of maliciousness; expected and false-positive dispositions are not proof of safety.',
97
+ `A family remains an insufficient sample below ${MIN_LOOKALIKE_CALIBRATION_SAMPLE} reviewed candidates, and even larger local samples may not generalize across brands, languages, registries, or time.`,
98
+ 'Candidate identifiers, domains, notes, and evidence are intentionally omitted from output. Retain the reviewed source dataset under an appropriate local privacy policy.',
99
+ ],
100
+ };
101
+ }
102
+ function formatLookalikeCalibration(report) {
103
+ const output = [
104
+ 'Lookalike review-yield calibration',
105
+ `Reviewed candidates ${report.summary.reviewedCandidates}`,
106
+ `Mutation families ${report.summary.mutationFamilies}`,
107
+ `Sample state ${report.summary.sampleState}`,
108
+ `Review-lead rate ${(report.summary.reviewLeadRate * 100).toFixed(1)}%`,
109
+ `False-positive rate ${(report.summary.falsePositiveRate * 100).toFixed(1)}%`,
110
+ '',
111
+ ];
112
+ for (const family of report.families) {
113
+ output.push(`${family.id} [${family.sampleState}]`);
114
+ output.push(` ${family.reviewedCandidates} reviewed · ${(family.reviewLeadRate * 100).toFixed(1)}% review leads · ${(family.falsePositiveRate * 100).toFixed(1)}% false positives`);
115
+ }
116
+ output.push('', 'Limitations:');
117
+ for (const limitation of report.limitations)
118
+ output.push(` - ${limitation}`);
119
+ return `${output.join('\n')}\n`;
120
+ }
121
+ export { LOOKALIKE_CALIBRATION_INPUT_SCHEMA, LOOKALIKE_CALIBRATION_SCHEMA, LOOKALIKE_CALIBRATION_VERSION, MAX_LOOKALIKE_CALIBRATION_BYTES, MAX_LOOKALIKE_CALIBRATION_RECORDS, MIN_LOOKALIKE_CALIBRATION_SAMPLE, buildLookalikeCalibration, formatLookalikeCalibration, };
@@ -68,7 +68,10 @@ async function runLookupCommand(args, dependencies, context) {
68
68
  context.endProgress();
69
69
  }
70
70
  const now = context.now();
71
- const document = buildCliLookupDocument(query, classified, result, now, args.deep ? 'deep' : 'fast');
71
+ const document = buildCliLookupDocument(query, classified, result, now, args.deep ? 'deep' : 'fast', {
72
+ ...(args.observerLabel ? { observerLabel: args.observerLabel } : {}),
73
+ ...(args.vantageLabel ? { vantageLabel: args.vantageLabel } : {}),
74
+ });
72
75
  if (!args.quiet) {
73
76
  if (args.output === 'json')
74
77
  context.writeStdout(formatJsonDocument(document));
@@ -3,13 +3,13 @@ export const CLI_LOOKUP_PLAN_VERSION = 1;
3
3
  const RDAP = Object.freeze({
4
4
  source: 'rdap',
5
5
  purpose: 'Collect authoritative registration or allocation evidence where supported.',
6
- disclosure: 'The normalized target is sent to the applicable RDAP bootstrap and service endpoints.',
6
+ disclosure: 'The normalised target is sent to the applicable RDAP bootstrap and service endpoints.',
7
7
  conditional: false,
8
8
  });
9
9
  const WHOIS = Object.freeze({
10
10
  source: 'whois',
11
11
  purpose: 'Collect separately attributed registry and referral publications.',
12
- disclosure: 'The normalized target is sent over bounded TCP connections to applicable WHOIS services.',
12
+ disclosure: 'The normalised target is sent over bounded TCP connections to applicable WHOIS services.',
13
13
  conditional: false,
14
14
  });
15
15
  const FAST_DOMAIN_EVIDENCE = Object.freeze({
@@ -39,7 +39,7 @@ const NETWORK_CONTEXT = Object.freeze({
39
39
  const REVERSE_DNS = Object.freeze({
40
40
  source: 'reverse_dns',
41
41
  purpose: 'Collect operator-published reverse-DNS context for a public IP address.',
42
- disclosure: 'A DNS resolver receives the reverse lookup for the normalized IP address.',
42
+ disclosure: 'A DNS resolver receives the reverse lookup for the normalised IP address.',
43
43
  conditional: false,
44
44
  });
45
45
  function plannedSources(classified, deep) {
@@ -0,0 +1,141 @@
1
+ import { CliUsageError } from "./errors.mjs";
2
+ import { buildCliLookupDiff } from "./lookup-diff.mjs";
3
+ import { parseSavedLookupDocument } from "./saved-lookup.mjs";
4
+ const CLI_LOOKUP_RECONCILIATION_SCHEMA = 'whoisleuth.cli.lookup-reconciliation';
5
+ const CLI_LOOKUP_RECONCILIATION_VERSION = 1;
6
+ const MAX_LOOKUP_RECONCILIATION_INPUT_BYTES = 32 * 1024 * 1024;
7
+ function object(value) {
8
+ return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
9
+ }
10
+ function label(value) {
11
+ if (typeof value !== 'string')
12
+ return null;
13
+ const normalized = value.replace(/[\u0000-\u001f\u007f]+/gu, ' ').replace(/\s+/gu, ' ').trim().slice(0, 80);
14
+ return normalized || null;
15
+ }
16
+ function observationContext(document) {
17
+ const context = object(document.collectionContext);
18
+ return {
19
+ generatedAt: document.generatedAt,
20
+ mode: document.mode,
21
+ observerLabel: label(context.observerLabel),
22
+ vantageLabel: label(context.vantageLabel),
23
+ };
24
+ }
25
+ function comparable(value, sourceState) {
26
+ if (['error', 'not_recorded', 'partial', 'skipped', 'unavailable', 'unsupported'].includes(sourceState))
27
+ return false;
28
+ return value !== 'Not observed' && value !== 'Not recorded in compact Bulk evidence';
29
+ }
30
+ function normalizedValue(value) {
31
+ return value.replace(/\s+/gu, ' ').trim().toLocaleLowerCase('en-US');
32
+ }
33
+ function buildCliLookupReconciliation(inputs, generatedAt = new Date().toISOString()) {
34
+ if (inputs.length < 2 || inputs.length > 5)
35
+ throw new CliUsageError('Lookup reconciliation requires from 2 to 5 observations.');
36
+ const documents = inputs.map((input, index) => parseSavedLookupDocument(input, {
37
+ label: `Lookup reconciliation observation ${index + 1}`,
38
+ }));
39
+ const domain = documents[0]?.registrableDomain;
40
+ if (!domain || documents.some((document) => document.registrableDomain !== domain)) {
41
+ throw new CliUsageError('Lookup reconciliation requires observations for one domain.');
42
+ }
43
+ const comparisons = documents.slice(1).map((document, index) => buildCliLookupDiff(inputs[0], inputs[index + 1], generatedAt, { domainMode: 'same' }));
44
+ const firstRows = comparisons[0]?.comparison.rows || [];
45
+ const fields = firstRows.map((firstRow) => {
46
+ const values = [{
47
+ observation: 0,
48
+ value: firstRow.left,
49
+ sourceState: firstRow.leftSourceState,
50
+ comparable: comparable(firstRow.left, firstRow.leftSourceState),
51
+ }];
52
+ for (let index = 0; index < comparisons.length; index++) {
53
+ const row = comparisons[index].comparison.rows.find((candidate) => candidate.id === firstRow.id);
54
+ if (!row)
55
+ continue;
56
+ values.push({
57
+ observation: index + 1,
58
+ value: row.right,
59
+ sourceState: row.rightSourceState,
60
+ comparable: comparable(row.right, row.rightSourceState),
61
+ });
62
+ }
63
+ const comparableValues = values.filter((value) => value.comparable);
64
+ const distinctValues = new Set(comparableValues.map((value) => normalizedValue(value.value)));
65
+ const state = comparableValues.length < 2
66
+ ? 'non_comparable'
67
+ : distinctValues.size === 1
68
+ ? 'agreement'
69
+ : 'disagreement';
70
+ return {
71
+ id: firstRow.id,
72
+ category: firstRow.category,
73
+ label: firstRow.label,
74
+ source: firstRow.source,
75
+ state,
76
+ method: firstRow.method,
77
+ values,
78
+ limitations: firstRow.limitations,
79
+ };
80
+ });
81
+ const observations = documents.map(observationContext);
82
+ const identities = observations.map((observation) => (observation.observerLabel && observation.vantageLabel
83
+ ? `${observation.observerLabel}\u0000${observation.vantageLabel}`
84
+ : null));
85
+ const labelsComplete = identities.every((identity) => identity !== null);
86
+ const labelsDistinct = labelsComplete && new Set(identities).size === identities.length;
87
+ const independence = labelsDistinct
88
+ ? {
89
+ state: 'verified_distinct_labels',
90
+ reason: 'Every observation carries a different analyst-supplied observer and vantage label.',
91
+ }
92
+ : {
93
+ state: 'unverified',
94
+ reason: 'Observer and vantage labels are missing or repeated; independent collection cannot be inferred.',
95
+ };
96
+ return {
97
+ schema: CLI_LOOKUP_RECONCILIATION_SCHEMA,
98
+ version: CLI_LOOKUP_RECONCILIATION_VERSION,
99
+ generatedAt,
100
+ domain,
101
+ independence,
102
+ observations,
103
+ fields,
104
+ summary: {
105
+ observationCount: observations.length,
106
+ agreement: fields.filter((field) => field.state === 'agreement').length,
107
+ disagreement: fields.filter((field) => field.state === 'disagreement').length,
108
+ nonComparable: fields.filter((field) => field.state === 'non_comparable').length,
109
+ },
110
+ privacy: { filenamesRetained: 0, rawRegistryPayloadsCopied: 0 },
111
+ limitations: [
112
+ 'This report reconciles only bounded normalised values already present in saved Lookup documents and makes no network request.',
113
+ 'Observer and vantage labels are analyst-supplied metadata. Distinct labels do not prove independent resolvers, networks, providers, caches, or collection paths.',
114
+ 'Agreement can reflect shared upstream data or collection conditions. Disagreement can reflect collection timing, propagation, source health, or a real domain change.',
115
+ 'Non-comparable evidence remains separate from agreement and is never interpreted as absence, safety, or a negative finding.',
116
+ ],
117
+ };
118
+ }
119
+ function formatCliLookupReconciliation(document) {
120
+ const output = [
121
+ 'Lookup observation reconciliation',
122
+ `Domain ${document.domain}`,
123
+ `Observations ${document.summary.observationCount}`,
124
+ `Independence ${document.independence.state.replaceAll('_', ' ')}`,
125
+ `Agreement ${document.summary.agreement}`,
126
+ `Disagreement ${document.summary.disagreement}`,
127
+ `Non-comparable ${document.summary.nonComparable}`,
128
+ '',
129
+ ];
130
+ for (const field of document.fields.filter((candidate) => candidate.state !== 'agreement')) {
131
+ output.push(`${field.label} [${field.state.replaceAll('_', ' ')}]`);
132
+ for (const value of field.values) {
133
+ output.push(` Observation ${value.observation + 1}: ${value.value} (${value.sourceState.replaceAll('_', ' ')})`);
134
+ }
135
+ }
136
+ output.push('', 'Limitations:');
137
+ for (const limitation of document.limitations)
138
+ output.push(` - ${limitation}`);
139
+ return `${output.join('\n')}\n`;
140
+ }
141
+ export { CLI_LOOKUP_RECONCILIATION_SCHEMA, CLI_LOOKUP_RECONCILIATION_VERSION, MAX_LOOKUP_RECONCILIATION_INPUT_BYTES, buildCliLookupReconciliation, formatCliLookupReconciliation, };
@@ -85,7 +85,7 @@ function buildCliLookupTimeline(inputs, generatedAt = new Date().toISOString())
85
85
  rawRegistryPayloadsCopied: 0,
86
86
  },
87
87
  limitations: [
88
- 'The timeline compares only bounded normalized fields already present in the selected saved Lookup documents and makes no network request.',
88
+ 'The timeline compares only bounded normalised fields already present in the selected saved Lookup documents and makes no network request.',
89
89
  'Input filenames and raw registry payloads are not copied into the timeline output.',
90
90
  'Missing, unavailable, and not-recorded evidence remain separate from an observed difference.',
91
91
  'A difference can reflect a domain change or changed collection conditions and does not by itself establish current state, ownership, intent, safety, or maliciousness.',
@@ -1,8 +1,9 @@
1
1
  import { Buffer } from 'node:buffer';
2
2
  import { getDomain } from 'tldts';
3
+ import { analyzeTlsaEvidence } from "../lib/tlsa-evidence.mjs";
3
4
  import { CliUsageError } from "./errors.mjs";
4
5
  export const CLI_MAIL_REVIEW_SCHEMA = 'whoisleuth.cli.mail-review';
5
- export const CLI_MAIL_REVIEW_VERSION = 1;
6
+ export const CLI_MAIL_REVIEW_VERSION = 2;
6
7
  export const MAX_MAIL_REVIEW_INPUT_BYTES = 16 * 1024 * 1024;
7
8
  export const MAX_MAIL_REVIEW_ROWS = 500;
8
9
  function record(value) {
@@ -53,6 +54,21 @@ function mailState(input) {
53
54
  return 'no_explicit_mx';
54
55
  return 'evidence_incomplete';
55
56
  }
57
+ function alignMailDaneEvidence(report, hosts) {
58
+ const aligned = report.service?.transport === 'tcp'
59
+ && report.service.port === 25
60
+ && hosts.includes(report.service.hostname);
61
+ if (aligned || report.state === 'invalid')
62
+ return report;
63
+ return Object.freeze({
64
+ ...report,
65
+ state: 'invalid',
66
+ limitations: Object.freeze([
67
+ ...report.limitations,
68
+ 'The TLSA service name did not identify port 25 on an MX hostname retained for this domain.',
69
+ ]),
70
+ });
71
+ }
56
72
  function normalizeMailRow(value) {
57
73
  const item = record(value);
58
74
  if (item.ok !== true)
@@ -70,6 +86,18 @@ function normalizeMailRow(value) {
70
86
  const hasSpf = boolean(availability.hasSpf) ?? boolean(summary.hasSpf);
71
87
  const hasDmarc = boolean(availability.hasDmarc) ?? boolean(summary.hasDmarc);
72
88
  const state = mailState({ dnsStatus, hasMx, hasNullMx, hasSpf, hasDmarc });
89
+ const tlsaInput = record(item.tlsaEvidence);
90
+ const dane = Object.keys(tlsaInput).length
91
+ ? alignMailDaneEvidence(analyzeTlsaEvidence({
92
+ serviceName: tlsaInput.serviceName,
93
+ dnssecState: tlsaInput.dnssecState,
94
+ pkixValidationState: tlsaInput.pkixValidationState,
95
+ records: tlsaInput.records,
96
+ certificateDerBase64: tlsaInput.certificateDerBase64,
97
+ spkiDerBase64: tlsaInput.spkiDerBase64,
98
+ authorityMaterials: tlsaInput.authorityMaterials,
99
+ }), hosts)
100
+ : null;
73
101
  return {
74
102
  domain,
75
103
  state,
@@ -80,9 +108,11 @@ function normalizeMailRow(value) {
80
108
  hasDmarc,
81
109
  mxHosts: hosts,
82
110
  providerDomains: providerDomains(hosts),
111
+ dane,
83
112
  limitations: [
84
113
  ...(dnsStatus === 'partial' ? ['DNS evidence was partial.'] : []),
85
114
  ...(state === 'no_explicit_mx' ? ['No explicit MX is not equivalent to a null MX or proof that delivery is impossible.'] : []),
115
+ ...(dane?.state === 'matched' ? [] : dane ? ['Supplied TLSA evidence did not establish one complete DNSSEC-validated match.'] : ['No TLSA evidence was supplied for offline DANE comparison.']),
86
116
  'Mail configuration does not establish use, control, intent, safety, or maliciousness.',
87
117
  ],
88
118
  };
@@ -142,6 +172,17 @@ function buildCliMailReview(textValue, generatedAt = new Date().toISOString()) {
142
172
  };
143
173
  for (const row of rows)
144
174
  counts[row.state] += 1;
175
+ const daneCounts = {
176
+ matched: 0,
177
+ different: 0,
178
+ partial: 0,
179
+ untrusted: 0,
180
+ unavailable: 0,
181
+ invalid: 0,
182
+ not_supplied: 0,
183
+ };
184
+ for (const row of rows)
185
+ daneCounts[row.dane?.state ?? 'not_supplied'] += 1;
145
186
  const providerMap = new Map();
146
187
  for (const row of rows) {
147
188
  for (const provider of row.providerDomains) {
@@ -165,12 +206,14 @@ function buildCliMailReview(textValue, generatedAt = new Date().toISOString()) {
165
206
  version: CLI_MAIL_REVIEW_VERSION,
166
207
  generatedAt,
167
208
  counts,
209
+ daneCounts,
168
210
  rows,
169
211
  providerRelationships,
170
212
  limitations: [
171
213
  'This review is passive and uses DNS evidence already retained in a WHOISleuth Bulk result; it makes no network request.',
172
214
  'Null MX, no explicit MX, receiving mail, authentication gaps, and incomplete evidence remain separate states.',
173
- 'SMTP delivery, mailbox existence, catch-all behavior, banner collection, and message acceptance were not tested.',
215
+ 'Optional TLSA evidence is compared offline. A DANE match requires separately validated DNSSEC evidence from the same observation.',
216
+ 'SMTP delivery, mailbox existence, catch-all behaviour, banner collection, and message acceptance were not tested.',
174
217
  ],
175
218
  };
176
219
  }
@@ -183,12 +226,14 @@ function formatCliMailReview(document) {
183
226
  `Null MX ${document.counts.null_mx}`,
184
227
  `No explicit MX ${document.counts.no_explicit_mx}`,
185
228
  `Incomplete ${document.counts.evidence_incomplete + document.counts.mail_auth_incomplete}`,
229
+ `DANE matched ${document.daneCounts.matched}`,
186
230
  '',
187
231
  ];
188
232
  for (const row of document.rows) {
189
233
  lines.push(`${row.domain} ${row.state.replaceAll('_', ' ')}`);
190
234
  lines.push(` MX providers ${row.providerDomains.join(', ') || 'None observed'}`);
191
235
  lines.push(` SPF / DMARC ${row.hasSpf === null ? 'unknown' : row.hasSpf ? 'observed' : 'not observed'} / ${row.hasDmarc === null ? 'unknown' : row.hasDmarc ? 'observed' : 'not observed'}`);
236
+ lines.push(` DANE / TLSA ${row.dane?.state ?? 'not supplied'}`);
192
237
  }
193
238
  if (document.providerRelationships.length) {
194
239
  lines.push('', 'Shared mail-provider relationships');
@@ -201,4 +246,4 @@ function formatCliMailReview(document) {
201
246
  lines.push(` - ${limitation}`);
202
247
  return `${lines.join('\n')}\n`;
203
248
  }
204
- export { buildCliMailReview, formatCliMailReview, mailState, normalizeMailRow };
249
+ export { alignMailDaneEvidence, buildCliMailReview, formatCliMailReview, mailState, normalizeMailRow };
@@ -0,0 +1,159 @@
1
+ import { Buffer } from 'node:buffer';
2
+ import { normalizeEncryptedDnsAdapter, planEncryptedDnsQuery, } from "../lib/encrypted-dns-contract.mjs";
3
+ import { validateDnssecEvidence } from "../lib/dnssec-evidence-validation.mjs";
4
+ import { buildLocalGeoIpDatabase, lookupLocalGeoIp, } from "../lib/local-geoip-evidence.mjs";
5
+ import { normalizeRdapSearchHelp, planRdapReverseSearch, } from "../lib/rdap-search-workbench.mjs";
6
+ import { reviewRpkiRoute } from "../lib/rpki-evidence.mjs";
7
+ import { analyzeTlsaEvidence } from "../lib/tlsa-evidence.mjs";
8
+ import { CliUsageError } from "./errors.mjs";
9
+ import { LOCAL_MMDB_QUERY_SCHEMA, reviewLocalMmdb } from "./local-mmdb-review.mjs";
10
+ const OFFLINE_EVIDENCE_REVIEW_SCHEMA = 'whoisleuth.cli.offline-evidence-review';
11
+ const OFFLINE_EVIDENCE_REVIEW_VERSION = 1;
12
+ const MAX_OFFLINE_EVIDENCE_INPUT_BYTES = 16 * 1024 * 1024;
13
+ function record(value) {
14
+ return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
15
+ }
16
+ function parseInput(value) {
17
+ if (typeof value !== 'string' || Buffer.byteLength(value, 'utf8') > MAX_OFFLINE_EVIDENCE_INPUT_BYTES) {
18
+ throw new CliUsageError(`Offline evidence input is limited to ${MAX_OFFLINE_EVIDENCE_INPUT_BYTES} bytes.`);
19
+ }
20
+ let parsed;
21
+ try {
22
+ parsed = JSON.parse(value.replace(/^\uFEFF/u, ''));
23
+ }
24
+ catch {
25
+ throw new CliUsageError('Offline evidence review requires one valid JSON document.');
26
+ }
27
+ const document = record(parsed);
28
+ if (document.version !== 1 || typeof document.schema !== 'string') {
29
+ throw new CliUsageError('Offline evidence review requires a supported versioned input schema.');
30
+ }
31
+ return document;
32
+ }
33
+ function buildOfflineEvidenceReview(value, generatedAt = new Date().toISOString()) {
34
+ const input = parseInput(value);
35
+ let kind;
36
+ let result;
37
+ if (input.schema === 'whoisleuth.rdap-search-input') {
38
+ kind = 'rdap_search';
39
+ const request = record(input.request);
40
+ const help = normalizeRdapSearchHelp(input.help);
41
+ result = Object.freeze({
42
+ help,
43
+ plan: planRdapReverseSearch(help, {
44
+ searchableResourceType: String(request.searchableResourceType ?? ''),
45
+ relatedResourceType: String(request.relatedResourceType ?? ''),
46
+ property: String(request.property ?? ''),
47
+ value: request.value,
48
+ }),
49
+ });
50
+ }
51
+ else if (input.schema === 'whoisleuth.dnssec-evidence-input') {
52
+ kind = 'dnssec';
53
+ result = validateDnssecEvidence({
54
+ ownerName: input.ownerName,
55
+ delegationSigned: input.delegationSigned,
56
+ dsRecords: input.dsRecords,
57
+ dnskeyRecords: input.dnskeyRecords,
58
+ rrSigRecords: input.rrSigRecords,
59
+ observedAt: input.observedAt,
60
+ });
61
+ }
62
+ else if (input.schema === 'whoisleuth.tlsa-evidence-input') {
63
+ kind = 'tlsa';
64
+ result = analyzeTlsaEvidence({
65
+ serviceName: input.serviceName,
66
+ dnssecState: input.dnssecState,
67
+ pkixValidationState: input.pkixValidationState,
68
+ records: input.records,
69
+ certificateDerBase64: input.certificateDerBase64,
70
+ spkiDerBase64: input.spkiDerBase64,
71
+ authorityMaterials: input.authorityMaterials,
72
+ });
73
+ }
74
+ else if (input.schema === 'whoisleuth.rpki-route-input') {
75
+ kind = 'rpki';
76
+ result = reviewRpkiRoute({
77
+ routePrefix: input.routePrefix,
78
+ originAsn: input.originAsn,
79
+ authorizations: input.authorizations,
80
+ });
81
+ }
82
+ else if (input.schema === 'whoisleuth.local-geoip-query') {
83
+ kind = 'geoip';
84
+ result = lookupLocalGeoIp(buildLocalGeoIpDatabase(input.database), input.address);
85
+ }
86
+ else if (input.schema === 'whoisleuth.encrypted-dns-plan-input') {
87
+ kind = 'encrypted_dns';
88
+ const query = record(input.query);
89
+ result = planEncryptedDnsQuery(normalizeEncryptedDnsAdapter(input.adapter), { name: query.name, type: query.type });
90
+ }
91
+ else {
92
+ throw new CliUsageError('Offline evidence review does not recognise this input schema.');
93
+ }
94
+ return Object.freeze({
95
+ schema: OFFLINE_EVIDENCE_REVIEW_SCHEMA,
96
+ version: OFFLINE_EVIDENCE_REVIEW_VERSION,
97
+ generatedAt,
98
+ kind,
99
+ result,
100
+ limitations: Object.freeze([
101
+ 'The review is local and uses only the supplied document. It does not refresh, transmit, or independently establish the current completeness of the evidence.',
102
+ ]),
103
+ });
104
+ }
105
+ async function buildOfflineEvidenceReviewWithLocalResources(value, generatedAt = new Date().toISOString(), options = {}) {
106
+ const input = parseInput(value);
107
+ if (input.schema !== LOCAL_MMDB_QUERY_SCHEMA)
108
+ return buildOfflineEvidenceReview(value, generatedAt);
109
+ if (!options.mmdbPath)
110
+ throw new CliUsageError('Local MMDB review requires --mmdb <database-file>.');
111
+ return Object.freeze({
112
+ schema: OFFLINE_EVIDENCE_REVIEW_SCHEMA,
113
+ version: OFFLINE_EVIDENCE_REVIEW_VERSION,
114
+ generatedAt,
115
+ kind: 'geoip',
116
+ result: await reviewLocalMmdb(input, options.mmdbPath),
117
+ limitations: Object.freeze([
118
+ 'The review is local and uses only the supplied query metadata and analyst-supplied database. It does not transmit the address or refresh the database.',
119
+ ]),
120
+ });
121
+ }
122
+ function formatOfflineEvidenceReview(document) {
123
+ const result = record(document.result);
124
+ const state = typeof result.state === 'string' ? result.state : 'reviewed';
125
+ const lines = [
126
+ 'Offline evidence review',
127
+ `Kind ${document.kind.replaceAll('_', ' ')}`,
128
+ `State ${state.replaceAll('_', ' ')}`,
129
+ ];
130
+ if (document.kind === 'rdap_search') {
131
+ const help = record(result.help);
132
+ const plan = record(result.plan);
133
+ lines.push(`Help ${String(help.state ?? 'unavailable').replaceAll('_', ' ')}`);
134
+ lines.push(`Plan ${String(plan.state ?? 'unavailable').replaceAll('_', ' ')}`);
135
+ }
136
+ else {
137
+ for (const [label, key] of [
138
+ ['Records', 'records'],
139
+ ['Rejected', 'rejectedCount'],
140
+ ['Matches', 'matchingAuthorizationCount'],
141
+ ]) {
142
+ const candidate = result[key];
143
+ if (Array.isArray(candidate))
144
+ lines.push(`${label.padEnd(7)}${candidate.length}`);
145
+ else if (typeof candidate === 'number')
146
+ lines.push(`${label.padEnd(7)}${candidate}`);
147
+ }
148
+ }
149
+ lines.push('', 'Limitations:');
150
+ for (const limitation of document.limitations)
151
+ lines.push(` - ${limitation}`);
152
+ const nestedLimitations = Array.isArray(result.limitations) ? result.limitations : [];
153
+ for (const limitation of nestedLimitations) {
154
+ if (typeof limitation === 'string')
155
+ lines.push(` - ${limitation}`);
156
+ }
157
+ return `${lines.join('\n')}\n`;
158
+ }
159
+ export { MAX_OFFLINE_EVIDENCE_INPUT_BYTES, OFFLINE_EVIDENCE_REVIEW_SCHEMA, OFFLINE_EVIDENCE_REVIEW_VERSION, buildOfflineEvidenceReview, buildOfflineEvidenceReviewWithLocalResources, formatOfflineEvidenceReview, };