@slicedearth/whoisleuth-cli 1.36.1 → 1.39.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +17 -2
  2. package/bin/whoisleuth.mjs +41 -24
  3. package/cli/arguments.mjs +236 -5
  4. package/cli/artifact-verify.mjs +56 -14
  5. package/cli/compare.mjs +1 -1
  6. package/cli/completion.mjs +18 -3
  7. package/cli/doctor.mjs +11 -5
  8. package/cli/evidence-command-arguments.mjs +1 -1
  9. package/cli/evidence-command-runner.mjs +2 -2
  10. package/cli/evidence-signing.mjs +4 -4
  11. package/cli/formatters/html.mjs +1 -1
  12. package/cli/formatters/json.mjs +13 -1
  13. package/cli/formatters/terminal.mjs +7 -3
  14. package/cli/investigation-plan.mjs +2 -2
  15. package/cli/local-mmdb-review.mjs +76 -0
  16. package/cli/lookalike-calibration.mjs +121 -0
  17. package/cli/lookup-command-runner.mjs +4 -1
  18. package/cli/lookup-plan.mjs +3 -3
  19. package/cli/lookup-reconcile.mjs +141 -0
  20. package/cli/lookup-timeline.mjs +1 -1
  21. package/cli/mail-review.mjs +1 -1
  22. package/cli/offline-evidence-review.mjs +20 -2
  23. package/cli/registry-doctor.mjs +125 -0
  24. package/cli/risk-calibration.mjs +117 -9
  25. package/cli/runner.mjs +236 -17
  26. package/cli/saved-lookup.mjs +2 -2
  27. package/cli/sharing-review.mjs +202 -0
  28. package/cli/source-reliability.mjs +1 -1
  29. package/docs/cli-reference.md +1110 -0
  30. package/docs/cli.md +224 -940
  31. package/frontend/src/lib/analysis/acquisition-decision-packet.js +1 -1
  32. package/frontend/src/lib/analysis/acquisition-due-diligence.js +6 -6
  33. package/frontend/src/lib/analysis/bulk-domain-comparison.js +12 -12
  34. package/frontend/src/lib/analysis/bulk-mail-exposure.js +1 -1
  35. package/frontend/src/lib/analysis/bulk-session-model.js +7 -2
  36. package/frontend/src/lib/analysis/case-evidence-model.js +26 -4
  37. package/frontend/src/lib/analysis/case-migration-model.js +6 -1
  38. package/frontend/src/lib/analysis/case-record-contracts.js +4 -2
  39. package/frontend/src/lib/analysis/case-record-core.js +5 -1
  40. package/frontend/src/lib/analysis/case-record-operations.js +6 -1
  41. package/frontend/src/lib/analysis/case-response-packet.js +7 -7
  42. package/frontend/src/lib/analysis/evidence-coverage-ledger.js +328 -0
  43. package/frontend/src/lib/analysis/investigation-capsule.js +105 -0
  44. package/frontend/src/lib/analysis/lookup-asset-graph.js +806 -0
  45. package/frontend/src/lib/analysis/lookup-decision-support.js +480 -0
  46. package/frontend/src/lib/analysis/lookup-investigation-brief.js +153 -0
  47. package/frontend/src/lib/analysis/lookup-presentation.js +74 -0
  48. package/frontend/src/lib/analysis/lookup-response.js +2 -0
  49. package/frontend/src/lib/analysis/lookup-source-refresh.js +304 -0
  50. package/frontend/src/lib/analysis/lookup-summary-model.js +398 -0
  51. package/frontend/src/lib/analysis/page-similarity.js +1 -1
  52. package/frontend/src/lib/analysis/relationship-evidence.js +3 -3
  53. package/frontend/src/lib/analysis/relationship-observation-model.js +4 -4
  54. package/frontend/src/lib/analysis/scoring.js +8 -88
  55. package/frontend/src/lib/analysis/shortlist-model.js +8 -4
  56. package/frontend/src/lib/analysis/visualization-bounds.js +26 -0
  57. package/frontend/src/lib/analysis/visualization-force-graph.js +352 -0
  58. package/frontend/src/lib/analysis/visualization-matrix.js +88 -0
  59. package/frontend/src/lib/analysis/visualization-models.js +370 -0
  60. package/frontend/src/lib/analysis/visualization-time-series.js +144 -0
  61. package/frontend/src/lib/analysis/workspace-archive-crypto.js +1 -1
  62. package/frontend/src/lib/analysis/workspace-archive.js +2 -2
  63. package/lib/analyst-taxonomy.mjs +40 -0
  64. package/lib/availability.mjs +1 -1
  65. package/lib/browser-library-profile.mjs +15 -2
  66. package/lib/certificate-extension-profile.mjs +71 -0
  67. package/lib/credential-surface-profile.mjs +1 -1
  68. package/lib/domain-assurance.mjs +368 -0
  69. package/lib/domain-control-manifest.mjs +50 -7
  70. package/lib/domain-posture.mjs +9 -9
  71. package/lib/encrypted-dns-contract.mjs +1 -1
  72. package/lib/evidence-report-markdown.mjs +1 -1
  73. package/lib/evidence-report.mjs +1 -1
  74. package/lib/generated/cisa-kev-catalog.mjs +1668 -0
  75. package/lib/ip-prefix.mjs +34 -1
  76. package/lib/local-geoip-evidence.mjs +2 -2
  77. package/lib/lookup-response-contract.mjs +346 -0
  78. package/lib/observed-network-context.mjs +1 -1
  79. package/lib/opportunity-scoring.mjs +107 -0
  80. package/lib/page-fingerprints.mjs +1 -1
  81. package/lib/rdap-capabilities.mjs +6 -6
  82. package/lib/rdap-search-workbench.mjs +96 -6
  83. package/lib/registry-access.mjs +1 -1
  84. package/lib/registry-capability-catalogue.mjs +4 -4
  85. package/lib/registry-insights.mjs +6 -6
  86. package/lib/risk-scoring.mjs +206 -83
  87. package/lib/scoring-evidence-quality.mjs +107 -0
  88. package/lib/security-txt.mjs +3 -3
  89. package/lib/threat-intelligence-manifests.mjs +1 -1
  90. package/lib/threatfox-intelligence.mjs +1 -1
  91. package/lib/tls-intelligence.mjs +13 -3
  92. package/lib/urlhaus-intelligence.mjs +1 -1
  93. package/lib/website-security-posture.mjs +5 -5
  94. package/lib/website-technology.mjs +1 -1
  95. package/lib/whois-chain.mjs +1 -1
  96. package/package.json +5 -1
  97. package/third-party-notices.txt +911 -19
@@ -2,7 +2,7 @@ import { CLI_COMMANDS } from "./arguments.mjs";
2
2
  import { INVESTIGATION_PLAN_RECIPES } from "./investigation-plan.mjs";
3
3
  const COMMON_OPTIONS = Object.freeze(['--help', '--output', '--force']);
4
4
  const OPTIONS_BY_COMMAND = Object.freeze({
5
- lookup: ['--json', '--markdown', '--html', '--fast', '--deep', '--plan', '--summary', '--verbose', '--strict-exit', '--events', '--quiet', '--no-color'],
5
+ lookup: ['--json', '--markdown', '--html', '--fast', '--deep', '--observer', '--vantage', '--plan', '--summary', '--verbose', '--strict-exit', '--events', '--quiet', '--no-color'],
6
6
  bulk: ['--json', '--jsonl', '--csv', '--domains', '--queries', '--registered-only', '--inconclusive-only', '--errors-only', '--fast', '--deep', '--concurrency', '--checkpoint', '--resume', '--events', '--quiet', '--no-color'],
7
7
  'ct-search': ['--json', '--quiet', '--no-color'],
8
8
  discover: ['--tlds', '--preset', '--families', '--keyboard', '--dictionary', '--snapshot', '--json', '--jsonl', '--domains', '--quiet', '--no-color'],
@@ -11,7 +11,9 @@ const OPTIONS_BY_COMMAND = Object.freeze({
11
11
  http: ['--json', '--quiet', '--no-color'],
12
12
  tls: ['--json', '--quiet', '--no-color'],
13
13
  'registry-support': ['--json', '--quiet', '--no-color'],
14
+ 'registry-doctor': ['--json', '--quiet', '--no-color'],
14
15
  'risk-calibrate': ['--json', '--quiet', '--no-color'],
16
+ 'lookalike-calibrate': ['--json', '--quiet', '--no-color'],
15
17
  'verify-artifact': ['--passphrase-file', '--json', '--quiet', '--no-color'],
16
18
  'inspect-archive': ['--passphrase-file', '--search', '--require-match', '--reveal', '--json', '--quiet', '--no-color'],
17
19
  'sign-artifact': ['--private-key-file'],
@@ -20,10 +22,13 @@ const OPTIONS_BY_COMMAND = Object.freeze({
20
22
  compare: ['--json', '--quiet', '--no-color'],
21
23
  'page-compare': ['--json', '--quiet', '--no-color'],
22
24
  'mail-review': ['--json', '--quiet', '--no-color'],
23
- 'review-evidence': ['--json', '--quiet', '--no-color'],
25
+ 'review-evidence': ['--mmdb', '--json', '--quiet', '--no-color'],
24
26
  'domain-control': ['--json', '--quiet', '--no-color'],
27
+ assurance: ['--json', '--quiet', '--no-color'],
28
+ 'sharing-review': ['--marking', '--recipient-scope', '--purpose', '--human-reviewed', '--personal-data-reviewed', '--redactions-confirmed', '--json', '--quiet', '--no-color'],
25
29
  'workflow-plan': ['--json', '--quiet', '--no-color'],
26
30
  diff: ['--json', '--quiet', '--no-color'],
31
+ reconcile: ['--json', '--quiet', '--no-color'],
27
32
  timeline: ['--json', '--quiet', '--no-color'],
28
33
  export: ['--markdown', '--html', '--compact'],
29
34
  completion: [],
@@ -41,10 +46,12 @@ const COMMAND_DESCRIPTIONS = Object.freeze({
41
46
  http: 'Inspect one homepage request',
42
47
  tls: 'Inspect one TLS connection',
43
48
  'registry-support': 'Explain local registry coverage',
49
+ 'registry-doctor': 'Diagnose saved registry collection',
44
50
  'risk-calibrate': 'Replay reviewed Risk labels offline',
51
+ 'lookalike-calibrate': 'Summarise reviewed lookalike yield offline',
45
52
  'verify-artifact': 'Validate saved evidence offline',
46
53
  'inspect-archive': 'Inspect an archive locally',
47
- 'sign-artifact': 'Sign a reviewed artifact locally',
54
+ 'sign-artifact': 'Sign a reviewed artefact locally',
48
55
  'verify-signature': 'Verify a signed evidence package',
49
56
  'source-report': 'Build a target-free source report',
50
57
  compare: 'Compare registry publications in one lookup',
@@ -52,8 +59,11 @@ const COMMAND_DESCRIPTIONS = Object.freeze({
52
59
  'mail-review': 'Review saved passive mail evidence',
53
60
  'review-evidence': 'Review supplied evidence offline',
54
61
  'domain-control': 'Build or review a domain control manifest',
62
+ assurance: 'Review domain change, recovery, or retirement plans',
63
+ 'sharing-review': 'Lint an artefact before deliberate sharing',
55
64
  'workflow-plan': 'Plan a fixed investigation recipe',
56
65
  diff: 'Compare two saved domain lookups',
66
+ reconcile: 'Reconcile independently labelled observations',
57
67
  timeline: 'Build same-domain history from saved lookups',
58
68
  export: 'Convert a lookup to an evidence report',
59
69
  completion: 'Print shell completion',
@@ -65,6 +75,8 @@ const VALUE_OPTIONS = Object.freeze({
65
75
  '--preset': ['common', 'impersonation', 'all'],
66
76
  '--keyboard': ['qwerty', 'azerty', 'qwertz', 'all'],
67
77
  '--mail-profile': ['standard', 'defensive-no-mail', 'parked'],
78
+ '--marking': ['clear', 'green', 'amber', 'amber-strict', 'red'],
79
+ '--recipient-scope': ['public', 'community', 'organization', 'named-recipients'],
68
80
  '--concurrency': ['1', '2', '3', '4', '5', '6', '7', '8'],
69
81
  });
70
82
  const FILE_OPTIONS = Object.freeze([
@@ -81,10 +93,13 @@ const FILE_OPTIONS = Object.freeze([
81
93
  const TEXT_OPTIONS = Object.freeze([
82
94
  '--families',
83
95
  '--resolver',
96
+ '--purpose',
97
+ '--observer',
84
98
  '--retired-selectors',
85
99
  '--search',
86
100
  '--selectors',
87
101
  '--tlds',
102
+ '--vantage',
88
103
  ]);
89
104
  function commandOptions(command) {
90
105
  return [...COMMON_OPTIONS, ...(OPTIONS_BY_COMMAND[command] || [])].join(' ');
package/cli/doctor.mjs CHANGED
@@ -1,6 +1,4 @@
1
- import { resolvePublicAddresses, safeFetch } from "../lib/safe-fetch.mjs";
2
1
  import { withTimeout } from "../lib/abort.mjs";
3
- import { whoisQuery } from "../lib/whois-transport.mjs";
4
2
  const DOCTOR_SCHEMA = 'whoisleuth.cli.doctor';
5
3
  const DOCTOR_VERSION = 1;
6
4
  const MINIMUM_NODE_MAJOR = 24;
@@ -62,9 +60,17 @@ async function buildDoctorReport(options) {
62
60
  }));
63
61
  }
64
62
  else {
65
- const resolve = options.resolveAddresses || resolvePublicAddresses;
66
- const fetchHttps = options.fetchHttps || safeFetch;
67
- const query = options.queryWhois || whoisQuery;
63
+ const safeFetchModule = options.resolveAddresses && options.fetchHttps
64
+ ? null
65
+ : await import("../lib/safe-fetch.mjs");
66
+ const whoisModule = options.queryWhois
67
+ ? null
68
+ : await import("../lib/whois-transport.mjs");
69
+ const resolve = options.resolveAddresses || safeFetchModule?.resolvePublicAddresses;
70
+ const fetchHttps = options.fetchHttps || safeFetchModule?.safeFetch;
71
+ const query = options.queryWhois || whoisModule?.whoisQuery;
72
+ if (!resolve || !fetchHttps || !query)
73
+ throw new TypeError('Network diagnostic dependencies are unavailable.');
68
74
  const networkTimeoutMs = Number.isSafeInteger(options.networkTimeoutMs)
69
75
  && Number(options.networkTimeoutMs) >= 1
70
76
  && Number(options.networkTimeoutMs) <= NETWORK_TIMEOUT_MS
@@ -92,7 +92,7 @@ export function parseSignArtifactArguments(argv) {
92
92
  else if (source === null)
93
93
  source = argument;
94
94
  else
95
- throw new CliUsageError('sign-artifact accepts one optional JSON file. Otherwise pipe one artifact on stdin.');
95
+ throw new CliUsageError('sign-artifact accepts one optional JSON file. Otherwise pipe one artefact on stdin.');
96
96
  }
97
97
  if (!privateKeySource)
98
98
  throw new CliUsageError('sign-artifact requires --private-key-file.');
@@ -80,9 +80,9 @@ export async function runEvidenceCommand(args, dependencies) {
80
80
  return EXIT_CODES.SUCCESS;
81
81
  }
82
82
  if (args.action === 'sign-artifact') {
83
- const input = await readArtifact(args.source, 'Evidence artifact input', dependencies);
83
+ const input = await readArtifact(args.source, 'Evidence artefact input', dependencies);
84
84
  if (!input.trim()) {
85
- throw new CliUsageError('sign-artifact requires one reviewed artifact file or an artifact on stdin.');
85
+ throw new CliUsageError('sign-artifact requires one reviewed artefact file or an artefact on stdin.');
86
86
  }
87
87
  const privateKey = await readKey(args.privateKeySource, 'Private key file', dependencies.readPrivateKeyFile);
88
88
  const signed = await signEvidencePackage(input, privateKey, dependencies.now ? dependencies.now() : new Date().toISOString());
@@ -100,10 +100,10 @@ function payloadBytes(payload) {
100
100
  return Buffer.from(canonicalArtifactJson(payload), 'utf8');
101
101
  }
102
102
  async function signableArtifact(raw) {
103
- const artifact = parseObject(raw, 'Evidence artifact');
103
+ const artifact = parseObject(raw, 'Evidence artefact');
104
104
  const verification = await verifyOfflineArtifact(raw);
105
- if (!['case_response_packet', 'signed_review_artifact'].includes(verification.artifact.kind)) {
106
- throw new TypeError('Only reviewed response packets and supported review manifests can be signed.');
105
+ if (!['case_response_packet', 'investigation_capsule', 'signed_review_artifact'].includes(verification.artifact.kind)) {
106
+ throw new TypeError('Only reviewed response packets, investigation capsules, and supported review manifests can be signed.');
107
107
  }
108
108
  return { artifact, verification };
109
109
  }
@@ -193,7 +193,7 @@ export async function verifyEvidencePackageSignature(raw, trustedPublicKeyPem) {
193
193
  trustedPublicKeyPem
194
194
  ? 'The signature and supplied public key match; recipient trust still depends on obtaining that public key through an authenticated channel.'
195
195
  : 'The signature is internally valid for the embedded public key, but signer identity was not authenticated because no trusted public key was supplied.',
196
- 'The signature authenticates the canonical sorted-JSON content, not the original whitespace, indentation, or object-key order of the serialized file.',
196
+ 'The signature authenticates the canonical sorted-JSON content, not the original whitespace, indentation, or object-key order of the serialised file.',
197
197
  'A valid signature proves the canonical package content has not changed since signing; it does not establish that the retained observations or analyst statements are accurate.',
198
198
  'WHOISleuth does not generate, store, recover, rotate, or publish signing keys.',
199
199
  ]),
@@ -12,7 +12,7 @@ function renderGroups(groups) {
12
12
  }
13
13
  function renderComparison(comparison) {
14
14
  if (!comparison.fields.length)
15
- return '<p class="empty">No comparable normalized fields were published.</p>';
15
+ return '<p class="empty">No comparable normalised fields were published.</p>';
16
16
  const rows = comparison.fields.map((field) => `<tr><th scope="row">${escapeHtml(field.label)}</th><td><span class="status">${escapeHtml(field.status)}</span></td><td>${escapeHtml(field.rdap)}</td><td>${escapeHtml(field.whois)}</td></tr>`).join('');
17
17
  const omitted = comparison.omitted
18
18
  ? `<p class="omission">${escapeHtml(comparison.omitted)} additional comparison fields omitted.</p>`
@@ -17,7 +17,13 @@ const CLI_POSTURE_SCHEMA_VERSION = 1;
17
17
  const CLI_HTTP_SCHEMA_VERSION = 1;
18
18
  const CLI_TLS_SCHEMA_VERSION = 1;
19
19
  const CLI_COMPARE_SCHEMA_VERSION = 3;
20
- function buildCliLookupDocument(query, classified, result, generatedAt = new Date().toISOString(), mode = 'fast') {
20
+ function buildCliLookupDocument(query, classified, result, generatedAt = new Date().toISOString(), mode = 'fast', collectionContext = {}) {
21
+ const observerLabel = typeof collectionContext.observerLabel === 'string'
22
+ ? collectionContext.observerLabel
23
+ : null;
24
+ const vantageLabel = typeof collectionContext.vantageLabel === 'string'
25
+ ? collectionContext.vantageLabel
26
+ : null;
21
27
  return {
22
28
  schema: CLI_LOOKUP_SCHEMA,
23
29
  version: CLI_LOOKUP_SCHEMA_VERSION,
@@ -28,6 +34,12 @@ function buildCliLookupDocument(query, classified, result, generatedAt = new Dat
28
34
  inputHostname: classified.inputHostname,
29
35
  registrableDomain: classified.registrableDomain,
30
36
  isSubdomain: classified.isSubdomain,
37
+ ...((observerLabel || vantageLabel) ? {
38
+ collectionContext: {
39
+ ...(observerLabel ? { observerLabel } : {}),
40
+ ...(vantageLabel ? { vantageLabel } : {}),
41
+ },
42
+ } : {}),
31
43
  ...result,
32
44
  };
33
45
  }
@@ -593,7 +593,7 @@ function formatTerminalCompare(document) {
593
593
  }
594
594
  lines.push('');
595
595
  if (!fields.length) {
596
- lines.push('Neither source published a comparable normalized field.');
596
+ lines.push('Neither source published a comparable normalised field.');
597
597
  }
598
598
  else {
599
599
  for (const value of fields) {
@@ -613,7 +613,7 @@ function formatTerminalCompare(document) {
613
613
  const publicationDifferences = publicationFields.length - (Number(publicationCounts.equivalent) || 0);
614
614
  lines.push('', 'Registry / registrar RDAP publication', `Registry RDAP ${comparisonStatusLabel(registryHealth.status)}`, `Registrar RDAP ${comparisonStatusLabel(registrarHealth.status)}`, `Compared ${safeTerminalValue(publicationFields.length, '0')} field${publicationFields.length === 1 ? '' : 's'}`, `Equivalent ${safeTerminalValue(publicationCounts.equivalent, '0')}`, `Differences ${safeTerminalValue(publicationDifferences, '0')}`, '');
615
615
  if (!publicationFields.length) {
616
- lines.push('Neither RDAP publication exposed a comparable normalized field.');
616
+ lines.push('Neither RDAP publication exposed a comparable normalised field.');
617
617
  }
618
618
  else {
619
619
  for (const value of publicationFields) {
@@ -635,6 +635,7 @@ function formatTerminalRiskCalibration(document) {
635
635
  const bands = terminalRecord(summary.scoreBands);
636
636
  const thresholds = Array.isArray(document.thresholds) ? document.thresholds : [];
637
637
  const records = Array.isArray(document.records) ? document.records : [];
638
+ const comparison = terminalRecord(document.modelComparison);
638
639
  const visible = records.slice(0, MAX_RISK_CALIBRATION_TERMINAL_RECORDS);
639
640
  const lines = [
640
641
  `Risk model v${safeTerminalValue(document.riskModelVersion)}`,
@@ -648,7 +649,10 @@ function formatTerminalRiskCalibration(document) {
648
649
  ];
649
650
  for (const value of thresholds) {
650
651
  const threshold = terminalRecord(value);
651
- lines.push(`${String(safeTerminalValue(threshold.threshold)).padStart(3)}+ TP ${safeTerminalValue(threshold.truePositive, '0')} FP ${safeTerminalValue(threshold.falsePositive, '0')} TN ${safeTerminalValue(threshold.trueNegative, '0')} FN ${safeTerminalValue(threshold.falseNegative, '0')} precision ${safeTerminalValue(threshold.precision)} recall ${safeTerminalValue(threshold.recall)}`);
652
+ lines.push(`${String(safeTerminalValue(threshold.threshold)).padStart(3)}+ TP ${safeTerminalValue(threshold.truePositive, '0')} FP ${safeTerminalValue(threshold.falsePositive, '0')} TN ${safeTerminalValue(threshold.trueNegative, '0')} FN ${safeTerminalValue(threshold.falseNegative, '0')} precision ${safeTerminalValue(threshold.precision)} recall ${safeTerminalValue(threshold.recall)} F1 ${safeTerminalValue(threshold.f1)}`);
653
+ }
654
+ if (comparison.available === true) {
655
+ lines.push('', `Model comparison v${safeTerminalValue(comparison.previousModelVersion)} → v${safeTerminalValue(comparison.currentModelVersion)} · ${safeTerminalValue(comparison.scoresChanged, '0')} score changes · ${safeTerminalValue(comparison.thresholdClassificationsChanged, '0')} review-band changes`);
652
656
  }
653
657
  lines.push('', 'Records');
654
658
  for (const value of visible) {
@@ -15,7 +15,7 @@ const RECIPES = Object.freeze({
15
15
  steps: (domain) => Object.freeze([
16
16
  step('collect', 'Collect a Deep lookup', 'lookup', [domain, '--deep', '--json'], 'network', 'network_disclosure', 'whoisleuth.cli.lookup', 'Review source health and limitations before using missing fields.'),
17
17
  step('export', 'Create a portable evidence report', 'export', ['<saved-lookup.json>'], 'offline', 'analyst_selection', 'whoisleuth.lookup-evidence', 'Select the reviewed lookup file; the plan never guesses a path.'),
18
- step('verify', 'Verify the exported artifact', 'verify-artifact', ['<evidence.json>', '--json'], 'offline', 'analyst_selection', 'whoisleuth.offline-artifact-verification', 'Keep verification distinct from a claim that the observations are correct or current.'),
18
+ step('verify', 'Verify the exported artefact', 'verify-artifact', ['<evidence.json>', '--json'], 'offline', 'analyst_selection', 'whoisleuth.offline-artifact-verification', 'Keep verification distinct from a claim that the observations are correct or current.'),
19
19
  ]),
20
20
  }),
21
21
  'lookalike-review': Object.freeze({
@@ -89,7 +89,7 @@ export function buildInvestigationPlan(recipeId, subjectValue, generatedAtValue
89
89
  limitations: Object.freeze([
90
90
  'This document is a fixed plan. It does not execute commands, expand placeholders, make requests, read files, change cases, or submit reports.',
91
91
  'Network steps require deliberate execution and disclose the selected target to the sources described by that command.',
92
- 'Analyst-selection steps require reviewed local artifacts; placeholders are never interpreted as file paths by this planner.',
92
+ 'Analyst-selection steps require reviewed local artefacts; placeholders are never interpreted as file paths by this planner.',
93
93
  ]),
94
94
  });
95
95
  }
@@ -0,0 +1,76 @@
1
+ import { stat } from 'node:fs/promises';
2
+ import { isIP } from 'node:net';
3
+ import { open } from 'maxmind';
4
+ import { formatIpPrefix, parseIpPrefix } from "../lib/ip-prefix.mjs";
5
+ import { CliUsageError } from "./errors.mjs";
6
+ export const LOCAL_MMDB_QUERY_SCHEMA = 'whoisleuth.local-mmdb-query';
7
+ export const MAX_LOCAL_MMDB_BYTES = 512 * 1024 * 1024;
8
+ function record(value) {
9
+ return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
10
+ }
11
+ function boundedText(value, maximum) {
12
+ return typeof value === 'string' && value.length <= maximum && !/[\u0000-\u001f\u007f]/u.test(value)
13
+ ? value.replace(/\s+/gu, ' ').trim() || null
14
+ : null;
15
+ }
16
+ function names(value) {
17
+ const source = record(value);
18
+ return boundedText(source.en, 120);
19
+ }
20
+ export async function reviewLocalMmdb(input, databasePath) {
21
+ const address = boundedText(input.address, 64)?.toLowerCase() ?? '';
22
+ const sourceLabel = boundedText(input.sourceLabel, 120);
23
+ const databaseVersion = boundedText(input.databaseVersion, 80);
24
+ const license = boundedText(input.license, 240);
25
+ if (!isIP(address))
26
+ throw new CliUsageError('Local MMDB review requires one valid IP address.');
27
+ if (!sourceLabel || !databaseVersion || !license) {
28
+ throw new CliUsageError('Local MMDB review requires bounded sourceLabel, databaseVersion, and licence metadata.');
29
+ }
30
+ let info;
31
+ try {
32
+ info = await stat(databasePath);
33
+ }
34
+ catch {
35
+ throw new CliUsageError('The supplied MMDB file could not be read.');
36
+ }
37
+ if (!info.isFile() || info.size <= 0 || info.size > MAX_LOCAL_MMDB_BYTES) {
38
+ throw new CliUsageError(`The supplied MMDB must be a file no larger than ${MAX_LOCAL_MMDB_BYTES} bytes.`);
39
+ }
40
+ let reader;
41
+ try {
42
+ reader = await open(databasePath, { cache: { max: 1_000 } });
43
+ }
44
+ catch {
45
+ throw new CliUsageError('The supplied file is not a readable MaxMind DB database.');
46
+ }
47
+ const [rawMatch, prefixLength] = reader.getWithPrefixLength(address);
48
+ const match = record(rawMatch);
49
+ const traits = record(match.traits);
50
+ const country = record(match.country);
51
+ const registeredCountry = record(match.registered_country);
52
+ const subdivisions = Array.isArray(match.subdivisions) ? match.subdivisions : [];
53
+ const subdivision = record(subdivisions[0]);
54
+ const city = record(match.city);
55
+ const asnValue = match.autonomous_system_number ?? traits.autonomous_system_number;
56
+ const asn = typeof asnValue === 'number' && Number.isSafeInteger(asnValue) && asnValue >= 0 ? asnValue : null;
57
+ const prefix = parseIpPrefix(`${address}/${prefixLength}`);
58
+ return Object.freeze({
59
+ state: rawMatch ? 'matched' : 'not_found',
60
+ address,
61
+ match: rawMatch ? Object.freeze({
62
+ network: prefix ? formatIpPrefix(prefix) : null,
63
+ countryCode: boundedText(country.iso_code ?? registeredCountry.iso_code, 2)?.toUpperCase() ?? null,
64
+ region: names(subdivision.names) ?? boundedText(subdivision.iso_code, 120),
65
+ city: names(city.names),
66
+ asn,
67
+ asName: boundedText(match.autonomous_system_organization ?? traits.autonomous_system_organization, 240),
68
+ }) : null,
69
+ source: Object.freeze({ label: sourceLabel, version: databaseVersion, license }),
70
+ limitations: Object.freeze([
71
+ 'This CLI-only result comes from an analyst-supplied local MMDB file. WHOISleuth does not bundle, download, update, license, or transmit the database.',
72
+ 'Geolocation and network attribution are provider estimates and may be stale, coarse, shared, or incorrect.',
73
+ 'A location or ASN association does not establish the operator, owner, user, intent, safety, or maliciousness of an address.',
74
+ ]),
75
+ });
76
+ }
@@ -0,0 +1,121 @@
1
+ import { Buffer } from 'node:buffer';
2
+ const LOOKALIKE_CALIBRATION_INPUT_SCHEMA = 'whoisleuth.lookalike-calibration-input';
3
+ const LOOKALIKE_CALIBRATION_SCHEMA = 'whoisleuth.lookalike-calibration';
4
+ const LOOKALIKE_CALIBRATION_VERSION = 1;
5
+ const MAX_LOOKALIKE_CALIBRATION_BYTES = 2 * 1024 * 1024;
6
+ const MAX_LOOKALIKE_CALIBRATION_RECORDS = 5_000;
7
+ const MIN_LOOKALIKE_CALIBRATION_SAMPLE = 20;
8
+ const DISPOSITIONS = ['suspicious', 'confirmed_abuse', 'false_positive', 'expected', 'closed_no_action'];
9
+ function record(value) {
10
+ return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
11
+ }
12
+ function token(value, field) {
13
+ if (typeof value !== 'string')
14
+ throw new TypeError(`${field} must be text.`);
15
+ const normalized = value.trim().toLowerCase();
16
+ if (!/^[a-z0-9][a-z0-9_-]{0,79}$/u.test(normalized))
17
+ throw new TypeError(`${field} must be a bounded identifier.`);
18
+ return normalized;
19
+ }
20
+ function ratio(value, total) {
21
+ return total ? Number((value / total).toFixed(4)) : 0;
22
+ }
23
+ function emptyCounts() {
24
+ return { suspicious: 0, confirmed_abuse: 0, false_positive: 0, expected: 0, closed_no_action: 0 };
25
+ }
26
+ function buildLookalikeCalibration(raw, generatedAt = new Date().toISOString()) {
27
+ if (Buffer.byteLength(raw, 'utf8') > MAX_LOOKALIKE_CALIBRATION_BYTES) {
28
+ throw new TypeError(`Lookalike calibration input is limited to ${MAX_LOOKALIKE_CALIBRATION_BYTES} bytes.`);
29
+ }
30
+ let parsed;
31
+ try {
32
+ parsed = JSON.parse(raw);
33
+ }
34
+ catch {
35
+ throw new TypeError('Lookalike calibration input must be valid JSON.');
36
+ }
37
+ const input = record(parsed);
38
+ if (input.schema !== LOOKALIKE_CALIBRATION_INPUT_SCHEMA || input.version !== LOOKALIKE_CALIBRATION_VERSION) {
39
+ throw new TypeError(`Lookalike calibration input must use ${LOOKALIKE_CALIBRATION_INPUT_SCHEMA} version ${LOOKALIKE_CALIBRATION_VERSION}.`);
40
+ }
41
+ if (!Array.isArray(input.records) || input.records.length < 1 || input.records.length > MAX_LOOKALIKE_CALIBRATION_RECORDS) {
42
+ throw new TypeError(`Lookalike calibration input must contain from 1 to ${MAX_LOOKALIKE_CALIBRATION_RECORDS} reviewed records.`);
43
+ }
44
+ const ids = new Set();
45
+ const familyCounts = new Map();
46
+ const totalCounts = emptyCounts();
47
+ for (let index = 0; index < input.records.length; index++) {
48
+ const item = record(input.records[index]);
49
+ const id = token(item.id, `records[${index}].id`);
50
+ if (ids.has(id))
51
+ throw new TypeError(`Lookalike calibration record id "${id}" is duplicated.`);
52
+ ids.add(id);
53
+ if (typeof item.disposition !== 'string' || !DISPOSITIONS.includes(item.disposition)) {
54
+ throw new TypeError(`records[${index}].disposition is not a supported reviewed disposition.`);
55
+ }
56
+ const disposition = item.disposition;
57
+ if (!Array.isArray(item.mutationFamilies) || item.mutationFamilies.length < 1 || item.mutationFamilies.length > 12) {
58
+ throw new TypeError(`records[${index}].mutationFamilies must contain from 1 to 12 identifiers.`);
59
+ }
60
+ const families = [...new Set(item.mutationFamilies.map((value) => token(value, `records[${index}].mutationFamilies`)))];
61
+ totalCounts[disposition] += 1;
62
+ for (const family of families) {
63
+ const counts = familyCounts.get(family) || emptyCounts();
64
+ counts[disposition] += 1;
65
+ familyCounts.set(family, counts);
66
+ }
67
+ }
68
+ const families = [...familyCounts.entries()].map(([id, dispositions]) => {
69
+ const reviewedCandidates = Object.values(dispositions).reduce((sum, count) => sum + count, 0);
70
+ return {
71
+ id,
72
+ reviewedCandidates,
73
+ sampleState: reviewedCandidates >= MIN_LOOKALIKE_CALIBRATION_SAMPLE ? 'reviewed' : 'insufficient',
74
+ dispositions,
75
+ reviewLeadRate: ratio(dispositions.suspicious + dispositions.confirmed_abuse, reviewedCandidates),
76
+ falsePositiveRate: ratio(dispositions.false_positive, reviewedCandidates),
77
+ };
78
+ }).sort((left, right) => right.reviewedCandidates - left.reviewedCandidates || left.id.localeCompare(right.id));
79
+ const reviewedCandidates = input.records.length;
80
+ return {
81
+ schema: LOOKALIKE_CALIBRATION_SCHEMA,
82
+ version: LOOKALIKE_CALIBRATION_VERSION,
83
+ generatedAt,
84
+ summary: {
85
+ reviewedCandidates,
86
+ mutationFamilies: families.length,
87
+ sampleState: reviewedCandidates >= MIN_LOOKALIKE_CALIBRATION_SAMPLE ? 'reviewed' : 'insufficient',
88
+ reviewLeadRate: ratio(totalCounts.suspicious + totalCounts.confirmed_abuse, reviewedCandidates),
89
+ falsePositiveRate: ratio(totalCounts.false_positive, reviewedCandidates),
90
+ expectedRate: ratio(totalCounts.expected, reviewedCandidates),
91
+ },
92
+ families,
93
+ privacy: { candidateIdsRetained: 0, domainsRetained: 0, notesRetained: 0 },
94
+ limitations: [
95
+ 'This offline report summarises analyst dispositions for the supplied reviewed sample and never changes mutation generation, ordering, Risk, or filtering automatically.',
96
+ 'Suspicious and confirmed-abuse dispositions are counted as review leads, not proof of maliciousness; expected and false-positive dispositions are not proof of safety.',
97
+ `A family remains an insufficient sample below ${MIN_LOOKALIKE_CALIBRATION_SAMPLE} reviewed candidates, and even larger local samples may not generalize across brands, languages, registries, or time.`,
98
+ 'Candidate identifiers, domains, notes, and evidence are intentionally omitted from output. Retain the reviewed source dataset under an appropriate local privacy policy.',
99
+ ],
100
+ };
101
+ }
102
+ function formatLookalikeCalibration(report) {
103
+ const output = [
104
+ 'Lookalike review-yield calibration',
105
+ `Reviewed candidates ${report.summary.reviewedCandidates}`,
106
+ `Mutation families ${report.summary.mutationFamilies}`,
107
+ `Sample state ${report.summary.sampleState}`,
108
+ `Review-lead rate ${(report.summary.reviewLeadRate * 100).toFixed(1)}%`,
109
+ `False-positive rate ${(report.summary.falsePositiveRate * 100).toFixed(1)}%`,
110
+ '',
111
+ ];
112
+ for (const family of report.families) {
113
+ output.push(`${family.id} [${family.sampleState}]`);
114
+ output.push(` ${family.reviewedCandidates} reviewed · ${(family.reviewLeadRate * 100).toFixed(1)}% review leads · ${(family.falsePositiveRate * 100).toFixed(1)}% false positives`);
115
+ }
116
+ output.push('', 'Limitations:');
117
+ for (const limitation of report.limitations)
118
+ output.push(` - ${limitation}`);
119
+ return `${output.join('\n')}\n`;
120
+ }
121
+ export { LOOKALIKE_CALIBRATION_INPUT_SCHEMA, LOOKALIKE_CALIBRATION_SCHEMA, LOOKALIKE_CALIBRATION_VERSION, MAX_LOOKALIKE_CALIBRATION_BYTES, MAX_LOOKALIKE_CALIBRATION_RECORDS, MIN_LOOKALIKE_CALIBRATION_SAMPLE, buildLookalikeCalibration, formatLookalikeCalibration, };
@@ -68,7 +68,10 @@ async function runLookupCommand(args, dependencies, context) {
68
68
  context.endProgress();
69
69
  }
70
70
  const now = context.now();
71
- const document = buildCliLookupDocument(query, classified, result, now, args.deep ? 'deep' : 'fast');
71
+ const document = buildCliLookupDocument(query, classified, result, now, args.deep ? 'deep' : 'fast', {
72
+ ...(args.observerLabel ? { observerLabel: args.observerLabel } : {}),
73
+ ...(args.vantageLabel ? { vantageLabel: args.vantageLabel } : {}),
74
+ });
72
75
  if (!args.quiet) {
73
76
  if (args.output === 'json')
74
77
  context.writeStdout(formatJsonDocument(document));
@@ -3,13 +3,13 @@ export const CLI_LOOKUP_PLAN_VERSION = 1;
3
3
  const RDAP = Object.freeze({
4
4
  source: 'rdap',
5
5
  purpose: 'Collect authoritative registration or allocation evidence where supported.',
6
- disclosure: 'The normalized target is sent to the applicable RDAP bootstrap and service endpoints.',
6
+ disclosure: 'The normalised target is sent to the applicable RDAP bootstrap and service endpoints.',
7
7
  conditional: false,
8
8
  });
9
9
  const WHOIS = Object.freeze({
10
10
  source: 'whois',
11
11
  purpose: 'Collect separately attributed registry and referral publications.',
12
- disclosure: 'The normalized target is sent over bounded TCP connections to applicable WHOIS services.',
12
+ disclosure: 'The normalised target is sent over bounded TCP connections to applicable WHOIS services.',
13
13
  conditional: false,
14
14
  });
15
15
  const FAST_DOMAIN_EVIDENCE = Object.freeze({
@@ -39,7 +39,7 @@ const NETWORK_CONTEXT = Object.freeze({
39
39
  const REVERSE_DNS = Object.freeze({
40
40
  source: 'reverse_dns',
41
41
  purpose: 'Collect operator-published reverse-DNS context for a public IP address.',
42
- disclosure: 'A DNS resolver receives the reverse lookup for the normalized IP address.',
42
+ disclosure: 'A DNS resolver receives the reverse lookup for the normalised IP address.',
43
43
  conditional: false,
44
44
  });
45
45
  function plannedSources(classified, deep) {
@@ -0,0 +1,141 @@
1
+ import { CliUsageError } from "./errors.mjs";
2
+ import { buildCliLookupDiff } from "./lookup-diff.mjs";
3
+ import { parseSavedLookupDocument } from "./saved-lookup.mjs";
4
+ const CLI_LOOKUP_RECONCILIATION_SCHEMA = 'whoisleuth.cli.lookup-reconciliation';
5
+ const CLI_LOOKUP_RECONCILIATION_VERSION = 1;
6
+ const MAX_LOOKUP_RECONCILIATION_INPUT_BYTES = 32 * 1024 * 1024;
7
+ function object(value) {
8
+ return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
9
+ }
10
+ function label(value) {
11
+ if (typeof value !== 'string')
12
+ return null;
13
+ const normalized = value.replace(/[\u0000-\u001f\u007f]+/gu, ' ').replace(/\s+/gu, ' ').trim().slice(0, 80);
14
+ return normalized || null;
15
+ }
16
+ function observationContext(document) {
17
+ const context = object(document.collectionContext);
18
+ return {
19
+ generatedAt: document.generatedAt,
20
+ mode: document.mode,
21
+ observerLabel: label(context.observerLabel),
22
+ vantageLabel: label(context.vantageLabel),
23
+ };
24
+ }
25
+ function comparable(value, sourceState) {
26
+ if (['error', 'not_recorded', 'partial', 'skipped', 'unavailable', 'unsupported'].includes(sourceState))
27
+ return false;
28
+ return value !== 'Not observed' && value !== 'Not recorded in compact Bulk evidence';
29
+ }
30
+ function normalizedValue(value) {
31
+ return value.replace(/\s+/gu, ' ').trim().toLocaleLowerCase('en-US');
32
+ }
33
+ function buildCliLookupReconciliation(inputs, generatedAt = new Date().toISOString()) {
34
+ if (inputs.length < 2 || inputs.length > 5)
35
+ throw new CliUsageError('Lookup reconciliation requires from 2 to 5 observations.');
36
+ const documents = inputs.map((input, index) => parseSavedLookupDocument(input, {
37
+ label: `Lookup reconciliation observation ${index + 1}`,
38
+ }));
39
+ const domain = documents[0]?.registrableDomain;
40
+ if (!domain || documents.some((document) => document.registrableDomain !== domain)) {
41
+ throw new CliUsageError('Lookup reconciliation requires observations for one domain.');
42
+ }
43
+ const comparisons = documents.slice(1).map((document, index) => buildCliLookupDiff(inputs[0], inputs[index + 1], generatedAt, { domainMode: 'same' }));
44
+ const firstRows = comparisons[0]?.comparison.rows || [];
45
+ const fields = firstRows.map((firstRow) => {
46
+ const values = [{
47
+ observation: 0,
48
+ value: firstRow.left,
49
+ sourceState: firstRow.leftSourceState,
50
+ comparable: comparable(firstRow.left, firstRow.leftSourceState),
51
+ }];
52
+ for (let index = 0; index < comparisons.length; index++) {
53
+ const row = comparisons[index].comparison.rows.find((candidate) => candidate.id === firstRow.id);
54
+ if (!row)
55
+ continue;
56
+ values.push({
57
+ observation: index + 1,
58
+ value: row.right,
59
+ sourceState: row.rightSourceState,
60
+ comparable: comparable(row.right, row.rightSourceState),
61
+ });
62
+ }
63
+ const comparableValues = values.filter((value) => value.comparable);
64
+ const distinctValues = new Set(comparableValues.map((value) => normalizedValue(value.value)));
65
+ const state = comparableValues.length < 2
66
+ ? 'non_comparable'
67
+ : distinctValues.size === 1
68
+ ? 'agreement'
69
+ : 'disagreement';
70
+ return {
71
+ id: firstRow.id,
72
+ category: firstRow.category,
73
+ label: firstRow.label,
74
+ source: firstRow.source,
75
+ state,
76
+ method: firstRow.method,
77
+ values,
78
+ limitations: firstRow.limitations,
79
+ };
80
+ });
81
+ const observations = documents.map(observationContext);
82
+ const identities = observations.map((observation) => (observation.observerLabel && observation.vantageLabel
83
+ ? `${observation.observerLabel}\u0000${observation.vantageLabel}`
84
+ : null));
85
+ const labelsComplete = identities.every((identity) => identity !== null);
86
+ const labelsDistinct = labelsComplete && new Set(identities).size === identities.length;
87
+ const independence = labelsDistinct
88
+ ? {
89
+ state: 'verified_distinct_labels',
90
+ reason: 'Every observation carries a different analyst-supplied observer and vantage label.',
91
+ }
92
+ : {
93
+ state: 'unverified',
94
+ reason: 'Observer and vantage labels are missing or repeated; independent collection cannot be inferred.',
95
+ };
96
+ return {
97
+ schema: CLI_LOOKUP_RECONCILIATION_SCHEMA,
98
+ version: CLI_LOOKUP_RECONCILIATION_VERSION,
99
+ generatedAt,
100
+ domain,
101
+ independence,
102
+ observations,
103
+ fields,
104
+ summary: {
105
+ observationCount: observations.length,
106
+ agreement: fields.filter((field) => field.state === 'agreement').length,
107
+ disagreement: fields.filter((field) => field.state === 'disagreement').length,
108
+ nonComparable: fields.filter((field) => field.state === 'non_comparable').length,
109
+ },
110
+ privacy: { filenamesRetained: 0, rawRegistryPayloadsCopied: 0 },
111
+ limitations: [
112
+ 'This report reconciles only bounded normalised values already present in saved Lookup documents and makes no network request.',
113
+ 'Observer and vantage labels are analyst-supplied metadata. Distinct labels do not prove independent resolvers, networks, providers, caches, or collection paths.',
114
+ 'Agreement can reflect shared upstream data or collection conditions. Disagreement can reflect collection timing, propagation, source health, or a real domain change.',
115
+ 'Non-comparable evidence remains separate from agreement and is never interpreted as absence, safety, or a negative finding.',
116
+ ],
117
+ };
118
+ }
119
+ function formatCliLookupReconciliation(document) {
120
+ const output = [
121
+ 'Lookup observation reconciliation',
122
+ `Domain ${document.domain}`,
123
+ `Observations ${document.summary.observationCount}`,
124
+ `Independence ${document.independence.state.replaceAll('_', ' ')}`,
125
+ `Agreement ${document.summary.agreement}`,
126
+ `Disagreement ${document.summary.disagreement}`,
127
+ `Non-comparable ${document.summary.nonComparable}`,
128
+ '',
129
+ ];
130
+ for (const field of document.fields.filter((candidate) => candidate.state !== 'agreement')) {
131
+ output.push(`${field.label} [${field.state.replaceAll('_', ' ')}]`);
132
+ for (const value of field.values) {
133
+ output.push(` Observation ${value.observation + 1}: ${value.value} (${value.sourceState.replaceAll('_', ' ')})`);
134
+ }
135
+ }
136
+ output.push('', 'Limitations:');
137
+ for (const limitation of document.limitations)
138
+ output.push(` - ${limitation}`);
139
+ return `${output.join('\n')}\n`;
140
+ }
141
+ export { CLI_LOOKUP_RECONCILIATION_SCHEMA, CLI_LOOKUP_RECONCILIATION_VERSION, MAX_LOOKUP_RECONCILIATION_INPUT_BYTES, buildCliLookupReconciliation, formatCliLookupReconciliation, };