@slicedearth/whoisleuth-cli 1.39.1 → 1.45.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +23 -4
  2. package/bin/whoisleuth.mjs +31 -9
  3. package/cli/archive-inspect.mjs +37 -2
  4. package/cli/arguments.mjs +361 -256
  5. package/cli/artifact-verify.mjs +35 -1
  6. package/cli/bulk-checkpoint.mjs +72 -39
  7. package/cli/bulk-command-runner.mjs +34 -4
  8. package/cli/bulk.mjs +18 -7
  9. package/cli/case-pack.mjs +136 -0
  10. package/cli/ci-report.mjs +73 -0
  11. package/cli/collection-preflight.mjs +59 -0
  12. package/cli/command-reference.mjs +395 -0
  13. package/cli/completion.mjs +67 -13
  14. package/cli/config-profile.mjs +183 -0
  15. package/cli/ct-event-intake.mjs +159 -0
  16. package/cli/discovery-observation-snapshot.mjs +8 -14
  17. package/cli/discovery-scan-command-runner.mjs +29 -2
  18. package/cli/discovery-snapshot.mjs +8 -15
  19. package/cli/domain-control-monitor.mjs +104 -0
  20. package/cli/domain-control-observations.mjs +235 -0
  21. package/cli/errors.mjs +7 -1
  22. package/cli/evidence-command-arguments.mjs +11 -0
  23. package/cli/evidence-command-runner.mjs +1 -0
  24. package/cli/external-observation-mapping.mjs +162 -0
  25. package/cli/fail-policy.mjs +64 -0
  26. package/cli/formatters/html.mjs +1 -1
  27. package/cli/formatters/json.mjs +12 -4
  28. package/cli/formatters/terminal.mjs +1 -1
  29. package/cli/http.mjs +2 -1
  30. package/cli/investigation-manifest.mjs +113 -0
  31. package/cli/investigation-run.mjs +112 -0
  32. package/cli/lookup-brief.mjs +116 -0
  33. package/cli/lookup-command-runner.mjs +8 -1
  34. package/cli/lookup-diff.mjs +17 -19
  35. package/cli/lookup-reconcile.mjs +1 -1
  36. package/cli/mail-review.mjs +5 -5
  37. package/cli/manual.mjs +1 -1
  38. package/cli/network-command-runner.mjs +4 -1
  39. package/cli/offline-evidence-review.mjs +105 -8
  40. package/cli/open-asset-model-bridge.mjs +169 -0
  41. package/cli/registry-cohort.mjs +121 -0
  42. package/cli/registry-doctor.mjs +106 -11
  43. package/cli/registry-fixture-scaffold.mjs +74 -0
  44. package/cli/runner.mjs +388 -315
  45. package/docs/cli-reference.md +266 -21
  46. package/docs/cli.md +43 -7
  47. package/frontend/src/lib/analysis/acquisition-due-diligence.js +2 -2
  48. package/frontend/src/lib/analysis/brand-profile-model.js +50 -6
  49. package/frontend/src/lib/analysis/case-evidence-model.js +1 -1
  50. package/frontend/src/lib/analysis/case-record-core.js +1 -1
  51. package/frontend/src/lib/analysis/case-record-operations.js +1 -1
  52. package/frontend/src/lib/analysis/case-report.js +573 -0
  53. package/frontend/src/lib/analysis/case-response-model.js +30 -1
  54. package/frontend/src/lib/analysis/detection-rule-model.js +36 -0
  55. package/frontend/src/lib/analysis/external-findings-import.js +320 -0
  56. package/frontend/src/lib/analysis/lookup-asset-graph.js +8 -8
  57. package/frontend/src/lib/analysis/lookup-decision-support.js +9 -2
  58. package/frontend/src/lib/analysis/lookup-presentation.js +14 -30
  59. package/frontend/src/lib/analysis/lookup-source-refresh.js +36 -63
  60. package/frontend/src/lib/analysis/scoring.js +1 -1
  61. package/frontend/src/lib/analysis/utils.js +1 -1
  62. package/frontend/src/lib/analysis/visualization-models.js +2 -3
  63. package/frontend/src/lib/analysis/workspace-archive-crypto.js +1 -4
  64. package/frontend/src/lib/analysis/workspace-archive.js +5 -7
  65. package/lib/abusech-auth.mjs +1 -1
  66. package/lib/availability.mjs +1 -1
  67. package/lib/bounded-contract-normalizers.mjs +33 -0
  68. package/lib/bounded-file.mjs +69 -0
  69. package/lib/bounded-json-response.mjs +157 -0
  70. package/lib/browser-library-profile.mjs +167 -38
  71. package/lib/dns-convergence-review.mjs +183 -0
  72. package/lib/dns-delegation-health.mjs +196 -3
  73. package/lib/domain-change-packet.mjs +133 -0
  74. package/lib/domain-change-review.mjs +307 -0
  75. package/lib/domain-control-flight-recorder.mjs +268 -0
  76. package/lib/domain-portfolio-review.mjs +151 -0
  77. package/lib/evidence-export.mjs +1 -1
  78. package/lib/html-signals.mjs +5 -11
  79. package/lib/lookup-cache.mjs +5 -1
  80. package/lib/nameserver-preflight-review.mjs +159 -0
  81. package/lib/page-language-signals.mjs +124 -0
  82. package/lib/rdap-transport.mjs +6 -6
  83. package/lib/risk-scoring.mjs +42 -1
  84. package/lib/safe-fetch.mjs +37 -32
  85. package/lib/static-html-analysis.mjs +29 -1
  86. package/lib/threat-intelligence-runtime.mjs +1 -2
  87. package/lib/trust-store-comparison.mjs +146 -0
  88. package/lib/website-technology.mjs +22 -12
  89. package/lib/zone-intent-review.mjs +534 -0
  90. package/package.json +1 -1
@@ -9,7 +9,10 @@ import { WORKSPACE_ARCHIVE_SCHEMA, readWorkspaceArchive, } from "../frontend/src
9
9
  import { sha256ArtifactDigest } from "../frontend/src/lib/analysis/artifact-integrity.js";
10
10
  import { SAVED_LOOKUP_SCHEMA, SAVED_LOOKUP_SCHEMA_VERSION, parseSavedLookupDocument, } from "./saved-lookup.mjs";
11
11
  import { DOMAIN_CONTROL_MANIFEST_SCHEMA, DOMAIN_CONTROL_MANIFEST_VERSION, verifyDomainControlManifest, } from "../lib/domain-control-manifest.mjs";
12
+ import { DOMAIN_CHANGE_PACKET_SCHEMA, DOMAIN_CHANGE_PACKET_VERSION, } from "../lib/domain-change-packet.mjs";
12
13
  import { INVESTIGATION_CAPSULE_SCHEMA, INVESTIGATION_CAPSULE_VERSION, verifyInvestigationCapsule, } from "../frontend/src/lib/analysis/investigation-capsule.js";
14
+ import { INVESTIGATION_MANIFEST_SCHEMA, INVESTIGATION_MANIFEST_VERSION, } from "./investigation-manifest.mjs";
15
+ import { CLI_CASE_PACK_SCHEMA, CLI_CASE_PACK_VERSION, verifyCliCasePack, } from "./case-pack.mjs";
13
16
  export const OFFLINE_ARTIFACT_VERIFICATION_SCHEMA = 'whoisleuth.offline-artifact-verification';
14
17
  export const OFFLINE_ARTIFACT_VERIFICATION_VERSION = 1;
15
18
  export const MAX_OFFLINE_ARTIFACT_BYTES = 15 * 1024 * 1024;
@@ -27,6 +30,8 @@ const SIGNED_ARTIFACT_VERSIONS = Object.freeze({
27
30
  [BULK_MAIL_EXPOSURE_SCHEMA]: new Set([BULK_MAIL_EXPOSURE_VERSION]),
28
31
  [BULK_REVIEW_MANIFEST_SCHEMA]: new Set([BULK_REVIEW_MANIFEST_VERSION]),
29
32
  [DOMAIN_CONTROL_MANIFEST_SCHEMA]: new Set([DOMAIN_CONTROL_MANIFEST_VERSION]),
33
+ [DOMAIN_CHANGE_PACKET_SCHEMA]: new Set([DOMAIN_CHANGE_PACKET_VERSION]),
34
+ [INVESTIGATION_MANIFEST_SCHEMA]: new Set([INVESTIGATION_MANIFEST_VERSION]),
30
35
  });
31
36
  function record(value) {
32
37
  return value && typeof value === 'object' && !Array.isArray(value)
@@ -72,7 +77,7 @@ function archiveReport(raw, archive, encrypted, ciphertextBytes) {
72
77
  artifact: Object.freeze({
73
78
  kind: encrypted ? 'encrypted_workspace_archive' : 'workspace_archive',
74
79
  schema: WORKSPACE_ARCHIVE_SCHEMA,
75
- version: archive.version,
80
+ version: archive.sourceVersion,
76
81
  }),
77
82
  state: 'verified',
78
83
  valid: true,
@@ -134,6 +139,35 @@ async function verifySignedArtifact(raw, value, schema, version) {
134
139
  }
135
140
  export async function verifyOfflineArtifact(raw, options = {}) {
136
141
  const value = parseJson(raw);
142
+ const casePackPacket = record(value.packet);
143
+ if (casePackPacket?.schema === CLI_CASE_PACK_SCHEMA) {
144
+ const verified = verifyCliCasePack(value);
145
+ return Object.freeze({
146
+ schema: OFFLINE_ARTIFACT_VERIFICATION_SCHEMA,
147
+ version: OFFLINE_ARTIFACT_VERIFICATION_VERSION,
148
+ artifact: Object.freeze({
149
+ kind: 'cli_case_pack',
150
+ schema: CLI_CASE_PACK_SCHEMA,
151
+ version: CLI_CASE_PACK_VERSION,
152
+ }),
153
+ state: 'verified',
154
+ valid: true,
155
+ checks: Object.freeze({
156
+ structure: 'verified',
157
+ contentIntegrity: 'verified',
158
+ authenticatedEncryption: 'not_applicable',
159
+ }),
160
+ summary: Object.freeze({
161
+ inputBytes: inputBytes(raw),
162
+ sectionCount: null,
163
+ recordCount: verified.caseCount,
164
+ ciphertextBytes: null,
165
+ }),
166
+ limitations: Object.freeze([
167
+ 'Case-pack verification detects changes after export and checks the bounded case collection; it does not authenticate the analyst, review decision, source observations, or recipient authorisation.',
168
+ ]),
169
+ });
170
+ }
137
171
  if (isEncryptedWorkspaceArchive(value)) {
138
172
  const inspected = inspectEncryptedWorkspaceArchive(value);
139
173
  if (!options.passphrase) {
@@ -1,20 +1,18 @@
1
1
  import { createHash } from 'node:crypto';
2
2
  import { open } from 'node:fs/promises';
3
- import { CliUsageError } from "./errors.mjs";
3
+ import { recordOrNull } from "../lib/bounded-contract-normalizers.mjs";
4
+ import { readBoundedRegularTextFile } from "../lib/bounded-file.mjs";
5
+ import { boundedCliInputError, CliUsageError } from "./errors.mjs";
4
6
  import { writePrivateFile } from "./output-file.mjs";
5
7
  export const CLI_BULK_CHECKPOINT_SCHEMA = 'whoisleuth.cli.bulk-checkpoint';
6
- export const CLI_BULK_CHECKPOINT_VERSION = 1;
8
+ export const CLI_BULK_CHECKPOINT_VERSION = 2;
9
+ const SUPPORTED_BULK_CHECKPOINT_VERSIONS = new Set([1, CLI_BULK_CHECKPOINT_VERSION]);
7
10
  export const MAX_BULK_CHECKPOINT_BYTES = 16 * 1024 * 1024;
8
11
  const MAX_CHECKPOINT_JSON_DEPTH = 12;
9
12
  const MAX_CHECKPOINT_OBJECT_KEYS = 256;
10
13
  const MAX_CHECKPOINT_ARRAY_ITEMS = 1_024;
11
14
  const MAX_CHECKPOINT_STRING_LENGTH = 32_768;
12
15
  const COMPACT_RESULT_KEYS = new Set(['availability', 'diagnostics']);
13
- function record(value) {
14
- return value && typeof value === 'object' && !Array.isArray(value)
15
- ? value
16
- : null;
17
- }
18
16
  function checkpointDigest(queries, deep) {
19
17
  return createHash('sha256')
20
18
  .update(deep ? 'deep\n' : 'fast\n')
@@ -37,7 +35,7 @@ function isBoundedCheckpointJson(value, depth = 0) {
37
35
  return value.length <= MAX_CHECKPOINT_ARRAY_ITEMS
38
36
  && value.every((item) => isBoundedCheckpointJson(item, depth + 1));
39
37
  }
40
- const object = record(value);
38
+ const object = recordOrNull(value);
41
39
  if (!object)
42
40
  return false;
43
41
  const entries = Object.entries(object);
@@ -45,37 +43,44 @@ function isBoundedCheckpointJson(value, depth = 0) {
45
43
  && entries.every(([key, item]) => key.length <= 128 && isBoundedCheckpointJson(item, depth + 1));
46
44
  }
47
45
  function normalizeCompactResult(value) {
48
- const result = record(value);
46
+ const result = recordOrNull(value);
49
47
  if (!result || !isBoundedCheckpointJson(result))
50
48
  return null;
51
49
  const keys = Object.keys(result);
52
50
  if (keys.length !== COMPACT_RESULT_KEYS.size || keys.some((key) => !COMPACT_RESULT_KEYS.has(key)))
53
51
  return null;
54
- if (!record(result.availability) || !record(result.diagnostics))
52
+ if (!recordOrNull(result.availability) || !recordOrNull(result.diagnostics))
55
53
  return null;
56
54
  return result;
57
55
  }
58
- function normalizeCheckpointResult(value, queries, classifyQuery) {
59
- const item = record(value);
56
+ function normalizeCheckpointResult(value, queries, classifyQuery, version = CLI_BULK_CHECKPOINT_VERSION) {
57
+ const item = recordOrNull(value);
60
58
  const index = item?.index;
61
59
  if (!Number.isSafeInteger(index) || Number(index) < 0 || Number(index) >= queries.length)
62
60
  return null;
63
61
  const query = queries[Number(index)];
64
62
  if (typeof item?.query !== 'string' || item.query !== query)
65
63
  return null;
64
+ const observedAt = version >= 2
65
+ ? item.observedAt === null
66
+ ? null
67
+ : validTimestamp(item.observedAt) ? new Date(item.observedAt).toISOString() : undefined
68
+ : null;
69
+ if (observedAt === undefined)
70
+ return null;
66
71
  if (item.ok === false) {
67
72
  if (typeof item.error !== 'string' || !item.error || item.error.length > 300)
68
73
  return null;
69
- return { index: Number(index), query, ok: false, error: item.error };
74
+ return { index: Number(index), query, ok: false, error: item.error, observedAt };
70
75
  }
71
76
  const result = normalizeCompactResult(item.result);
72
77
  if (item.ok !== true || !result)
73
78
  return null;
74
79
  const classified = classifyQuery(query);
75
- const storedClassified = record(item.classified);
80
+ const storedClassified = recordOrNull(item.classified);
76
81
  if (storedClassified?.type !== classified.type || storedClassified.value !== classified.value)
77
82
  return null;
78
- return { index: Number(index), query, ok: true, classified, result };
83
+ return { index: Number(index), query, ok: true, classified, result, observedAt };
79
84
  }
80
85
  function parseBulkCheckpoint(text, options) {
81
86
  if (typeof text !== 'string' || Buffer.byteLength(text, 'utf8') > MAX_BULK_CHECKPOINT_BYTES) {
@@ -88,10 +93,13 @@ function parseBulkCheckpoint(text, options) {
88
93
  catch {
89
94
  throw new CliUsageError('Bulk checkpoint must be valid JSON.');
90
95
  }
91
- const document = record(parsed);
96
+ const document = recordOrNull(parsed);
92
97
  const expectedDigest = checkpointDigest(options.queries, options.deep);
93
- if (document?.schema !== CLI_BULK_CHECKPOINT_SCHEMA || document.version !== CLI_BULK_CHECKPOINT_VERSION) {
94
- throw new CliUsageError(`Bulk checkpoint must use ${CLI_BULK_CHECKPOINT_SCHEMA} version ${CLI_BULK_CHECKPOINT_VERSION}.`);
98
+ const version = typeof document?.version === 'number' && Number.isSafeInteger(document.version)
99
+ ? document.version
100
+ : 0;
101
+ if (document?.schema !== CLI_BULK_CHECKPOINT_SCHEMA || !SUPPORTED_BULK_CHECKPOINT_VERSIONS.has(version)) {
102
+ throw new CliUsageError(`Bulk checkpoint must use ${CLI_BULK_CHECKPOINT_SCHEMA} version 1 or ${CLI_BULK_CHECKPOINT_VERSION}.`);
95
103
  }
96
104
  if (document.mode !== (options.deep ? 'deep' : 'fast')
97
105
  || document.inputDigestSha256 !== expectedDigest
@@ -104,11 +112,11 @@ function parseBulkCheckpoint(text, options) {
104
112
  const results = [];
105
113
  const seen = new Set();
106
114
  for (const candidate of document.results.slice(0, options.queries.length + 1)) {
107
- const result = normalizeCheckpointResult(candidate, options.queries, options.classifyQuery);
115
+ const result = normalizeCheckpointResult(candidate, options.queries, options.classifyQuery, version);
108
116
  if (!result || seen.has(result.index))
109
117
  throw new CliUsageError('Bulk checkpoint contains an invalid or duplicate result.');
110
118
  seen.add(result.index);
111
- results.push(result);
119
+ results.push({ ...result, collectionOrigin: 'resumed_checkpoint' });
112
120
  }
113
121
  return {
114
122
  schema: CLI_BULK_CHECKPOINT_SCHEMA,
@@ -122,16 +130,15 @@ function parseBulkCheckpoint(text, options) {
122
130
  };
123
131
  }
124
132
  async function readCheckpointFile(path) {
125
- const handle = await open(path, 'r');
126
133
  try {
127
- const stats = await handle.stat();
128
- if (!stats.isFile() || stats.size > MAX_BULK_CHECKPOINT_BYTES) {
129
- throw new CliUsageError(`Bulk checkpoint input is limited to ${MAX_BULK_CHECKPOINT_BYTES} bytes.`);
130
- }
131
- return await handle.readFile({ encoding: 'utf8' });
134
+ return await readBoundedRegularTextFile(path, {
135
+ maximumBytes: MAX_BULK_CHECKPOINT_BYTES,
136
+ label: 'Bulk checkpoint input',
137
+ allowSymbolicLink: true,
138
+ });
132
139
  }
133
- finally {
134
- await handle.close();
140
+ catch (error) {
141
+ throw boundedCliInputError(error, 'Bulk checkpoint input');
135
142
  }
136
143
  }
137
144
  async function checkpointFileExists(path) {
@@ -144,11 +151,12 @@ async function checkpointFileExists(path) {
144
151
  const code = error && typeof error === 'object' && 'code' in error ? error.code : null;
145
152
  if (code === 'ENOENT')
146
153
  return false;
147
- throw error;
154
+ throw boundedCliInputError(error, 'Bulk checkpoint input');
148
155
  }
149
156
  }
150
157
  async function createBulkCheckpointWriter(options) {
151
158
  const now = options.now || (() => new Date().toISOString());
159
+ const writeCheckpoint = options.writeFile ?? writePrivateFile;
152
160
  const exists = await checkpointFileExists(options.path);
153
161
  if (options.resume && !exists)
154
162
  throw new CliUsageError('Bulk checkpoint does not exist; remove --resume to start a new checkpoint.');
@@ -159,7 +167,8 @@ async function createBulkCheckpointWriter(options) {
159
167
  : null;
160
168
  const startedAt = resumed?.startedAt || now();
161
169
  const results = new Map((resumed?.results || []).map((item) => [item.index, item]));
162
- let chain = Promise.resolve();
170
+ let activeWrite = null;
171
+ let dirty = false;
163
172
  let writeFailure = null;
164
173
  let created = resumed !== null;
165
174
  function document() {
@@ -172,19 +181,20 @@ async function createBulkCheckpointWriter(options) {
172
181
  queryCount: options.queries.length,
173
182
  startedAt,
174
183
  updatedAt,
175
- results: [...results.values()].sort((left, right) => left.index - right.index),
184
+ results: [...results.values()]
185
+ .sort((left, right) => left.index - right.index)
186
+ .map(({ collectionOrigin: _collectionOrigin, ...result }) => result),
176
187
  };
177
188
  }
178
- function scheduleWrite() {
179
- chain = chain.then(async () => {
180
- if (writeFailure)
181
- return;
189
+ async function drainWrites() {
190
+ while (dirty && !writeFailure) {
191
+ dirty = false;
182
192
  try {
183
193
  const content = `${JSON.stringify(document(), null, 2)}\n`;
184
194
  if (Buffer.byteLength(content, 'utf8') > MAX_BULK_CHECKPOINT_BYTES) {
185
195
  throw new CliUsageError(`Bulk checkpoints are limited to ${MAX_BULK_CHECKPOINT_BYTES} bytes.`);
186
196
  }
187
- await writePrivateFile(options.path, content, {
197
+ await writeCheckpoint(options.path, content, {
188
198
  force: created,
189
199
  existingFileMessage: 'Bulk checkpoint already exists; use --resume or choose another path.',
190
200
  });
@@ -193,6 +203,23 @@ async function createBulkCheckpointWriter(options) {
193
203
  catch (error) {
194
204
  writeFailure = error;
195
205
  }
206
+ }
207
+ }
208
+ function scheduleWrite() {
209
+ dirty = true;
210
+ if (activeWrite || writeFailure)
211
+ return;
212
+ const pending = drainWrites();
213
+ activeWrite = pending;
214
+ void pending.then(() => {
215
+ if (activeWrite === pending)
216
+ activeWrite = null;
217
+ if (dirty && !writeFailure)
218
+ scheduleWrite();
219
+ }, (error) => {
220
+ writeFailure ||= error;
221
+ if (activeWrite === pending)
222
+ activeWrite = null;
196
223
  });
197
224
  }
198
225
  if (!resumed)
@@ -200,16 +227,22 @@ async function createBulkCheckpointWriter(options) {
200
227
  return Object.freeze({
201
228
  initialResults: Object.freeze([...results.values()].sort((left, right) => left.index - right.index)),
202
229
  record(result) {
203
- const normalized = normalizeCheckpointResult(result, options.queries, options.classifyQuery);
230
+ const observedAt = validTimestamp(result.observedAt) ? new Date(result.observedAt).toISOString() : now();
231
+ const normalized = normalizeCheckpointResult({ ...result, observedAt }, options.queries, options.classifyQuery);
204
232
  if (!normalized) {
205
233
  writeFailure ||= new CliUsageError('Bulk checkpoint result is not a bounded compact lookup result.');
206
234
  return;
207
235
  }
208
- results.set(normalized.index, normalized);
236
+ results.set(normalized.index, { ...normalized, collectionOrigin: 'current_run' });
209
237
  scheduleWrite();
210
238
  },
211
239
  async flush() {
212
- await chain;
240
+ while (activeWrite || dirty) {
241
+ if (!activeWrite)
242
+ scheduleWrite();
243
+ if (activeWrite)
244
+ await activeWrite;
245
+ }
213
246
  if (writeFailure)
214
247
  throw writeFailure;
215
248
  },
@@ -8,6 +8,9 @@ import { boundedCliErrorMessage, CliUsageError } from "./errors.mjs";
8
8
  import EXIT_CODES from "./exit-codes.mjs";
9
9
  import { buildCliBulkDocument, formatJsonDocument, formatJsonLines } from "./formatters/json.mjs";
10
10
  import { formatTerminalBulk } from "./formatters/terminal.mjs";
11
+ import { buildCollectionPreflight, formatCollectionPreflight } from "./collection-preflight.mjs";
12
+ import { evaluateCliFailPolicies, formatFailPolicyNotice } from "./fail-policy.mjs";
13
+ import { formatCliJunit } from "./ci-report.mjs";
11
14
  import { createCliProgressEvents } from "./progress-events.mjs";
12
15
  async function runBulkCommand(args, dependencies, context) {
13
16
  const eventProgress = createCliProgressEvents(context.stderr, {
@@ -23,7 +26,7 @@ async function runBulkCommand(args, dependencies, context) {
23
26
  ? await dependencies.readBulkInput(args.source)
24
27
  : await readTextStreamBounded(args.source
25
28
  ? createReadStream(args.source, { highWaterMark: 64 * 1024 })
26
- : dependencies.stdin || process.stdin, MAX_BULK_INPUT_BYTES);
29
+ : dependencies.stdin || process.stdin, MAX_BULK_INPUT_BYTES, dependencies.signal);
27
30
  }
28
31
  catch (error) {
29
32
  if (error instanceof CliUsageError)
@@ -31,6 +34,14 @@ async function runBulkCommand(args, dependencies, context) {
31
34
  throw new CliUsageError(`Could not read bulk input: ${boundedCliErrorMessage(error, 'Input could not be read')}`);
32
35
  }
33
36
  const parsed = parseBulkQueries(input, { deep: args.deep });
37
+ if (args.plan) {
38
+ const document = buildCollectionPreflight({
39
+ command: 'bulk', targetCount: parsed.queries.length, targetLimit: args.deep ? 50 : 500,
40
+ deep: args.deep, concurrency: args.concurrency, output: args.output, checkpoint: false,
41
+ });
42
+ context.writeStdout(args.output === 'json' ? formatJsonDocument(document) : context.terminal(formatCollectionPreflight(document), args.color));
43
+ return EXIT_CODES.SUCCESS;
44
+ }
34
45
  const classify = dependencies.classifyQuery || classifyQuery;
35
46
  const checkpointWriter = dependencies.createBulkCheckpointWriter || createBulkCheckpointWriter;
36
47
  const checkpoint = args.checkpoint
@@ -45,6 +56,8 @@ async function runBulkCommand(args, dependencies, context) {
45
56
  : null;
46
57
  const indicator = context.beginProgress(`Collecting 0 of ${parsed.queries.length} targets`);
47
58
  let completed = checkpoint?.initialResults.length || 0;
59
+ const resumedItems = new Map((checkpoint?.initialResults ?? []).map((item) => [item.index, item]));
60
+ const settledItems = new Map();
48
61
  if (completed)
49
62
  indicator.update(`Resumed ${completed} of ${parsed.queries.length} targets`);
50
63
  let items;
@@ -58,12 +71,23 @@ async function runBulkCommand(args, dependencies, context) {
58
71
  ...(checkpoint ? { initialResults: checkpoint.initialResults } : {}),
59
72
  ...(dependencies.signal ? { signal: dependencies.signal } : {}),
60
73
  onItemSettled: (item) => {
74
+ const observed = {
75
+ ...item,
76
+ observedAt: context.now(),
77
+ collectionOrigin: 'current_run',
78
+ };
79
+ settledItems.set(item.index, observed);
61
80
  completed += 1;
62
81
  indicator.update(`Collected ${completed} of ${parsed.queries.length} targets`);
63
82
  eventProgress.emit({ event: 'item_settled', index: item.index, ok: item.ok });
64
- checkpoint?.record(item);
83
+ checkpoint?.record(observed);
65
84
  },
66
85
  });
86
+ items = items.map((item) => resumedItems.get(item.index) ?? settledItems.get(item.index) ?? {
87
+ ...item,
88
+ observedAt: null,
89
+ collectionOrigin: 'current_run',
90
+ });
67
91
  }
68
92
  finally {
69
93
  context.endProgress();
@@ -82,9 +106,12 @@ async function runBulkCommand(args, dependencies, context) {
82
106
  filter: args.filter,
83
107
  };
84
108
  const selectedItems = selectBulkItems(items, args.filter);
109
+ const selectedDocument = buildCliBulkDocument(selectedItems, metadata);
85
110
  if (!args.quiet) {
86
111
  if (args.output === 'json')
87
- context.writeStdout(formatJsonDocument(buildCliBulkDocument(selectedItems, metadata)));
112
+ context.writeStdout(formatJsonDocument(selectedDocument));
113
+ else if (args.output === 'junit')
114
+ context.writeStdout(formatCliJunit(selectedDocument));
88
115
  else if (args.output === 'jsonl')
89
116
  context.writeStdout(formatJsonLines(selectedItems, metadata));
90
117
  else if (args.output === 'csv')
@@ -102,7 +129,10 @@ async function runBulkCommand(args, dependencies, context) {
102
129
  context.writeStderr(`Checkpoint warning: ${boundedCliErrorMessage(checkpointFailure, 'Checkpoint could not be written')}. Completed output is still available.\n`);
103
130
  }
104
131
  }
105
- const exitCode = checkpointFailure || items.some((item) => !item.ok)
132
+ const policyFindings = evaluateCliFailPolicies(buildCliBulkDocument(items, { ...metadata, filter: 'all' }), args.failOn || []);
133
+ if (policyFindings.length && !eventProgress.enabled)
134
+ context.writeStderr(formatFailPolicyNotice(policyFindings));
135
+ const exitCode = checkpointFailure || items.some((item) => !item.ok) || policyFindings.length
106
136
  ? EXIT_CODES.PARTIAL_FAILURE
107
137
  : EXIT_CODES.SUCCESS;
108
138
  eventProgress.emit({ event: 'completed', exitCode });
package/cli/bulk.mjs CHANGED
@@ -3,17 +3,28 @@ import { abortable } from "../lib/abort.mjs";
3
3
  import { CliUsageError } from "./arguments.mjs";
4
4
  import { MAX_DEEP_BULK_QUERIES, MAX_FAST_BULK_QUERIES, } from "../lib/bulk-limits.mjs";
5
5
  const MAX_BULK_INPUT_BYTES = 1024 * 1024;
6
- async function readTextStreamBounded(stream, limit = MAX_BULK_INPUT_BYTES) {
6
+ async function readTextStreamBounded(stream, limit = MAX_BULK_INPUT_BYTES, signal) {
7
7
  if (!stream || stream.isTTY)
8
8
  return '';
9
+ const abort = () => stream.destroy?.(new DOMException('Aborted', 'AbortError'));
10
+ if (signal?.aborted)
11
+ throw new DOMException('Aborted', 'AbortError');
12
+ signal?.addEventListener('abort', abort, { once: true });
9
13
  const chunks = [];
10
14
  let total = 0;
11
- for await (const chunk of stream) {
12
- const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
13
- total += buffer.length;
14
- if (total > limit)
15
- throw new CliUsageError(`Bulk input is limited to ${limit} bytes.`);
16
- chunks.push(buffer);
15
+ try {
16
+ for await (const chunk of stream) {
17
+ if (signal?.aborted)
18
+ throw new DOMException('Aborted', 'AbortError');
19
+ const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
20
+ total += buffer.length;
21
+ if (total > limit)
22
+ throw new CliUsageError(`Bulk input is limited to ${limit} bytes.`);
23
+ chunks.push(buffer);
24
+ }
25
+ }
26
+ finally {
27
+ signal?.removeEventListener('abort', abort);
17
28
  }
18
29
  return Buffer.concat(chunks).toString('utf8');
19
30
  }
@@ -0,0 +1,136 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { Buffer } from 'node:buffer';
3
+ import { canonicalArtifactJson } from "../frontend/src/lib/analysis/artifact-integrity.js";
4
+ import { buildCaseReport } from "../frontend/src/lib/analysis/case-report.js";
5
+ import { CASE_IMPORT_VERSIONS, MAX_CASE_IMPORT_BYTES, CASE_SCHEMA_VERSION, normalizeCaseStore, } from "../frontend/src/lib/analysis/case-model.js";
6
+ import { CliUsageError } from "./errors.mjs";
7
+ export const CLI_CASE_PACK_SCHEMA = 'whoisleuth.cli.case-pack';
8
+ export const CLI_CASE_PACK_VERSION = 1;
9
+ export const MAX_CASE_PACK_INPUT_BYTES = 4 * 1024 * 1024;
10
+ export const MAX_CASE_PACK_CASES = 25;
11
+ function redactedCase(record, audience) {
12
+ if (audience === 'internal')
13
+ return structuredClone(record);
14
+ return {
15
+ ...structuredClone(record),
16
+ notes: [],
17
+ actions: audience === 'public' ? [] : record.actions.map((item) => ({ ...item, recipient: '[redacted]' })),
18
+ manualTrail: record.manualTrail.map((item) => ({ ...item, target: null })),
19
+ assertions: audience === 'public' ? [] : structuredClone(record.assertions),
20
+ };
21
+ }
22
+ export function buildCliCasePack(input, options, generatedAt = new Date().toISOString()) {
23
+ if (!options.reviewed)
24
+ throw new CliUsageError('case-pack requires --reviewed after the audience-specific output has been checked.');
25
+ if (Buffer.byteLength(input, 'utf8') > MAX_CASE_PACK_INPUT_BYTES)
26
+ throw new CliUsageError('Case-pack input is limited to 4 MiB.');
27
+ let parsed;
28
+ try {
29
+ parsed = JSON.parse(input.replace(/^\uFEFF/u, ''));
30
+ }
31
+ catch {
32
+ throw new CliUsageError('Case-pack input must be valid JSON.');
33
+ }
34
+ const root = parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {};
35
+ if (typeof root.version !== 'number' || !CASE_IMPORT_VERSIONS.includes(root.version) || !Array.isArray(root.cases)) {
36
+ throw new CliUsageError(`Case-pack input must be a supported WHOISleuth case export through schema ${CASE_SCHEMA_VERSION}.`);
37
+ }
38
+ const normalised = normalizeCaseStore(root).cases;
39
+ if (!normalised.length)
40
+ throw new CliUsageError('Case-pack input did not contain a valid case.');
41
+ if (normalised.length !== root.cases.length) {
42
+ throw new CliUsageError('Case-pack input contains an invalid or duplicate case. Correct the browser export before packaging it.');
43
+ }
44
+ if (normalised.length > MAX_CASE_PACK_CASES) {
45
+ throw new CliUsageError(`Case packs are limited to ${MAX_CASE_PACK_CASES} reviewed cases. Export a smaller selected set so no case is silently omitted.`);
46
+ }
47
+ const cases = normalised.map((item) => redactedCase(item, options.audience));
48
+ const reports = cases.map((item) => buildCaseReport(item, { includeNotes: false, generatedAt }).json);
49
+ const exclusions = options.audience === 'internal'
50
+ ? ['Raw upstream payloads and credentials are outside the case schema.']
51
+ : options.audience === 'trusted'
52
+ ? ['Case notes', 'Recipient values', 'Manual trail targets', 'Raw upstream payloads and credentials']
53
+ : ['Case notes', 'Actions and recipient values', 'Analyst assertions', 'Manual trail targets', 'Raw upstream payloads and credentials'];
54
+ const packet = Object.freeze({
55
+ schema: CLI_CASE_PACK_SCHEMA,
56
+ version: CLI_CASE_PACK_VERSION,
57
+ audience: options.audience,
58
+ reviewed: true,
59
+ reports: Object.freeze(reports),
60
+ redactionManifest: Object.freeze({ excluded: Object.freeze(exclusions), sourceCaseCount: normalised.length }),
61
+ limitations: Object.freeze([
62
+ 'This local package is browser-importable through its top-level case collection and does not upload or submit evidence.',
63
+ 'The reviewed flag records a deliberate CLI choice; it does not prove recipient authorisation, factual correctness, or legal sufficiency.',
64
+ 'Importing the package does not restore fields excluded by its audience profile.',
65
+ ]),
66
+ });
67
+ const unsigned = { version: CASE_SCHEMA_VERSION, exportedAt: generatedAt, cases, packet };
68
+ const document = Object.freeze({
69
+ ...unsigned,
70
+ integrity: Object.freeze({
71
+ algorithm: 'SHA-256',
72
+ canonicalization: 'sorted-json-v1',
73
+ digestSha256: `sha256:${createHash('sha256').update(canonicalArtifactJson(unsigned)).digest('hex')}`,
74
+ }),
75
+ });
76
+ if (Buffer.byteLength(JSON.stringify(document), 'utf8') > MAX_CASE_IMPORT_BYTES) {
77
+ throw new CliUsageError('The generated case pack exceeds the browser 2 MiB import limit. Select fewer cases or a more restrictive audience so no evidence is silently omitted.');
78
+ }
79
+ return document;
80
+ }
81
+ export function verifyCliCasePack(input) {
82
+ const root = input && typeof input === 'object' && !Array.isArray(input)
83
+ ? input
84
+ : null;
85
+ const packet = root?.packet && typeof root.packet === 'object' && !Array.isArray(root.packet)
86
+ ? root.packet
87
+ : null;
88
+ const integrity = root?.integrity && typeof root.integrity === 'object' && !Array.isArray(root.integrity)
89
+ ? root.integrity
90
+ : null;
91
+ if (!root
92
+ || !packet
93
+ || packet.schema !== CLI_CASE_PACK_SCHEMA
94
+ || packet.version !== CLI_CASE_PACK_VERSION
95
+ || packet.reviewed !== true
96
+ || !['internal', 'public', 'trusted'].includes(String(packet.audience))
97
+ || !Array.isArray(packet.reports)
98
+ || !Array.isArray(root.cases)
99
+ || root.cases.length < 1
100
+ || root.cases.length > MAX_CASE_PACK_CASES
101
+ || packet.reports.length !== root.cases.length
102
+ || typeof root.version !== 'number'
103
+ || !CASE_IMPORT_VERSIONS.includes(root.version)
104
+ || !integrity
105
+ || integrity.algorithm !== 'SHA-256'
106
+ || integrity.canonicalization !== 'sorted-json-v1'
107
+ || typeof integrity.digestSha256 !== 'string'
108
+ || !/^sha256:[a-f0-9]{64}$/u.test(integrity.digestSha256)) {
109
+ throw new TypeError('The CLI case pack structure or integrity envelope is invalid.');
110
+ }
111
+ if (Buffer.byteLength(JSON.stringify(root), 'utf8') > MAX_CASE_IMPORT_BYTES) {
112
+ throw new TypeError('The CLI case pack exceeds the browser import limit.');
113
+ }
114
+ const { integrity: _integrity, ...unsigned } = root;
115
+ const calculated = `sha256:${createHash('sha256').update(canonicalArtifactJson(unsigned)).digest('hex')}`;
116
+ if (calculated !== integrity.digestSha256) {
117
+ throw new TypeError('The CLI case pack failed its SHA-256 integrity check.');
118
+ }
119
+ const normalised = normalizeCaseStore(root).cases;
120
+ if (normalised.length !== root.cases.length) {
121
+ throw new TypeError('The CLI case pack contains an invalid case collection.');
122
+ }
123
+ return Object.freeze({ caseCount: normalised.length });
124
+ }
125
+ export function formatCliCasePack(document) {
126
+ return [
127
+ 'Reviewed case pack',
128
+ `Audience ${document.packet.audience}`,
129
+ `Cases ${document.cases.length}`,
130
+ `Reports ${document.packet.reports.length}`,
131
+ `Digest ${document.integrity.digestSha256}`,
132
+ '',
133
+ ...document.packet.redactionManifest.excluded.map((item) => `Excluded: ${item}`),
134
+ '',
135
+ ].join('\n');
136
+ }
@@ -0,0 +1,73 @@
1
+ import { WHOISLEUTH_REQUEST_POLICY_URL } from "../lib/project-metadata.mjs";
2
+ function record(value) {
3
+ return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
4
+ }
5
+ function xml(value) {
6
+ return String(value).replace(/&/gu, '&amp;').replace(/</gu, '&lt;').replace(/>/gu, '&gt;').replace(/"/gu, '&quot;').replace(/'/gu, '&apos;');
7
+ }
8
+ function junitCases(document) {
9
+ const schema = String(document.schema || '');
10
+ if (schema === 'whoisleuth.cli.lookup') {
11
+ const diagnostics = record(document.diagnostics);
12
+ const cases = Object.entries(diagnostics).slice(0, 64).map(([source, value]) => {
13
+ const state = String(record(value).status ?? record(value).state ?? 'unavailable').toLowerCase();
14
+ return ['error', 'failed', 'partial', 'rate_limited', 'timeout', 'unavailable'].includes(state)
15
+ ? { name: `source ${source}`, failure: `Source state: ${state}` }
16
+ : { name: `source ${source}` };
17
+ });
18
+ return cases.length ? cases : [{ name: 'lookup diagnostics', failure: 'No source diagnostics were available.' }];
19
+ }
20
+ if (schema === 'whoisleuth.cli.bulk') {
21
+ const summary = record(document.summary);
22
+ const failed = Number(summary.failed ?? 0);
23
+ const results = Array.isArray(document.results) ? document.results.map(record) : [];
24
+ const inconclusive = results.filter((item) => ['unknown', 'inconclusive', 'partial', 'unavailable'].includes(String(record(item.availability).state || '').toLowerCase())).length;
25
+ return [
26
+ failed ? { name: 'collection failures', failure: `${failed} target collection(s) failed.` } : { name: 'collection failures' },
27
+ inconclusive ? { name: 'authority conclusions', failure: `${inconclusive} target state(s) remained inconclusive.` } : { name: 'authority conclusions' },
28
+ ];
29
+ }
30
+ if (schema === 'whoisleuth.cli.domain-control-monitor') {
31
+ const collection = record(document.collection);
32
+ const summary = record(record(document.flightRecorder).summary);
33
+ const failed = Number(collection.failed ?? 0);
34
+ const drift = Number(summary.unexpectedChanges ?? 0);
35
+ return [
36
+ failed ? { name: 'domain control collection', failure: `${failed} owned-domain collection(s) failed.` } : { name: 'domain control collection' },
37
+ drift ? { name: 'unexpected domain control changes', failure: `${drift} complete observation change(s) require review.` } : { name: 'unexpected domain control changes' },
38
+ ];
39
+ }
40
+ return [{ name: 'WHOISleuth result', failure: 'Unsupported document for JUnit conversion.' }];
41
+ }
42
+ export function formatCliJunit(document) {
43
+ const cases = junitCases(record(document));
44
+ const failures = cases.filter((item) => item.failure).length;
45
+ return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuite name="WHOISleuth" tests="${cases.length}" failures="${failures}">\n${cases.map((item) => ` <testcase classname="whoisleuth" name="${xml(item.name)}">${item.failure ? `<failure message="${xml(item.failure)}" />` : ''}</testcase>`).join('\n')}\n</testsuite>\n`;
46
+ }
47
+ export function buildPostureSarif(document) {
48
+ const root = record(document);
49
+ const checks = Array.isArray(root.checks) ? root.checks.map(record).slice(0, 64) : [];
50
+ const actionable = checks.filter((item) => ['warning', 'danger'].includes(String(item.status).toLowerCase()));
51
+ const rules = actionable.map((item) => ({
52
+ id: `WHOISLEUTH-${String(item.id || 'posture').replace(/[^a-z0-9_-]/giu, '-').slice(0, 64)}`,
53
+ shortDescription: { text: String(item.label || 'Domain posture finding').slice(0, 200) },
54
+ help: { text: String(item.remediation || item.detail || 'Review this owned-domain posture finding.').slice(0, 1000) },
55
+ }));
56
+ return Object.freeze({
57
+ version: '2.1.0',
58
+ $schema: 'https://json.schemastore.org/sarif-2.1.0.json',
59
+ runs: [Object.freeze({
60
+ tool: { driver: { name: 'WHOISleuth', informationUri: WHOISLEUTH_REQUEST_POLICY_URL, rules } },
61
+ results: actionable.map((item) => ({
62
+ ruleId: `WHOISLEUTH-${String(item.id || 'posture').replace(/[^a-z0-9_-]/giu, '-').slice(0, 64)}`,
63
+ level: String(item.status).toLowerCase() === 'danger' ? 'error' : 'warning',
64
+ message: { text: `${String(item.summary || item.label || 'Posture finding').slice(0, 500)}${item.remediation ? ` Next: ${String(item.remediation).slice(0, 500)}` : ''}` },
65
+ })),
66
+ properties: {
67
+ ownedDomainConfirmed: true,
68
+ generatedAt: typeof root.generatedAt === 'string' ? root.generatedAt : null,
69
+ evidenceBoundary: 'Passive owned-domain DNS posture; raw records and contact data omitted from SARIF.',
70
+ },
71
+ })],
72
+ });
73
+ }