@slicedearth/whoisleuth-cli 2.5.1 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +9 -6
- package/NOTICE +3 -2
- package/README.md +60 -67
- package/cli/artifact-validation/case-response.mjs +6 -814
- package/cli/artifact-validation/lookup-evidence.mjs +30 -3
- package/cli/artifact-validation/signed-review.mjs +2 -51
- package/cli/artifact-verify.mjs +41 -3
- package/cli/bulk-command-runner.mjs +4 -6
- package/cli/case-command.mjs +92 -16
- package/cli/case-pack.mjs +14 -3
- package/cli/ci-report.mjs +17 -5
- package/cli/command-definition.mjs +7 -6
- package/cli/discovery-scan-command-runner.mjs +4 -6
- package/cli/doctor.mjs +37 -3
- package/cli/domain-feed.mjs +61 -0
- package/cli/formatters/html.mjs +3 -0
- package/cli/formatters/json.mjs +9 -14
- package/cli/formatters/terminal-lookup.mjs +24 -1
- package/cli/history-command-definitions.mjs +1 -1
- package/cli/indicator-set-command.mjs +1 -2
- package/cli/intake-command.mjs +43 -3
- package/cli/investigation-artifacts.mjs +2 -3
- package/cli/investigation-package-review.mjs +2 -0
- package/cli/investigation-plan.mjs +15 -0
- package/cli/local-mmdb-review.mjs +27 -4
- package/cli/lookup-brief.mjs +11 -5
- package/cli/lookup-command-runner.mjs +21 -13
- package/cli/offline-evidence-review.mjs +51 -5
- package/cli/review-arguments.mjs +3 -2
- package/cli/review-command-definitions.mjs +23 -17
- package/cli/review-command-runner.mjs +6 -3
- package/cli/risk-calibration.mjs +6 -4
- package/cli/saved-lookup.mjs +22 -7
- package/cli/selected-input-worker.mjs +12 -2
- package/cli/source-reliability.mjs +12 -10
- package/cli/support-arguments.mjs +1 -1
- package/cli/support-command-definitions.mjs +5 -4
- package/cli/support-command-runner.mjs +9 -7
- package/cli/watchlist-review.mjs +43 -0
- package/cli/workflow-arguments.mjs +14 -0
- package/cli/workflow-command-definitions.mjs +27 -1
- package/cli/workflow-command-runner.mjs +32 -0
- package/docs/cli-reference.md +49 -11
- package/docs/cli.md +200 -26
- package/docs/contextual-reviews.md +38 -5
- package/docs/infrastructure-observations.md +135 -0
- package/docs/offline-intake-context.md +213 -0
- package/lib/credential-form-attribution.mjs +136 -0
- package/lib/credential-surface-profile.mjs +6 -2
- package/lib/dmarc-discovery.mjs +4 -2
- package/lib/dns-delegation-health.mjs +23 -7
- package/lib/dns-inheritance-review.mjs +14 -1
- package/lib/dns-intelligence.mjs +18 -4
- package/lib/domain-posture-analysis.mjs +53 -2
- package/lib/domain-posture-parsers.mjs +9 -0
- package/lib/domain-posture.mjs +29 -10
- package/lib/evidence-export-privacy.mjs +5 -2
- package/lib/evidence-export.mjs +10 -4
- package/lib/evidence-report.mjs +3 -1
- package/lib/favicon.mjs +13 -4
- package/lib/generated/cisa-kev-catalog.mjs +17 -4
- package/lib/generated/registrar-standing-catalogue.mjs +7 -7
- package/lib/generated/retire-browser-catalog.mjs +178 -5
- package/lib/html-signals.mjs +6 -0
- package/lib/idn-confusables.mjs +5 -1
- package/lib/lookup-child-profile-contract.mjs +28 -11
- package/lib/lookup-collection-receipt.mjs +52 -0
- package/lib/lookup-diagnostics.mjs +2 -12
- package/lib/lookup-progress.mjs +29 -31
- package/lib/lookup-response-contract.mjs +10 -0
- package/lib/lookup-response.mjs +15 -14
- package/lib/lookup-source-progress.mjs +4 -2
- package/lib/lookup.mjs +33 -2
- package/lib/operation-budget.mjs +4 -0
- package/lib/rdap-capabilities.mjs +43 -9
- package/lib/registry-insights.mjs +17 -6
- package/lib/service-binding-dns.mjs +19 -2
- package/lib/sslbl-certificates.generated.mjs +142 -140
- package/lib/static-html-analysis.mjs +32 -4
- package/lib/technology-evidence-role.mjs +1 -51
- package/lib/tls-intelligence.mjs +30 -5
- package/lib/website-enrichment.mjs +39 -23
- package/lib/whois-authority.mjs +56 -1
- package/lib/whois-chain.mjs +14 -4
- package/lib/whois-common-formats.mjs +3 -1
- package/lib/whois-contacts.mjs +13 -2
- package/lib/whois-contracts.mjs +2 -0
- package/lib/whois-normalization.mjs +4 -4
- package/package.json +9 -7
- package/packages/analysis/bounded-json.mjs +12 -2
- package/packages/analysis/risk-scoring.mjs +12 -9
- package/packages/cases/case-attachment-model.mjs +36 -1
- package/packages/cases/case-checkpoint-receipts.mjs +68 -0
- package/packages/cases/case-containment-handoff.mjs +195 -0
- package/packages/cases/case-evidence-links.mjs +44 -0
- package/packages/cases/case-evidence-model.mjs +51 -7
- package/packages/cases/case-migration-model.mjs +9 -1
- package/packages/cases/case-object-coverage.mjs +119 -0
- package/packages/cases/case-packet-correction.mjs +106 -0
- package/packages/cases/case-recheck-model.mjs +35 -5
- package/packages/cases/case-record-operations.mjs +87 -9
- package/packages/cases/case-record-projection.mjs +43 -5
- package/packages/cases/case-record-version-input.mjs +23 -1
- package/packages/cases/case-report.mjs +35 -4
- package/packages/cases/case-requested-evidence.mjs +1 -1
- package/packages/cases/case-response-actions.mjs +57 -1
- package/packages/cases/case-response-model.mjs +7 -0
- package/packages/cases/case-response-object.mjs +119 -0
- package/packages/cases/case-response-outcomes.mjs +81 -14
- package/packages/cases/case-response-packet-render.mjs +22 -3
- package/packages/cases/case-response-packet-validation.mjs +880 -0
- package/packages/cases/case-response-packet-vocabulary.mjs +12 -1
- package/packages/cases/case-response-packet.mjs +32 -17
- package/packages/cases/case-response-review-inputs.mjs +66 -8
- package/packages/cases/case-type-presentation.mjs +10 -0
- package/packages/cases/case-workflow-metadata.mjs +1 -17
- package/packages/comparison/capture-context.mjs +3 -2
- package/packages/contracts/analyst-interchange.mjs +7 -5
- package/packages/contracts/analyst-review-state-contract.mjs +1 -1
- package/packages/contracts/candidate-watch-review.mjs +54 -0
- package/packages/contracts/case-portability.mjs +292 -43
- package/packages/contracts/cli-command-semantics.mjs +3 -0
- package/packages/contracts/cli-doctor.mjs +4 -0
- package/packages/contracts/cli-lookup.mjs +57 -46
- package/packages/contracts/context-review.mjs +2 -1
- package/packages/contracts/external-observation-interchange.mjs +96 -28
- package/packages/contracts/lookup-evidence.mjs +13 -2
- package/packages/contracts/message-intake.mjs +7 -1
- package/packages/contracts/registry-insights.mjs +6 -0
- package/packages/contracts/risk-calibration.mjs +44 -29
- package/packages/contracts/schema-lifecycle.mjs +3 -1
- package/packages/contracts/web-capture.mjs +1 -1
- package/packages/contracts/workspace-portability.mjs +86 -16
- package/packages/evidence/collection-quality.mjs +1 -1
- package/packages/evidence/credential-form-attribution.mjs +60 -0
- package/packages/evidence/dmarc-policy-context.mjs +80 -0
- package/packages/evidence/image-regions.mjs +15 -0
- package/packages/evidence/lookup-collection-receipt.mjs +140 -0
- package/packages/evidence/lookup-collection-review.mjs +74 -0
- package/packages/evidence/record-identity.mjs +60 -16
- package/packages/evidence/technology-evidence-role.mjs +51 -0
- package/packages/interchange/external-findings-import.mjs +40 -21
- package/packages/interchange/json-output.mjs +14 -0
- package/packages/interchange/managed-indicator-export.mjs +3 -2
- package/packages/interchange/managed-indicator-set.mjs +13 -0
- package/packages/investigation/acquisition-decision-packet.mjs +78 -0
- package/packages/investigation/bulk-mail-exposure.mjs +2 -2
- package/packages/investigation/context-review.mjs +3 -3
- package/packages/investigation/document-intake.mjs +9 -3
- package/packages/investigation/domain-history-review.mjs +50 -12
- package/packages/investigation/domain-transition-review.mjs +2 -0
- package/packages/investigation/infrastructure-observation.mjs +234 -0
- package/packages/investigation/intake-context.mjs +284 -0
- package/packages/investigation/intake-indicators.mjs +116 -0
- package/packages/investigation/intake-phones.mjs +155 -0
- package/packages/investigation/intake-report.mjs +2 -1
- package/packages/investigation/investigation-manifest.mjs +33 -7
- package/packages/investigation/link-intake.mjs +38 -8
- package/packages/investigation/message-intake.mjs +55 -15
- package/packages/monitoring/domain-feed.mjs +321 -0
- package/packages/workspace/brand-candidate-workflow.mjs +351 -0
- package/packages/workspace/brand-keyword-campaign.mjs +241 -0
- package/packages/workspace/brand-profile-model.mjs +46 -2
- package/packages/workspace/bulk-session-model.mjs +12 -0
- package/packages/workspace/candidate-watch-handoff.mjs +195 -0
- package/packages/workspace/detection-rule-model.mjs +10 -2
- package/packages/workspace/shortlist-model.mjs +10 -2
- package/packages/workspace/watchlist-history.mjs +31 -4
- package/packages/workspace/watchlist-store.mjs +136 -7
- package/third-party-notices.txt +30 -30
package/cli/doctor.mjs
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { withTimeout } from "../lib/abort.mjs";
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
import { CLI_COMMAND_REGISTRY } from "./command-reference.mjs";
|
|
3
|
+
import { DOCTOR_SCHEMA, DOCTOR_VERSION, DOCTOR_SUMMARY_SCHEMA, DOCTOR_SUMMARY_VERSION } from "../packages/contracts/cli-doctor.mjs";
|
|
4
|
+
export { DOCTOR_SCHEMA, DOCTOR_VERSION, DOCTOR_SUMMARY_SCHEMA, DOCTOR_SUMMARY_VERSION };
|
|
4
5
|
const MINIMUM_NODE_MAJOR = 24;
|
|
5
6
|
const NETWORK_HOST = 'whois.iana.org';
|
|
6
7
|
const NETWORK_QUERY = 'example.com';
|
|
7
8
|
const NETWORK_HTTPS_URL = 'https://data.iana.org/rdap/dns.json';
|
|
8
9
|
const NETWORK_TIMEOUT_MS = 6_000;
|
|
9
10
|
const MAX_DOCTOR_DETAIL_LENGTH = 180;
|
|
11
|
+
const DOCTOR_CHECK_IDS = ['runtime', 'platform', 'terminal', 'offline_features', 'network', 'public_dns', 'https_transport', 'whois_transport'];
|
|
10
12
|
function boundedDetail(value, fallback) {
|
|
11
13
|
return String(value || fallback)
|
|
12
14
|
.replace(/[\u0000-\u001f\u007f]+/gu, ' ')
|
|
@@ -162,6 +164,38 @@ async function buildDoctorReport(options) {
|
|
|
162
164
|
checks: Object.freeze(checks),
|
|
163
165
|
});
|
|
164
166
|
}
|
|
167
|
+
function diagnosticVersion(value) {
|
|
168
|
+
return /^(\d{1,6}\.\d{1,6}\.\d{1,6})(?:[-+].*)?$/u.exec(value)?.[1] ?? null;
|
|
169
|
+
}
|
|
170
|
+
function buildDoctorSummary(report, nodeVersion = process.versions.node) {
|
|
171
|
+
const checkCodes = DOCTOR_CHECK_IDS.flatMap(id => {
|
|
172
|
+
const check = report.checks.find(item => item.id === id);
|
|
173
|
+
if (!check)
|
|
174
|
+
return [];
|
|
175
|
+
const state = check.state === 'pass' || check.state === 'partial' || check.state === 'skipped' ? check.state : 'unknown';
|
|
176
|
+
return [`${id}_${state}`];
|
|
177
|
+
});
|
|
178
|
+
const omittedCheckCount = report.checks.length - checkCodes.length;
|
|
179
|
+
return Object.freeze({
|
|
180
|
+
schema: DOCTOR_SUMMARY_SCHEMA,
|
|
181
|
+
version: DOCTOR_SUMMARY_VERSION,
|
|
182
|
+
cliVersion: diagnosticVersion(report.cliVersion),
|
|
183
|
+
nodeVersion: diagnosticVersion(nodeVersion),
|
|
184
|
+
state: report.state === 'pass' && omittedCheckCount === 0 ? 'pass' : 'partial',
|
|
185
|
+
networkRequested: report.networkRequested === true,
|
|
186
|
+
checkCodes: Object.freeze(checkCodes),
|
|
187
|
+
omittedCheckCount,
|
|
188
|
+
declaredFormatIdentifiers: Object.freeze([...new Set(CLI_COMMAND_REGISTRY.flatMap(command => command.documentation.supportedSchemaIdentifiers))].sort()),
|
|
189
|
+
formatScope: 'Public command declarations only; not an installed-package or input-compatibility test.',
|
|
190
|
+
omissions: Object.freeze([
|
|
191
|
+
'Targets, URLs, queries and fragments',
|
|
192
|
+
'File paths, folder names and user names',
|
|
193
|
+
'Launch tokens, cookies, credentials and headers',
|
|
194
|
+
'Input files, response content and free-form diagnostic details',
|
|
195
|
+
'Timestamps, platform details and version prerelease/build metadata',
|
|
196
|
+
]),
|
|
197
|
+
});
|
|
198
|
+
}
|
|
165
199
|
function formatDoctorReport(report) {
|
|
166
200
|
const lines = [
|
|
167
201
|
'Runtime:',
|
|
@@ -175,4 +209,4 @@ function formatDoctorReport(report) {
|
|
|
175
209
|
}
|
|
176
210
|
return `${lines.join('\n')}\n`;
|
|
177
211
|
}
|
|
178
|
-
export { buildDoctorReport, formatDoctorReport, };
|
|
212
|
+
export { buildDoctorReport, buildDoctorSummary, formatDoctorReport, };
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { constants } from 'node:fs';
|
|
2
|
+
import { open } from 'node:fs/promises';
|
|
3
|
+
import { DOMAIN_FEED_LIMITS, normalizeDomainFeedSelection } from "../packages/monitoring/domain-feed.mjs";
|
|
4
|
+
import { CliUsageError } from "./errors.mjs";
|
|
5
|
+
export function parseDomainFeedSelectors(values) {
|
|
6
|
+
const hosts = [], terms = [], negativeTerms = [];
|
|
7
|
+
for (const value of values) {
|
|
8
|
+
if (value.startsWith('host:'))
|
|
9
|
+
hosts.push(value.slice(5));
|
|
10
|
+
else if (value.startsWith('term:'))
|
|
11
|
+
terms.push(value.slice(5));
|
|
12
|
+
else if (value.startsWith('exclude:'))
|
|
13
|
+
negativeTerms.push(value.slice(8));
|
|
14
|
+
else
|
|
15
|
+
throw new CliUsageError('--select requires host:<plain-hostname>, term:<literal-text> or exclude:<literal-text>.');
|
|
16
|
+
}
|
|
17
|
+
const selection = normalizeDomainFeedSelection({ hosts, terms, ...(negativeTerms.length ? { negativeTerms } : {}) });
|
|
18
|
+
if (!selection.hosts.length && !selection.terms.length)
|
|
19
|
+
throw new CliUsageError('Select at least one exact host or literal term.');
|
|
20
|
+
return selection;
|
|
21
|
+
}
|
|
22
|
+
/** Stream one descriptor-bound local regular file; URLs, pipes and symlinks are not inputs. */
|
|
23
|
+
export async function* readDomainFeedFile(source, signal) {
|
|
24
|
+
if (!source || /^[a-z][a-z0-9+.-]*:\/\//iu.test(source))
|
|
25
|
+
throw new CliUsageError('Domain feed review requires an explicit local regular file, not a URL.');
|
|
26
|
+
if (signal?.aborted)
|
|
27
|
+
throw new DOMException('Feed reading was cancelled.', 'AbortError');
|
|
28
|
+
const handle = await open(source, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW);
|
|
29
|
+
let stream;
|
|
30
|
+
const abort = () => stream?.destroy(new DOMException('Feed reading was cancelled.', 'AbortError'));
|
|
31
|
+
try {
|
|
32
|
+
if (signal?.aborted)
|
|
33
|
+
throw new DOMException('Feed reading was cancelled.', 'AbortError');
|
|
34
|
+
const before = await handle.stat();
|
|
35
|
+
if (!before.isFile())
|
|
36
|
+
throw new CliUsageError('The domain feed must be a regular file.');
|
|
37
|
+
if (before.size > DOMAIN_FEED_LIMITS.bytes)
|
|
38
|
+
throw new CliUsageError('The domain feed exceeds its 256 MiB raw-byte ceiling.');
|
|
39
|
+
stream = handle.createReadStream({ autoClose: false, highWaterMark: 64 * 1024 });
|
|
40
|
+
signal?.addEventListener('abort', abort, { once: true });
|
|
41
|
+
if (signal?.aborted)
|
|
42
|
+
abort();
|
|
43
|
+
let bytes = 0;
|
|
44
|
+
for await (const chunk of stream) {
|
|
45
|
+
if (signal?.aborted)
|
|
46
|
+
throw new DOMException('Feed reading was cancelled.', 'AbortError');
|
|
47
|
+
if (!(chunk instanceof Uint8Array) || chunk.byteLength > DOMAIN_FEED_LIMITS.bytes - bytes)
|
|
48
|
+
throw new CliUsageError('The domain feed exceeds its raw-byte ceiling.');
|
|
49
|
+
bytes += chunk.byteLength;
|
|
50
|
+
yield chunk;
|
|
51
|
+
}
|
|
52
|
+
const after = await handle.stat();
|
|
53
|
+
if (bytes !== before.size || after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs)
|
|
54
|
+
throw new CliUsageError('The domain feed changed while it was being read; no review was produced.');
|
|
55
|
+
}
|
|
56
|
+
finally {
|
|
57
|
+
signal?.removeEventListener('abort', abort);
|
|
58
|
+
stream?.destroy();
|
|
59
|
+
await handle.close();
|
|
60
|
+
}
|
|
61
|
+
}
|
package/cli/formatters/html.mjs
CHANGED
|
@@ -40,6 +40,7 @@ function formatLookupEvidenceHtml(document, options = {}) {
|
|
|
40
40
|
<style>
|
|
41
41
|
:root{color-scheme:dark;--bg:#07110d;--panel:#0d1b15;--panel2:#10241b;--line:#254837;--text:#d8f5e4;--muted:#91b7a0;--accent:#6dff9d;--shadow:rgba(0,0,0,.28)}
|
|
42
42
|
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text);font:15px/1.55 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}main{width:min(1120px,calc(100% - 32px));margin:0 auto;padding:42px 0 64px}header{border:1px solid var(--line);background:linear-gradient(145deg,var(--panel2),var(--panel));padding:28px;box-shadow:0 18px 50px var(--shadow)}h1,h2,h3{line-height:1.2;margin:0;color:var(--accent)}h1{font-size:clamp(1.5rem,4vw,2.35rem);overflow-wrap:anywhere}h2{font-size:1.22rem;margin-bottom:16px}h3{font-size:1rem;margin-bottom:12px}.eyebrow{margin:0 0 10px;color:var(--muted);letter-spacing:.12em;text-transform:uppercase}.notice{margin:18px 0 0;color:var(--muted);max-width:85ch}.block{margin-top:24px;border:1px solid var(--line);background:var(--panel);padding:22px}.fields{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:0 24px;margin:0}.fields div{display:grid;grid-template-columns:minmax(130px,.8fr) minmax(0,1.6fr);gap:12px;border-top:1px solid var(--line);padding:9px 0}.fields div:first-child,.fields div:nth-child(2){border-top:0}.fields dt{color:var(--muted)}.fields dd{margin:0;overflow-wrap:anywhere}.card-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,320px),1fr));gap:16px}.card{border:1px solid var(--line);background:var(--panel2);padding:18px}.card .fields{display:block}.card .fields div{grid-template-columns:minmax(120px,.8fr) minmax(0,1.4fr)}.card .fields div:nth-child(2){border-top:1px solid var(--line)}.table-scroll{overflow-x:auto}table{width:100%;border-collapse:collapse;min-width:720px}caption{text-align:left;color:var(--muted);padding:0 0 10px}th,td{border:1px solid var(--line);padding:9px 10px;text-align:left;vertical-align:top;overflow-wrap:anywhere}thead th{background:var(--panel2);color:var(--accent)}tbody th{color:var(--text)}.status{white-space:nowrap;color:var(--accent)}.empty,.omission{color:var(--muted)}ul{margin:0;padding-left:22px}li+li{margin-top:8px}footer{margin-top:24px;color:var(--muted);font-size:.88rem;text-align:center}
|
|
43
|
+
footer{overflow-wrap:anywhere}
|
|
43
44
|
@media(max-width:700px){main{width:min(100% - 20px,1120px);padding-top:16px}header,.block{padding:17px}.fields{display:block}.fields div,.card .fields div{grid-template-columns:1fr;gap:2px}.fields div:nth-child(2){border-top:1px solid var(--line)}}
|
|
44
45
|
@media print{:root{color-scheme:light;--bg:#fff;--panel:#fff;--panel2:#f5f7f5;--line:#b9c5bd;--text:#17231b;--muted:#46564b;--accent:#155c2f;--shadow:transparent}body{font:10pt/1.42 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}main{width:100%;padding:0}header,.block,.card{box-shadow:none;break-inside:avoid}.block{margin-top:12px}}
|
|
45
46
|
</style>
|
|
@@ -55,6 +56,8 @@ function formatLookupEvidenceHtml(document, options = {}) {
|
|
|
55
56
|
<section class="block"><h2>Query</h2>${renderFields(report.query)}</section>
|
|
56
57
|
<section class="block"><h2>Assessment</h2>${renderFields(report.assessment)}</section>
|
|
57
58
|
<section class="block"><h2>Registry sources</h2>${renderGroups(report.registryGroups)}</section>
|
|
59
|
+
${report.registrarStanding.length ? `<section class="block"><h2>Registrar standing</h2>${renderFields(report.registrarStanding)}</section>` : ''}
|
|
60
|
+
${report.registryInterpretation.length ? `<section class="block"><h2>Registry interpretation</h2>${renderFields(report.registryInterpretation)}</section>` : ''}
|
|
58
61
|
<section class="block"><h2>Registry-source comparison</h2>${renderFields(report.comparison.health)}${renderComparison(report.comparison)}</section>
|
|
59
62
|
${report.registrarComparison.fields.length ? `<section class="block"><h2>Registry / registrar RDAP comparison</h2>${renderFields(report.registrarComparison.health)}${renderRegistrarComparison(report.registrarComparison)}</section>` : ''}
|
|
60
63
|
<section class="block"><h2>Network evidence</h2>${renderGroups(report.networkGroups)}</section>
|
package/cli/formatters/json.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { bulkDnsSummary } from "../bulk-output.mjs";
|
|
2
2
|
import { SAVED_LOOKUP_SCHEMA_VERSION, buildCliLookupDocument, } from "../saved-lookup.mjs";
|
|
3
3
|
import { CLI_LOOKUP_SCHEMA } from "../../packages/contracts/cli-lookup.mjs";
|
|
4
|
+
import { terminalSafeJson } from "../../packages/interchange/json-output.mjs";
|
|
4
5
|
const CLI_BULK_SCHEMA = 'whoisleuth.cli.bulk';
|
|
5
6
|
const CLI_BULK_ITEM_SCHEMA = 'whoisleuth.cli.bulk.item';
|
|
6
7
|
const CLI_CT_SEARCH_SCHEMA = 'whoisleuth.cli.ct-search';
|
|
@@ -18,19 +19,6 @@ const CLI_POSTURE_SCHEMA_VERSION = 1;
|
|
|
18
19
|
const CLI_HTTP_SCHEMA_VERSION = 3;
|
|
19
20
|
const CLI_TLS_SCHEMA_VERSION = 1;
|
|
20
21
|
const CLI_COMPARE_SCHEMA_VERSION = 3;
|
|
21
|
-
const TERMINAL_UNSAFE_JSON_RE = /[\u007f-\u009f\u2028\u2029]|\p{Default_Ignorable_Code_Point}/gu;
|
|
22
|
-
function jsonUnicodeEscape(value) {
|
|
23
|
-
const codePoint = value.codePointAt(0);
|
|
24
|
-
if (codePoint <= 0xffff)
|
|
25
|
-
return `\\u${codePoint.toString(16).padStart(4, '0')}`;
|
|
26
|
-
const offset = codePoint - 0x10000;
|
|
27
|
-
const high = 0xd800 + (offset >> 10);
|
|
28
|
-
const low = 0xdc00 + (offset & 0x3ff);
|
|
29
|
-
return `\\u${high.toString(16).padStart(4, '0')}\\u${low.toString(16).padStart(4, '0')}`;
|
|
30
|
-
}
|
|
31
|
-
function terminalSafeJson(value, space) {
|
|
32
|
-
return JSON.stringify(value, null, space).replace(TERMINAL_UNSAFE_JSON_RE, jsonUnicodeEscape);
|
|
33
|
-
}
|
|
34
22
|
function formatJsonDocument(document) {
|
|
35
23
|
return `${terminalSafeJson(document, 2)}\n`;
|
|
36
24
|
}
|
|
@@ -81,7 +69,14 @@ function versionedResult(result, schema, version, generatedAt, requestedField, r
|
|
|
81
69
|
return { ...result, schema, version, generatedAt, [requestedField]: requestedValue };
|
|
82
70
|
}
|
|
83
71
|
function buildCliPostureDocument(requestedDomain, report, generatedAt = new Date().toISOString()) {
|
|
84
|
-
|
|
72
|
+
const checks = report.checks;
|
|
73
|
+
const retained = Array.isArray(checks) ? { ...report, checks: checks.map(check => {
|
|
74
|
+
if (!check || typeof check !== 'object' || Array.isArray(check))
|
|
75
|
+
return check;
|
|
76
|
+
const { dmarcPolicyContext: _transient, ...fields } = check;
|
|
77
|
+
return fields;
|
|
78
|
+
}) } : report;
|
|
79
|
+
return versionedResult(retained, CLI_POSTURE_SCHEMA, CLI_POSTURE_SCHEMA_VERSION, generatedAt, 'requestedDomain', requestedDomain);
|
|
85
80
|
}
|
|
86
81
|
function buildCliHttpDocument(requestedDomain, result, generatedAt = new Date().toISOString()) {
|
|
87
82
|
return versionedResult(result, CLI_HTTP_SCHEMA, CLI_HTTP_SCHEMA_VERSION, generatedAt, 'requestedDomain', requestedDomain);
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { registryAccessProfileLabel } from "../registry-access.mjs";
|
|
2
|
+
import { isSupportedRegistryInsightsVersion } from "../../packages/contracts/registry-insights.mjs";
|
|
2
3
|
import { dnsAddressFamilyEvidence } from "../../packages/evidence/dns-query-outcome.mjs";
|
|
4
|
+
import { lookupCollectionReceiptReview } from "../../packages/evidence/lookup-collection-review.mjs";
|
|
3
5
|
import { appendTechnologyLines } from "./terminal-technology.mjs";
|
|
4
6
|
import { appendDeliveryMetadataLines, appendPublicationMetadataLines } from "./terminal-metadata.mjs";
|
|
5
7
|
import { securityPostureReview, summarizeSecurityPostureReview } from "../../lib/website-security-posture.mjs";
|
|
@@ -252,6 +254,14 @@ function formatTerminalLookup(document, { detail = 'standard' } = {}) {
|
|
|
252
254
|
`RDAP ${titleCase(rdapDiagnostics.status)}`,
|
|
253
255
|
`WHOIS ${titleCase(whoisDiagnostics.status)}`,
|
|
254
256
|
];
|
|
257
|
+
if (detail !== 'summary') {
|
|
258
|
+
for (const [source, status] of [['rdap', rdapDiagnostics.status], ['whois', whoisDiagnostics.status]]) {
|
|
259
|
+
const sourceDetail = terminalRecord(document[source]).detail;
|
|
260
|
+
if (['unsupported', 'skipped', 'disabled'].includes(String(status)) && typeof sourceDetail === 'string' && sourceDetail) {
|
|
261
|
+
registrationLines.push(`${source.toUpperCase()} detail ${boundedTerminalComponent(sourceDetail, 300)}`);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
255
265
|
if (detail !== 'summary' && rdapDiagnostics.endpoint) {
|
|
256
266
|
registrationLines.splice(1, 0, `RDAP source ${safeTerminalValue(rdapDiagnostics.endpoint)}`);
|
|
257
267
|
}
|
|
@@ -267,7 +277,7 @@ function formatTerminalLookup(document, { detail = 'standard' } = {}) {
|
|
|
267
277
|
appendNetworkRegistrationLines(registrationLines, terminalRecord(rdap.parsed), document.type, detail);
|
|
268
278
|
}
|
|
269
279
|
const registryInsights = terminalRecord(document.registryInsights);
|
|
270
|
-
if (detail !== 'summary' && registryInsights.version
|
|
280
|
+
if (detail !== 'summary' && isSupportedRegistryInsightsVersion(registryInsights.version)) {
|
|
271
281
|
const lifecycle = terminalRecord(registryInsights.lifecycle);
|
|
272
282
|
const disclosure = terminalRecord(registryInsights.contactDisclosure);
|
|
273
283
|
const registryDisclosure = terminalRecord(disclosure.registryRdap);
|
|
@@ -712,6 +722,19 @@ function formatTerminalLookup(document, { detail = 'standard' } = {}) {
|
|
|
712
722
|
const collectionLines = [];
|
|
713
723
|
if (detail === 'verbose') {
|
|
714
724
|
collectionLines.push(`Generated ${safeTerminalValue(document.generatedAt, 'unknown')}`);
|
|
725
|
+
const receipt = lookupCollectionReceiptReview(document.collectionReceipt);
|
|
726
|
+
if (receipt) {
|
|
727
|
+
collectionLines.push(`Original plan ${receipt.selection}`);
|
|
728
|
+
for (const choice of receipt.choices)
|
|
729
|
+
collectionLines.push(`${choice.label}: ${choice.selected ? 'selected' : 'not selected'}`);
|
|
730
|
+
for (const source of receipt.sources) {
|
|
731
|
+
collectionLines.push(`${source.label}: ${source.summary}; ${source.outcome}`);
|
|
732
|
+
collectionLines.push(` ${source.explanation}`);
|
|
733
|
+
}
|
|
734
|
+
collectionLines.push('Collector start is local execution provenance, not proof of network delivery or complete evidence.');
|
|
735
|
+
}
|
|
736
|
+
else
|
|
737
|
+
collectionLines.push('Original plan Not retained; original optional choices and collector starts are unknown.');
|
|
715
738
|
const timing = terminalRecord(document.timing);
|
|
716
739
|
if (Number(timing.totalMs) >= 0)
|
|
717
740
|
collectionLines.push(`Total time ${safeTerminalValue(timing.totalMs)} ms`);
|
|
@@ -65,7 +65,7 @@ export const HISTORY_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
65
65
|
reference: {
|
|
66
66
|
description: 'Convert one saved lookup into a versioned evidence report.',
|
|
67
67
|
example: 'whoisleuth export lookup.json --markdown',
|
|
68
|
-
boundary: `
|
|
68
|
+
boundary: `Supported saved Lookups are capped at 8 MiB and scanned for duplicate keys, the prototype-sensitive __proto__ key, and bounded nesting, key, value, and per-container counts before parsing. Current schema-${LOOKUP_EVIDENCE_SCHEMA_VERSION} exports preserve evidence-source attribution and limitations; published v2 schemas ${PUBLISHED_V2_LOOKUP_EVIDENCE_VERSIONS} and exact v1 schema ${V1_PUBLIC_LOOKUP_EVIDENCE_SCHEMA_VERSION} remain readable, while other historical and unreleased shapes are unsupported. Markdown and HTML include a presentation-only generator footer unless --no-attribution is selected; JSON retains bounded generator provenance. Compact output intentionally omits raw registry payloads.`,
|
|
69
69
|
},
|
|
70
70
|
collection: { scope: 'Reads one saved Lookup and writes one bounded report.' },
|
|
71
71
|
summary: 'Convert a lookup to an evidence report',
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { buildManagedIndicatorRevision, managedIndicatorState, parseManagedIndicatorJson, readManagedIndicatorSet } from "../packages/interchange/managed-indicator-set.mjs";
|
|
2
2
|
import { exportManagedIndicators } from "../packages/interchange/managed-indicator-export.mjs";
|
|
3
3
|
import { MAX_MANAGED_INDICATOR_PLAN_BYTES, MAX_MANAGED_INDICATOR_SET_BYTES } from "../packages/contracts/analyst-interchange.mjs";
|
|
4
|
-
import { formatJsonDocument } from "./formatters/json.mjs";
|
|
5
4
|
import { safeTerminalValue } from "./formatters/terminal.mjs";
|
|
6
5
|
import EXIT_CODES from "./exit-codes.mjs";
|
|
7
6
|
export async function runIndicatorSetCommand(args, dependencies, context) {
|
|
@@ -18,7 +17,7 @@ export async function runIndicatorSetCommand(args, dependencies, context) {
|
|
|
18
17
|
if (args.operation === 'stix' || args.operation === 'misp')
|
|
19
18
|
context.writeStdout((await exportManagedIndicators(manifest, args.operation)).content);
|
|
20
19
|
else if (args.output === 'json')
|
|
21
|
-
context.writeStdout(
|
|
20
|
+
context.writeStdout((await exportManagedIndicators(manifest, 'manifest')).content);
|
|
22
21
|
else {
|
|
23
22
|
const lines = [`${safeTerminalValue(manifest.name)} · revision ${manifest.revision}`, `Set ${manifest.id}`, `Content digest ${manifest.integrity.digestSha256}`,
|
|
24
23
|
'Content verified; authorship and earlier revisions are not authenticated.'];
|
package/cli/intake-command.mjs
CHANGED
|
@@ -2,6 +2,8 @@ import { readBoundedRegularFile } from "../lib/bounded-file.mjs";
|
|
|
2
2
|
import { MAX_MESSAGE_INTAKE_BYTES, MESSAGE_INTAKE_INPUTS } from "../packages/contracts/message-intake.mjs";
|
|
3
3
|
import { reviewSelectedInputInWorker } from "./selected-input-worker.mjs";
|
|
4
4
|
import { reviewIdentityIncident } from "../packages/investigation/identity-incident-review.mjs";
|
|
5
|
+
import { intakeIndicators, intakeIndicatorSource } from "../packages/investigation/intake-indicators.mjs";
|
|
6
|
+
import { MAX_INTAKE_CONTEXT_BYTES, applyIntakeContextInput } from "../packages/investigation/intake-context.mjs";
|
|
5
7
|
import { selectReceiverTrust, authenticationHeaderLabel } from "../packages/investigation/mail-authentication-review.mjs";
|
|
6
8
|
import { CliUsageError } from "./errors.mjs";
|
|
7
9
|
import { formatJsonDocument } from "./formatters/json.mjs";
|
|
@@ -33,6 +35,28 @@ export function formatMessageIntake(report) {
|
|
|
33
35
|
lines.push(` Ambiguous parameters: ${link.authorisation.duplicateParameters.join(', ')}`);
|
|
34
36
|
}
|
|
35
37
|
}
|
|
38
|
+
for (const indicator of intakeIndicators(report))
|
|
39
|
+
lines.push(`${indicator.id}: ${indicator.kind.toUpperCase()} ${indicator.value} · literal selected text · ${indicator.location.partId}${indicator.location.page ? ` page ${indicator.location.page}` : ''} · source ${intakeIndicatorSource(report, indicator)}`);
|
|
40
|
+
if (report.schemaVersion === 2) {
|
|
41
|
+
lines.push(`Indicator coverage: ${report.indicatorCoverage.state.replaceAll('_', ' ')}. Hashes require explicit algorithm labels; no indicator was resolved or reputation-checked.`);
|
|
42
|
+
const supplied = report.distributionContext;
|
|
43
|
+
if (supplied)
|
|
44
|
+
lines.push(`Analyst-declared distribution: ${supplied.channel} · ${supplied.sourceLabel} · observed ${supplied.observedAt ?? 'unknown'}`, ` Reference ${supplied.reference ?? 'not supplied'} · observer ${supplied.observerLabel ?? 'unknown'} · vantage ${supplied.vantageLabel ?? 'unknown'}`, 'Distribution, observation time and vantage are supplied claims, not verified capture conditions or independent collection.');
|
|
45
|
+
const selected = report.selectedEvidence;
|
|
46
|
+
if (selected) {
|
|
47
|
+
lines.push(`Deliberately selected phone groups: ${selected.phones.length}. Original spans use zero-based, end-exclusive UTF-16 code units.`);
|
|
48
|
+
lines.push(`Phone extraction coverage: ${selected.phoneCoverage.state.replaceAll('_', ' ')}; ${selected.phoneCoverage.candidatesShown} transient candidates shown, ${selected.phoneCoverage.candidatesReviewed} candidate runs reviewed.`);
|
|
49
|
+
for (const phone of selected.phones) {
|
|
50
|
+
lines.push(` ${phone.canonical ?? 'No canonical number'} · ${phone.state.replaceAll('_', ' ')} · declared ${phone.declaration.role.replaceAll('_', ' ')} · ${phone.declaration.basis.replaceAll('_', ' ')} · country context ${phone.declaration.countryCallingCode ?? 'unknown'}`, ` Source ${phone.declaration.sourceLabel}; declared observation ${phone.declaration.observedAt ?? 'unknown'}; input ${selected.sourceDigestSha256}`);
|
|
51
|
+
for (const occurrence of phone.occurrences)
|
|
52
|
+
lines.push(` ${JSON.stringify(occurrence.original)} · [${occurrence.start}, ${occurrence.end})${phone.extension ? ` · separate extension ${phone.extension}` : ''}`);
|
|
53
|
+
}
|
|
54
|
+
const pair = selected.destinationPair;
|
|
55
|
+
if (pair)
|
|
56
|
+
lines.push(`Supplied destination pair: ${pair.state.replaceAll('_', ' ')} · displayed ${pair.displayed.hostname ?? pair.displayed.state} · destination ${pair.destination.hostname ?? pair.destination.state}`, ` Displayed source ${pair.displayedDeclaration.sourceLabel}; declared ${pair.displayedDeclaration.observedAt ?? 'unknown'}; ${pair.displayedDeclaration.basis}`, ` Destination role ${pair.destinationDeclaration.role}; source ${pair.destinationDeclaration.sourceLabel}; declared ${pair.destinationDeclaration.observedAt ?? 'unknown'}; ${pair.destinationDeclaration.basis}`);
|
|
57
|
+
lines.push(...selected.limitations);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
36
60
|
if (report.actionHints.length)
|
|
37
61
|
lines.push(`Review wording: ${report.actionHints.map(value => value.replaceAll('_', ' ')).join(', ')}`);
|
|
38
62
|
if (report.documentReview) {
|
|
@@ -70,7 +94,7 @@ export function formatMessageIntake(report) {
|
|
|
70
94
|
lines.push(`Partial analysis: ${report.coverage.boundsReached.join('; ')}`);
|
|
71
95
|
if (report.source.kind === 'qr' && !report.coverage.reviewedParts)
|
|
72
96
|
lines.push('No QR text was decoded. This does not establish that the image contains no QR symbol.');
|
|
73
|
-
lines.push('Exact paths, queries, fragments and
|
|
97
|
+
lines.push('Exact paths, queries, fragments and surrounding message content are not included. Only deliberately selected phone substrings can be retained. Link relationships and authorisation parameters are supplied claims, not proof of a completed redirect or account compromise.');
|
|
74
98
|
return `${lines.join('\n')}\n`;
|
|
75
99
|
}
|
|
76
100
|
export async function runIntakeCommand(args, dependencies, context) {
|
|
@@ -95,11 +119,27 @@ export async function runIntakeCommand(args, dependencies, context) {
|
|
|
95
119
|
throw new CliUsageError(cause.message);
|
|
96
120
|
throw cause;
|
|
97
121
|
}
|
|
98
|
-
|
|
122
|
+
let report = { ...result.report, authenticationReview, identityRecovery: reviewIdentityIncident({ reportedActions: args.reportedActions }) };
|
|
123
|
+
if (args.intakeContextSource) {
|
|
124
|
+
if (args.intakeContextSource === '-')
|
|
125
|
+
throw new CliUsageError('Distribution context requires a separate selected file, not stdin.');
|
|
126
|
+
report = applyIntakeContextInput({ ...result, report }, await context.readInput(args.intakeContextSource, MAX_INTAKE_CONTEXT_BYTES, 'Intake context'));
|
|
127
|
+
}
|
|
99
128
|
dependencies.signal?.throwIfAborted();
|
|
129
|
+
if (!args.quiet && result.phoneReview && (result.phoneReview.candidates.length || result.phoneReview.state === 'partial')) {
|
|
130
|
+
const phones = result.phoneReview;
|
|
131
|
+
context.writeStderr([
|
|
132
|
+
`Transient phone candidates: ${phones.candidates.length}; coverage ${phones.state}. Values are omitted until explicitly selected.`,
|
|
133
|
+
`Full source digest: ${phones.sourceDigestSha256}`,
|
|
134
|
+
...phones.candidates.map(candidate => `${candidate.id}: UTF-16 span [${candidate.start}, ${candidate.end}) · ${candidate.state.replaceAll('_', ' ')}`),
|
|
135
|
+
'Review these spans in the original plaintext. Use --intake-context with version 2, review.sourceDigestSha256, and review.phones entries containing start, end and an explicit declaration. Offsets are zero-based UTF-16 code units, end exclusive; retain BOMs and supplementary characters when counting.',
|
|
136
|
+
'Only selected values enter the private report. Public/trusted Case summaries and STIX/MISP mappings omit phone values.',
|
|
137
|
+
'',
|
|
138
|
+
].join('\n'));
|
|
139
|
+
}
|
|
100
140
|
if (!args.quiet)
|
|
101
141
|
context.writeStdout(args.output === 'json' ? formatJsonDocument(report) : context.terminal(formatMessageIntake(report), args.color));
|
|
102
|
-
return args.strictExit && report.coverage.state === 'partial' ? EXIT_CODES.PARTIAL_FAILURE : EXIT_CODES.SUCCESS;
|
|
142
|
+
return args.strictExit && (report.coverage.state === 'partial' || result.phoneReview?.state === 'partial') ? EXIT_CODES.PARTIAL_FAILURE : EXIT_CODES.SUCCESS;
|
|
103
143
|
}
|
|
104
144
|
finally {
|
|
105
145
|
bytes.fill(0);
|
|
@@ -10,7 +10,7 @@ import { validateLookupEvidenceArtifactStructure } from "./artifact-validation/l
|
|
|
10
10
|
import { CLI_DISCOVERY_SCAN_SCHEMA, CLI_DISCOVERY_SCAN_VERSION } from "./discovery-scan.mjs";
|
|
11
11
|
import { CLI_DISCOVER_SCHEMA, CLI_DISCOVER_SCHEMA_VERSION, CLI_POSTURE_SCHEMA, CLI_POSTURE_SCHEMA_VERSION, CLI_CT_SEARCH_SCHEMA, CLI_CT_SEARCH_SCHEMA_VERSION, CLI_COMPARE_SCHEMA, CLI_COMPARE_SCHEMA_VERSION, } from "./formatters/json.mjs";
|
|
12
12
|
import { CLI_LOOKUP_BRIEF_SCHEMA, CLI_LOOKUP_BRIEF_VERSION } from "../packages/contracts/cli-lookup-brief.mjs";
|
|
13
|
-
import { EXTERNAL_FINDINGS_SCHEMA,
|
|
13
|
+
import { EXTERNAL_FINDINGS_SCHEMA, parseExternalFindingsDocument } from "../packages/interchange/external-findings-import.mjs";
|
|
14
14
|
import { SOURCE_RELIABILITY_REPORT_SCHEMA, SOURCE_RELIABILITY_REPORT_VERSION } from "./source-reliability.mjs";
|
|
15
15
|
import { CLI_CASE_PACK_SCHEMA, CLI_CASE_PACK_VERSION, verifyCliCasePack } from "./case-pack.mjs";
|
|
16
16
|
import { SHARING_REVIEW_SCHEMA, SHARING_REVIEW_VERSION } from "./sharing-review.mjs";
|
|
@@ -59,8 +59,7 @@ export function validateWorkflowResult(step, result, retained) {
|
|
|
59
59
|
break;
|
|
60
60
|
case 'ct-intake':
|
|
61
61
|
schema = EXTERNAL_FINDINGS_SCHEMA;
|
|
62
|
-
version =
|
|
63
|
-
parseExternalFindingsDocument(value);
|
|
62
|
+
version = parseExternalFindingsDocument(value).schemaVersion;
|
|
64
63
|
break;
|
|
65
64
|
case 'brief':
|
|
66
65
|
schema = CLI_LOOKUP_BRIEF_SCHEMA;
|
|
@@ -74,6 +74,7 @@ export async function verifyOfflineInvestigationPackage(input, passphrase) {
|
|
|
74
74
|
}
|
|
75
75
|
entries.push({ id: reviewed.entry.id, byteLength: reviewed.entry.byteLength,
|
|
76
76
|
mediaType: 'mediaType' in reviewed.entry ? reviewed.entry.mediaType : 'application/json',
|
|
77
|
+
imageDerivation: 'imageDerivation' in reviewed.entry ? reviewed.entry.imageDerivation : null,
|
|
77
78
|
state, identity: reviewed.state === 'identity_verified' ? 'verified' : 'failed', verification, issue });
|
|
78
79
|
}
|
|
79
80
|
const complete = entries.every((entry) => entry.state === 'admitted' || entry.state === 'opaque')
|
|
@@ -97,6 +98,7 @@ export async function verifyOfflineInvestigationPackage(input, passphrase) {
|
|
|
97
98
|
'Capture attachment checks bind selected bytes to the capture manifest declaration, not to a trusted publisher or real website observation.',
|
|
98
99
|
'Case-file checks match original references to other verified package bytes by length and digest, not by filename or claimed source. Missing originals make the package partial; no file is fetched or imported.',
|
|
99
100
|
'Source declarations and packaging times are not authenticated. Signatures and timestamp tokens require their own verification and trust decisions.',
|
|
101
|
+
'Image derivation declares the immediate parent fingerprint and editing operations, not proof of transformation or complete redaction. Parent bytes are optional; missing declarations leave editing history unknown.',
|
|
100
102
|
'This command does not import, overwrite or collect evidence. Unchanged bytes and valid source structure do not establish factual accuracy, currentness or safe sharing.',
|
|
101
103
|
]),
|
|
102
104
|
});
|
|
@@ -75,6 +75,7 @@ export function formatWorkflowRecipeCatalogue(catalogue) {
|
|
|
75
75
|
`${recipe.id} — ${recipe.label}`,
|
|
76
76
|
` Subject: ${recipe.subjectRequirement.replaceAll('_', ' ')}`,
|
|
77
77
|
` Workflow run: ${recipe.runnableByWorkflowRun ? 'supported' : 'plan and explanation only'}`,
|
|
78
|
+
...formatWorkflowEffort(recipe.id, recipe.steps).map((line) => ` ${line}`),
|
|
78
79
|
` ${recipe.objective}`,
|
|
79
80
|
...recipe.steps.map((item, index) => ` ${index + 1}. ${item.label} [${item.mode}; ${item.approval.replaceAll('_', ' ')}]`),
|
|
80
81
|
...recipe.limitations.map((item) => ` Limitation: ${item}`),
|
|
@@ -84,6 +85,19 @@ export function formatWorkflowRecipeCatalogue(catalogue) {
|
|
|
84
85
|
'',
|
|
85
86
|
].join('\n');
|
|
86
87
|
}
|
|
88
|
+
function formatWorkflowEffort(recipeId, steps) {
|
|
89
|
+
const counts = { network: 0, offline: 0 };
|
|
90
|
+
for (const step of steps)
|
|
91
|
+
counts[step.mode] += 1;
|
|
92
|
+
const reuse = workflowStandardInputs(recipeId).length;
|
|
93
|
+
return [
|
|
94
|
+
`Work planned: ${counts.network} networked step${counts.network === 1 ? '' : 's'}; ${counts.offline} offline step${counts.offline === 1 ? '' : 's'}.`,
|
|
95
|
+
reuse
|
|
96
|
+
? `Evidence reuse: ${reuse} link${reuse === 1 ? '' : 's'} from earlier outputs to later steps.`
|
|
97
|
+
: 'Evidence reuse: no earlier-output links are declared.',
|
|
98
|
+
'Counts describe steps and reuse links, not requests. Saved files and cache state are not checked.',
|
|
99
|
+
];
|
|
100
|
+
}
|
|
87
101
|
function validatedSteps(steps) {
|
|
88
102
|
for (const step of steps) {
|
|
89
103
|
const invocationEffect = cliInvocationNetworkEffect(step.command, step.arguments);
|
|
@@ -147,6 +161,7 @@ export function formatInvestigationPlan(plan) {
|
|
|
147
161
|
`Investigation plan: ${plan.recipe.label}`,
|
|
148
162
|
`Subject ${plan.subject}`,
|
|
149
163
|
`Mode ${plan.execution}`,
|
|
164
|
+
...formatWorkflowEffort(plan.recipe.id, plan.steps),
|
|
150
165
|
'',
|
|
151
166
|
...plan.steps.flatMap((item, index) => [
|
|
152
167
|
`${index + 1}. ${item.label}`,
|
|
@@ -31,26 +31,33 @@ function names(value) {
|
|
|
31
31
|
const source = record(value);
|
|
32
32
|
return boundedText(source.en, 120);
|
|
33
33
|
}
|
|
34
|
-
async function readStableMmdb(databasePath, openFile = open) {
|
|
34
|
+
async function readStableMmdb(databasePath, openFile = open, signal) {
|
|
35
35
|
let handle = null;
|
|
36
|
+
let database = null;
|
|
36
37
|
try {
|
|
38
|
+
signal?.throwIfAborted();
|
|
37
39
|
const resolvedPath = await realpath(databasePath);
|
|
40
|
+
signal?.throwIfAborted();
|
|
38
41
|
handle = await openFile(resolvedPath, fsConstants.O_RDONLY | fsConstants.O_NONBLOCK | fsConstants.O_NOFOLLOW);
|
|
42
|
+
signal?.throwIfAborted();
|
|
39
43
|
const before = await handle.stat();
|
|
44
|
+
signal?.throwIfAborted();
|
|
40
45
|
if (!before.isFile() || before.size <= 0 || before.size > MAX_LOCAL_MMDB_BYTES) {
|
|
41
46
|
throw new CliUsageError(`The supplied MMDB must be a file no larger than ${MAX_LOCAL_MMDB_BYTES} bytes.`);
|
|
42
47
|
}
|
|
43
48
|
// Dedicated backing storage can be transferred without copying a pooled
|
|
44
49
|
// buffer or exposing bytes outside the stable file's exact length.
|
|
45
|
-
|
|
50
|
+
database = Buffer.allocUnsafeSlow(before.size);
|
|
46
51
|
let offset = 0;
|
|
47
52
|
while (offset < database.length) {
|
|
48
53
|
const { bytesRead } = await handle.read(database, offset, database.length - offset, offset);
|
|
54
|
+
signal?.throwIfAborted();
|
|
49
55
|
if (bytesRead === 0)
|
|
50
56
|
break;
|
|
51
57
|
offset += bytesRead;
|
|
52
58
|
}
|
|
53
59
|
const after = await handle.stat();
|
|
60
|
+
signal?.throwIfAborted();
|
|
54
61
|
if (offset !== database.length
|
|
55
62
|
|| before.dev !== after.dev
|
|
56
63
|
|| before.ino !== after.ino
|
|
@@ -62,6 +69,8 @@ async function readStableMmdb(databasePath, openFile = open) {
|
|
|
62
69
|
return database;
|
|
63
70
|
}
|
|
64
71
|
catch (cause) {
|
|
72
|
+
database?.fill(0);
|
|
73
|
+
signal?.throwIfAborted();
|
|
65
74
|
if (cause instanceof CliUsageError)
|
|
66
75
|
throw cause;
|
|
67
76
|
throw new CliUsageError('The supplied MMDB file could not be read.');
|
|
@@ -238,11 +247,17 @@ if (!isMainThread && parentPort && record(workerData).kind === LOCAL_MMDB_WORKER
|
|
|
238
247
|
/** One disposable worker pre-empts the existing synchronous reader. It is not
|
|
239
248
|
* a sandbox and never receives a database path or inherited environment. */
|
|
240
249
|
export async function reviewLocalMmdb(input, databasePath, checkedAt = new Date().toISOString(), dependencies = {}) {
|
|
250
|
+
const { signal } = dependencies;
|
|
251
|
+
signal?.throwIfAborted();
|
|
241
252
|
const query = prepareQuery(input, checkedAt);
|
|
242
253
|
const deadlineMs = dependencies.deadlineMs ?? LOCAL_MMDB_REVIEW_DEADLINE_MS;
|
|
243
254
|
if (!Number.isSafeInteger(deadlineMs) || deadlineMs <= 0 || deadlineMs > LOCAL_MMDB_REVIEW_DEADLINE_MS)
|
|
244
255
|
throw new CliUsageError('Invalid local database processing deadline.');
|
|
245
|
-
const database = await readStableMmdb(databasePath, dependencies.openFile);
|
|
256
|
+
const database = await readStableMmdb(databasePath, dependencies.openFile, signal);
|
|
257
|
+
if (signal?.aborted) {
|
|
258
|
+
database.fill(0);
|
|
259
|
+
signal.throwIfAborted();
|
|
260
|
+
}
|
|
246
261
|
const identity = { sha256: createHash('sha256').update(database).digest('hex'), byteLength: database.length };
|
|
247
262
|
const options = { workerData: { kind: LOCAL_MMDB_WORKER_KIND, query, bytes: database, identity },
|
|
248
263
|
transferList: [database.buffer], env: {}, execArgv: [], stdout: true, stderr: true,
|
|
@@ -254,6 +269,7 @@ export async function reviewLocalMmdb(input, databasePath, checkedAt = new Date(
|
|
|
254
269
|
catch {
|
|
255
270
|
if (database.byteLength)
|
|
256
271
|
database.fill(0);
|
|
272
|
+
signal?.throwIfAborted();
|
|
257
273
|
if (query.version === 2)
|
|
258
274
|
return freezeResult(currentResult(query, identity, { metadata: null, ageDays: null, reason: 'invalid_metadata' }, 'processing_unavailable'));
|
|
259
275
|
throw new CliUsageError('The supplied database could not be reviewed within its format and processing bounds.');
|
|
@@ -268,17 +284,24 @@ export async function reviewLocalMmdb(input, databasePath, checkedAt = new Date(
|
|
|
268
284
|
return;
|
|
269
285
|
settled = true;
|
|
270
286
|
clearTimeout(timer);
|
|
271
|
-
|
|
287
|
+
signal?.removeEventListener('abort', onAbort);
|
|
288
|
+
if (signal?.aborted)
|
|
289
|
+
reject(signal.reason);
|
|
290
|
+
else if (result)
|
|
272
291
|
resolve(freezeResult(result));
|
|
273
292
|
else if (query.version === 2)
|
|
274
293
|
resolve(freezeResult(currentResult(query, identity, { metadata: null, ageDays: null, reason: 'invalid_metadata' }, reason)));
|
|
275
294
|
else
|
|
276
295
|
reject(new CliUsageError('The supplied database could not be reviewed within its format and processing bounds.'));
|
|
277
296
|
};
|
|
297
|
+
const onAbort = () => finish();
|
|
278
298
|
const timer = setTimeout(() => finish(undefined, 'processing_deadline'), deadlineMs);
|
|
279
299
|
worker.once('message', (reply) => reply && 'result' in reply ? finish(reply.result) : finish());
|
|
280
300
|
worker.once('error', () => finish());
|
|
281
301
|
worker.once('exit', () => finish());
|
|
302
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
303
|
+
if (signal?.aborted)
|
|
304
|
+
onAbort();
|
|
282
305
|
});
|
|
283
306
|
}
|
|
284
307
|
finally {
|
package/cli/lookup-brief.mjs
CHANGED
|
@@ -214,23 +214,29 @@ export function buildCliLookupBrief(input, generatedAt = new Date().toISOString(
|
|
|
214
214
|
});
|
|
215
215
|
}
|
|
216
216
|
export function formatCliLookupBrief(document) {
|
|
217
|
+
const sourceFacts = document.sourceHealth.flatMap((source) => {
|
|
218
|
+
const facts = document.facts.filter((item) => item.source.id === source.id);
|
|
219
|
+
return [
|
|
220
|
+
` ${safeTerminalValue(source.label)} [${safeTerminalValue(source.state)}; ${source.observedAt ? `observed ${safeTerminalValue(source.observedAt)}` : 'source time not recorded'}]`,
|
|
221
|
+
...(facts.length
|
|
222
|
+
? facts.map((item) => ` ${safeTerminalValue(item.label)}: ${safeTerminalValue(item.value)}`)
|
|
223
|
+
: [' No fact selected for this brief.']),
|
|
224
|
+
];
|
|
225
|
+
});
|
|
217
226
|
return [
|
|
218
227
|
`Lookup brief: ${safeTerminalValue(document.target)}`,
|
|
219
228
|
`Observed ${safeTerminalValue(document.observedAt)}`,
|
|
220
229
|
`Mode ${safeTerminalValue(document.mode)}`,
|
|
221
230
|
'',
|
|
222
231
|
'Source-attributed facts',
|
|
223
|
-
...
|
|
232
|
+
...sourceFacts,
|
|
224
233
|
'',
|
|
225
234
|
'Contradictions',
|
|
226
235
|
...(document.contradictions.length ? document.contradictions.flatMap((item) => [
|
|
227
236
|
` ${safeTerminalValue(item.field)}: ${safeTerminalValue(item.detail)}`,
|
|
228
|
-
...item.observations.map((observation) => ` ${safeTerminalValue(observation.source.label)}: ${safeTerminalValue(observation.value)} (${safeTerminalValue(observation.state)}; ${observation.source.observedAt ? `observed ${safeTerminalValue(observation.source.observedAt)}` : 'source time not recorded'})`),
|
|
237
|
+
...item.observations.map((observation) => ` ${safeTerminalValue(observation.source.label)}: ${safeTerminalValue(observation.value)} (value state: ${safeTerminalValue(observation.state)}; source state: ${safeTerminalValue(observation.source.state)}; ${observation.source.observedAt ? `observed ${safeTerminalValue(observation.source.observedAt)}` : 'source time not recorded'})`),
|
|
229
238
|
]) : [' No conflicting publication field was retained.']),
|
|
230
239
|
'',
|
|
231
|
-
'Unknown or incomplete',
|
|
232
|
-
...(document.unknowns.length ? document.unknowns.map((item) => ` ${safeTerminalValue(item)}`) : [' No incomplete source state was identified.']),
|
|
233
|
-
'',
|
|
234
240
|
'Recommended manual actions',
|
|
235
241
|
...document.actionPlan.flatMap((item, index) => [
|
|
236
242
|
` ${index + 1}. ${safeTerminalValue(item.action)}`,
|
|
@@ -67,25 +67,32 @@ async function runLookupCommand(args, dependencies, context) {
|
|
|
67
67
|
return EXIT_CODES.SUCCESS;
|
|
68
68
|
}
|
|
69
69
|
let documentGeneratedAt = '';
|
|
70
|
-
const buildDocument = (result) => {
|
|
70
|
+
const buildDocument = ({ result, collectionReceipt }) => {
|
|
71
71
|
documentGeneratedAt = context.now();
|
|
72
72
|
return buildCliLookupDocument(query, classified, result, documentGeneratedAt, args.deep ? 'deep' : 'fast', {
|
|
73
|
+
collectionReceipt,
|
|
73
74
|
...(args.observerLabel ? { observerLabel: args.observerLabel } : {}),
|
|
74
75
|
...(args.vantageLabel ? { vantageLabel: args.vantageLabel } : {}),
|
|
75
76
|
});
|
|
76
77
|
};
|
|
77
|
-
const collect = (signal, onSourceSettled) => abortable(() =>
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
78
|
+
const collect = (signal, onSourceSettled) => abortable(async () => {
|
|
79
|
+
let collectionReceipt;
|
|
80
|
+
const result = await executeLookup(classified, {
|
|
81
|
+
fast: !args.deep,
|
|
82
|
+
compact: false,
|
|
83
|
+
onCollectionReceipt: value => { collectionReceipt = value; },
|
|
84
|
+
...(signal ? { signal } : {}),
|
|
85
|
+
...(args.deep ? {
|
|
86
|
+
...(selectedUrl ? { selectedUrl } : {}),
|
|
87
|
+
onSourceSettled: settlement => {
|
|
88
|
+
onSourceSettled(settlement);
|
|
89
|
+
eventProgress.emit({ event: 'source_settled', source: settlement.source, state: settlement.state });
|
|
90
|
+
},
|
|
91
|
+
} : {}),
|
|
92
|
+
});
|
|
93
|
+
signal?.throwIfAborted();
|
|
94
|
+
return { result, collectionReceipt };
|
|
95
|
+
}, signal);
|
|
89
96
|
let document;
|
|
90
97
|
if (args.browse === true) {
|
|
91
98
|
const browse = dependencies.browseLookupOperation || browseLookupOperation;
|
|
@@ -110,6 +117,7 @@ async function runLookupCommand(args, dependencies, context) {
|
|
|
110
117
|
const save = dependencies.writePrivateFile || writePrivateFile;
|
|
111
118
|
await save(args.saveLookup, serializeCliLookupDocument(document), {
|
|
112
119
|
existingFileMessage: 'Saved Lookup file already exists. Choose a new --save-lookup path or remove the existing file explicitly.',
|
|
120
|
+
beforePublish: async () => { dependencies.signal?.throwIfAborted(); },
|
|
113
121
|
});
|
|
114
122
|
context.writeStderr('Saved the completed private Lookup JSON. It can contain raw public registry and WHOIS responses plus target and collection context; review it before sharing.\n');
|
|
115
123
|
}
|