@slicedearth/whoisleuth-cli 1.45.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +7 -3
- package/NOTICE +5 -4
- package/README.md +74 -124
- package/SECURITY.md +6 -3
- package/bin/whoisleuth.mjs +7 -4
- package/cli/archive-inspect.mjs +28 -10
- package/cli/arguments.mjs +450 -173
- package/cli/artifact-structure.mjs +2507 -0
- package/cli/artifact-verify.mjs +286 -64
- package/cli/bounded-json.mjs +1 -0
- package/cli/bulk-checkpoint.mjs +35 -23
- package/cli/bulk-command-runner.mjs +2 -5
- package/cli/bulk.mjs +9 -2
- package/cli/case-pack.mjs +435 -46
- package/cli/ci-report.mjs +6 -1
- package/cli/collection-preflight.mjs +2 -2
- package/cli/command-catalogue.mjs +12 -3
- package/cli/command-reference.mjs +948 -106
- package/cli/completion.mjs +504 -218
- package/cli/config-profile.mjs +13 -3
- package/cli/ct-event-intake.mjs +32 -2
- package/cli/discover.mjs +7 -1
- package/cli/discovery-observation-snapshot.mjs +24 -28
- package/cli/discovery-scan-command-runner.mjs +13 -14
- package/cli/discovery-scan.mjs +17 -3
- package/cli/discovery-snapshot.mjs +15 -7
- package/cli/discovery-workflow.mjs +2 -3
- package/cli/doctor.mjs +2 -2
- package/cli/domain-control-monitor.mjs +215 -36
- package/cli/domain-control-observations.mjs +34 -17
- package/cli/errors.mjs +8 -3
- package/cli/evidence-command-runner.mjs +17 -11
- package/cli/evidence-signing.mjs +68 -50
- package/cli/export-evidence.mjs +11 -4
- package/cli/external-observation-mapping.mjs +3 -3
- package/cli/formatters/html.mjs +3 -3
- package/cli/formatters/json.mjs +20 -32
- package/cli/formatters/terminal.mjs +586 -22
- package/cli/input.mjs +62 -0
- package/cli/interactive-launcher.mjs +226 -0
- package/cli/interchange-report.mjs +226 -0
- package/cli/investigation-manifest.mjs +16 -17
- package/cli/investigation-plan.mjs +167 -16
- package/cli/investigation-run.mjs +80 -24
- package/cli/local-mmdb-review.mjs +45 -14
- package/cli/lookalike-calibration.mjs +3 -1
- package/cli/lookup-brief.mjs +13 -8
- package/cli/lookup-browser.mjs +844 -0
- package/cli/lookup-command-runner.mjs +96 -33
- package/cli/lookup-diff.mjs +11 -2
- package/cli/lookup-plan.mjs +1 -0
- package/cli/mail-review.mjs +206 -94
- package/cli/manual.mjs +5 -5
- package/cli/network-command-runner.mjs +78 -0
- package/cli/offline-evidence-review.mjs +45 -14
- package/cli/open-asset-model-bridge.mjs +1 -1
- package/cli/output-file.mjs +70 -14
- package/cli/page-compare.mjs +18 -10
- package/cli/progress-events.mjs +2 -1
- package/cli/progress.mjs +3 -3
- package/cli/registry-cohort.mjs +343 -61
- package/cli/registry-doctor.mjs +67 -28
- package/cli/registry-support.mjs +2 -1
- package/cli/retained-artifact-diff.mjs +375 -0
- package/cli/risk-calibration.mjs +89 -40
- package/cli/runner.mjs +322 -246
- package/cli/saved-lookup.mjs +439 -49
- package/cli/sharing-review.mjs +18 -7
- package/cli/source-reliability.mjs +75 -36
- package/cli/terminal-presentation.mjs +43 -18
- package/docs/cli-reference.md +125 -1334
- package/docs/cli.md +87 -236
- package/frontend/src/lib/analysis/domain-control-manifest-core.js +3 -0
- package/frontend/src/lib/analysis/domain-control-records.js +3 -0
- package/lib/application-version.mjs +3 -0
- package/lib/availability.mjs +72 -52
- package/lib/bounded-contract-normalizers.mjs +2 -4
- package/lib/bounded-file.mjs +91 -4
- package/lib/bounded-json.mjs +287 -0
- package/lib/browser-library-profile.mjs +1 -1
- package/lib/certificate-extension-profile.mjs +2 -1
- package/lib/classify.mjs +56 -5
- package/lib/client-behavior-profile.mjs +1 -1
- package/lib/credential-surface-profile.mjs +1 -1
- package/lib/cryptographic-assurance.mjs +173 -0
- package/lib/ct-query.mjs +3 -3
- package/lib/ct-search.mjs +120 -64
- package/lib/distributed-operation-budget.mjs +4 -2
- package/lib/dns-delegation-health.mjs +9 -6
- package/lib/dns-intelligence.mjs +4 -3
- package/lib/dns-resolver-selection.mjs +6 -4
- package/lib/dnssec-chain-validation.mjs +1052 -0
- package/lib/dnssec-evidence-validation.mjs +23 -10
- package/lib/domain-assurance.mjs +12 -12
- package/lib/domain-change-packet.mjs +7 -5
- package/lib/domain-control-flight-recorder.mjs +442 -78
- package/lib/domain-control-manifest.mjs +314 -171
- package/lib/domain-posture-analysis.mjs +5 -2
- package/lib/domain-posture.mjs +7 -4
- package/lib/encrypted-dns-contract.mjs +2 -3
- package/lib/evidence-export.mjs +1243 -88
- package/lib/evidence-report-markdown.mjs +3 -0
- package/lib/evidence-report.mjs +5 -0
- package/lib/external-intelligence-risk.mjs +26 -10
- package/lib/favicon.mjs +8 -5
- package/lib/feature-policy.mjs +1 -1
- package/lib/homepage-metadata-contract.mjs +290 -0
- package/lib/hostname.mjs +1 -1
- package/lib/html-signals.mjs +89 -5
- package/lib/http-evidence-bounds.mjs +3 -0
- package/lib/http-intelligence.mjs +332 -6
- package/lib/interchange-fidelity-registry.mjs +124 -0
- package/lib/local-geoip-evidence.mjs +2 -1
- package/lib/lookup-cache.mjs +6 -4
- package/lib/lookup-network-evidence-bounds.mjs +9 -0
- package/lib/lookup.mjs +27 -4
- package/lib/observed-network-context.mjs +1 -1
- package/lib/outbound-identity.mjs +2 -3
- package/lib/page-fingerprints.mjs +2 -2
- package/lib/page-role-profile.mjs +1 -1
- package/lib/perceptual-hash.mjs +12 -4
- package/lib/portable-generator.mjs +29 -0
- package/lib/provider-policy-admission.mjs +11 -4
- package/lib/rdap-bootstrap.mjs +76 -16
- package/lib/rdap-client.mjs +20 -12
- package/lib/rdap-normalization.mjs +324 -80
- package/lib/rdap-registrar.mjs +5 -1
- package/lib/rdap-transport.mjs +2 -2
- package/lib/rdap-validation.mjs +65 -14
- package/lib/rdap.mjs +2 -2
- package/lib/registrable-domain.mjs +36 -0
- package/lib/registry-capabilities.mjs +3 -0
- package/lib/registry-capability-catalogue.mjs +12 -2
- package/lib/registry-comparison.mjs +28 -11
- package/lib/registry-insights.mjs +7 -0
- package/lib/response-policy.mjs +27 -4
- package/lib/risk-calibration-summary.mjs +340 -0
- package/lib/risk-scoring.mjs +4 -3
- package/lib/rpki-evidence.mjs +9 -8
- package/lib/safe-fetch.mjs +121 -50
- package/lib/security-txt.mjs +27 -4
- package/lib/service-binding-dns.mjs +6 -1
- package/lib/smtp-transport-review.mjs +787 -0
- package/lib/sslbl-certificates.generated.mjs +136 -134
- package/lib/sslbl-intelligence.mjs +6 -8
- package/lib/static-html-analysis.mjs +301 -4
- package/lib/structured-data-identity.mjs +51 -19
- package/lib/threat-intelligence-fixtures.mjs +2 -4
- package/lib/threat-intelligence-manifests.mjs +2 -7
- package/lib/threat-intelligence-runtime.mjs +5 -15
- package/lib/threat-intelligence-types.mjs +21 -1
- package/lib/threatfox-intelligence.mjs +7 -3
- package/lib/tls-intelligence.mjs +15 -9
- package/lib/tlsa-evidence.mjs +37 -7
- package/lib/urlhaus-intelligence.mjs +4 -2
- package/lib/urlscan-intelligence.mjs +6 -5
- package/lib/website-security-posture.mjs +1 -1
- package/lib/website-technology.mjs +1 -1
- package/lib/whois-authority.mjs +43 -8
- package/lib/whois-chain.mjs +25 -3
- package/lib/whois-normalization.mjs +3 -1
- package/lib/whois-transport.mjs +12 -3
- package/lib/zone-intent-review.mjs +1 -1
- package/package.json +12 -4
- package/packages/cases/case-brand-profile-references.mjs +85 -0
- package/{frontend/src/lib/analysis/case-evidence-model.js → packages/cases/case-evidence-model.mjs} +43 -14
- package/packages/cases/case-investigation-branch-model.mjs +132 -0
- package/packages/cases/case-migration-model.mjs +476 -0
- package/{frontend/src/lib/analysis/case-model.js → packages/cases/case-model.mjs} +3 -3
- package/{frontend/src/lib/analysis/case-record-contracts.js → packages/cases/case-record-contracts.mjs} +3 -19
- package/{frontend/src/lib/analysis/case-record-core.js → packages/cases/case-record-core.mjs} +58 -31
- package/packages/cases/case-record-model.mjs +9 -0
- package/{frontend/src/lib/analysis/case-record-operations.js → packages/cases/case-record-operations.mjs} +106 -33
- package/{frontend/src/lib/analysis/case-report.js → packages/cases/case-report.mjs} +143 -16
- package/packages/cases/case-response-model.mjs +1604 -0
- package/packages/cases/case-response-packet.mjs +1312 -0
- package/{frontend/src/lib/analysis/case-storage-model.js → packages/cases/case-storage-model.mjs} +2 -2
- package/{frontend/src/lib/analysis/http-summary.js → packages/cases/http-summary.mjs} +1 -1
- package/packages/cases/opaque-reference-id.mjs +9 -0
- package/packages/comparison/comparison-ledger-bulk.mjs +366 -0
- package/packages/comparison/comparison-ledger-contract.mjs +241 -0
- package/packages/comparison/comparison-ledger-json.mjs +80 -0
- package/packages/comparison/comparison-ledger-serialization.mjs +52 -0
- package/packages/comparison/favicon-similarity.mjs +78 -0
- package/{frontend/src/lib/analysis/page-similarity.js → packages/comparison/page-similarity.mjs} +4 -3
- package/{frontend/src/lib/analysis/relationship-evidence.js → packages/comparison/relationship-evidence.mjs} +6 -7
- package/packages/contracts/analyst-review-state-contract.mjs +61 -0
- package/packages/contracts/analyst-review-state.mjs +50 -0
- package/packages/contracts/case-portability.mjs +1310 -0
- package/packages/contracts/cli-command-catalogue.mjs +2 -0
- package/packages/contracts/cli-lookup.mjs +394 -0
- package/packages/contracts/domain-control-flight-recorder.mjs +461 -0
- package/packages/contracts/domain-control-manifest.mjs +1027 -0
- package/packages/contracts/domain-control-monitor.mjs +310 -0
- package/packages/contracts/domain-control-review.mjs +828 -0
- package/packages/contracts/domain-name.mjs +4 -0
- package/packages/contracts/external-observation-interchange.mjs +224 -0
- package/packages/contracts/extracted-domain-lifecycle.mjs +218 -0
- package/packages/contracts/investigation-portability.mjs +331 -0
- package/packages/contracts/offline-comparison.mjs +160 -0
- package/packages/contracts/risk-calibration.mjs +1288 -0
- package/packages/contracts/schema-compatibility.mjs +71 -0
- package/packages/contracts/schema-lifecycle.mjs +1702 -0
- package/packages/contracts/sslbl-snapshot.mjs +17 -0
- package/packages/contracts/workspace-portability.mjs +1347 -0
- package/packages/evidence/artifact-integrity.mjs +65 -0
- package/packages/evidence/decision-fact.mjs +735 -0
- package/packages/evidence/domain-control-runtime.mjs +547 -0
- package/packages/evidence/domain-name.mjs +41 -0
- package/packages/evidence/observation.mjs +384 -0
- package/{frontend/src/lib/analysis/external-findings-import.js → packages/interchange/external-findings-import.mjs} +86 -20
- package/packages/interchange/retained-text.mjs +8 -0
- package/{frontend/src/lib/analysis/acquisition-decision-packet.js → packages/investigation/acquisition-decision-packet.mjs} +11 -8
- package/{frontend/src/lib/analysis/bulk-domain-comparison.js → packages/investigation/bulk-domain-comparison.mjs} +28 -9
- package/{frontend/src/lib/analysis/bulk-mail-exposure.js → packages/investigation/bulk-mail-exposure.mjs} +67 -18
- package/{frontend/src/lib/analysis/bulk-review-export.js → packages/investigation/bulk-review-export.mjs} +10 -9
- package/packages/investigation/bulk-review-policy.mjs +1 -0
- package/{frontend/src/lib/analysis/investigation-capsule.js → packages/investigation/investigation-capsule.mjs} +50 -18
- package/packages/investigation/lookup-artefact-inputs.mjs +1 -0
- package/{frontend/src/lib/analysis/lookup-asset-graph.js → packages/investigation/lookup-asset-graph.mjs} +4 -2
- package/packages/investigation/lookup-claim-passport.mjs +232 -0
- package/packages/investigation/lookup-claim-readiness.mjs +287 -0
- package/packages/investigation/lookup-investigation-brief.mjs +172 -0
- package/packages/monitoring/analyst-review-state.mjs +680 -0
- package/{frontend/src/lib/analysis/brand-profile-model.js → packages/workspace/brand-profile-model.mjs} +97 -50
- package/{frontend/src/lib/analysis/bulk-review-model.js → packages/workspace/bulk-review-model.mjs} +15 -15
- package/{frontend/src/lib/analysis/bulk-session-model.js → packages/workspace/bulk-session-model.mjs} +179 -39
- package/{frontend/src/lib/analysis/bulk-sort.js → packages/workspace/bulk-sort.mjs} +12 -5
- package/{frontend/src/lib/analysis/campaign-model.js → packages/workspace/campaign-model.mjs} +20 -22
- package/{frontend/src/lib/analysis/detection-rule-model.js → packages/workspace/detection-rule-model.mjs} +15 -17
- package/packages/workspace/hostile-input.mjs +172 -0
- package/{frontend/src/lib/analysis/investigation-guide.js → packages/workspace/investigation-guide.mjs} +51 -46
- package/{frontend/src/lib/analysis/investigation-template-model.js → packages/workspace/investigation-template-model.mjs} +15 -15
- package/{frontend/src/lib/analysis/page-baseline.js → packages/workspace/page-baseline.mjs} +6 -13
- package/{frontend/src/lib/analysis/relationship-observation-model.js → packages/workspace/relationship-observation-model.mjs} +13 -17
- package/{frontend/src/lib/analysis/shortlist-model.js → packages/workspace/shortlist-model.mjs} +20 -18
- package/{frontend/src/lib/analysis/watchlist-history.js → packages/workspace/watchlist-history.mjs} +9 -16
- package/{frontend/src/lib/analysis/watchlist-store.js → packages/workspace/watchlist-store.mjs} +24 -17
- package/{frontend/src/lib/analysis/website-snapshot-model.js → packages/workspace/website-snapshot-model.mjs} +18 -12
- package/{frontend/src/lib/analysis/workspace-archive-crypto.js → packages/workspace/workspace-archive-crypto.mjs} +49 -19
- package/{frontend/src/lib/analysis/workspace-archive.js → packages/workspace/workspace-archive.mjs} +195 -84
- package/third-party-notices.txt +112 -5834
- package/frontend/src/lib/analysis/artifact-integrity.js +0 -21
- package/frontend/src/lib/analysis/bulk-retry-plan.js +0 -79
- package/frontend/src/lib/analysis/bulk-source-coverage.js +0 -76
- package/frontend/src/lib/analysis/case-migration-model.js +0 -253
- package/frontend/src/lib/analysis/case-record-model.js +0 -7
- package/frontend/src/lib/analysis/case-response-model.js +0 -669
- package/frontend/src/lib/analysis/case-response-packet.js +0 -573
- package/frontend/src/lib/analysis/lookup-decision-support.js +0 -487
- package/frontend/src/lib/analysis/lookup-investigation-brief.js +0 -153
- package/frontend/src/lib/analysis/lookup-presentation.js +0 -58
- package/frontend/src/lib/analysis/lookup-response.js +0 -2
- package/frontend/src/lib/analysis/lookup-source-refresh.js +0 -277
- package/frontend/src/lib/analysis/lookup-summary-model.js +0 -398
- package/frontend/src/lib/analysis/scoring.js +0 -55
- package/frontend/src/lib/analysis/utils.js +0 -232
- package/frontend/src/lib/analysis/visualization-bounds.js +0 -26
- package/frontend/src/lib/analysis/visualization-force-graph.js +0 -352
- package/frontend/src/lib/analysis/visualization-matrix.js +0 -88
- package/frontend/src/lib/analysis/visualization-models.js +0 -369
- package/frontend/src/lib/analysis/visualization-time-series.js +0 -144
- package/lib/bounded-json-response.mjs +0 -157
- package/lib/lookup-response-contract.mjs +0 -346
- package/lib/observation.mjs +0 -114
- /package/{frontend/src/lib/analysis/acquisition-due-diligence.js → packages/investigation/acquisition-due-diligence.mjs} +0 -0
- /package/{frontend/src/lib/analysis/evidence-coverage-ledger.js → packages/investigation/evidence-coverage-ledger.mjs} +0 -0
- /package/{frontend/src/lib/analysis/dns-record-normalization.js → packages/workspace/dns-record-normalization.mjs} +0 -0
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
// Small environment-neutral integrity helper for deliberate local exports.
|
|
2
|
-
// Inputs must already be normalized and bounded by the owning artifact model.
|
|
3
|
-
export function canonicalArtifactJson(value) {
|
|
4
|
-
if (value === undefined)
|
|
5
|
-
return 'null';
|
|
6
|
-
if (value === null || typeof value !== 'object')
|
|
7
|
-
return JSON.stringify(value);
|
|
8
|
-
if (Array.isArray(value))
|
|
9
|
-
return `[${value.map(canonicalArtifactJson).join(',')}]`;
|
|
10
|
-
const entries = Object.entries(value)
|
|
11
|
-
.filter(([, entry]) => entry !== undefined)
|
|
12
|
-
.sort(([left], [right]) => left.localeCompare(right));
|
|
13
|
-
return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalArtifactJson(entry)}`).join(',')}}`;
|
|
14
|
-
}
|
|
15
|
-
export async function sha256ArtifactDigest(value) {
|
|
16
|
-
const bytes = new TextEncoder().encode(canonicalArtifactJson(value));
|
|
17
|
-
const digest = await globalThis.crypto.subtle.digest('SHA-256', bytes);
|
|
18
|
-
return `sha256:${[...new Uint8Array(digest)]
|
|
19
|
-
.map((byte) => byte.toString(16).padStart(2, '0'))
|
|
20
|
-
.join('')}`;
|
|
21
|
-
}
|
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
import { normalizeBulkSessionResult, } from "./bulk-session-model.js";
|
|
2
|
-
import { limitedBulkSources } from "./bulk-source-coverage.js";
|
|
3
|
-
export const MAX_BULK_RETRY_ROWS = 200;
|
|
4
|
-
export const BULK_REVIEW_STALE_AFTER_DAYS = 7;
|
|
5
|
-
function freshness(value, now = Date.now()) {
|
|
6
|
-
if (typeof value !== 'string' || !Number.isFinite(Date.parse(value))) {
|
|
7
|
-
return { observedAt: null, ageDays: null, state: 'unknown' };
|
|
8
|
-
}
|
|
9
|
-
const observedAt = new Date(value).toISOString();
|
|
10
|
-
const ageDays = Math.max(0, Math.floor((now - Date.parse(observedAt)) / 86_400_000));
|
|
11
|
-
return {
|
|
12
|
-
observedAt,
|
|
13
|
-
ageDays,
|
|
14
|
-
state: ageDays >= BULK_REVIEW_STALE_AFTER_DAYS ? 'stale' : 'current',
|
|
15
|
-
};
|
|
16
|
-
}
|
|
17
|
-
function retryRow(value, isStale) {
|
|
18
|
-
const limitedSources = limitedBulkSources(value.domain, value.sourceCoverage)
|
|
19
|
-
.sort()
|
|
20
|
-
.slice(0, 12);
|
|
21
|
-
const reasons = [];
|
|
22
|
-
if (value.status === 'error')
|
|
23
|
-
reasons.push('error');
|
|
24
|
-
if (limitedSources.length)
|
|
25
|
-
reasons.push('limited_source');
|
|
26
|
-
if (isStale)
|
|
27
|
-
reasons.push('stale');
|
|
28
|
-
return reasons.length ? { domain: value.domain, reasons, limitedSources } : null;
|
|
29
|
-
}
|
|
30
|
-
export function buildBulkRetryPlan(values, mode, observedAtRaw = null, now = Date.now()) {
|
|
31
|
-
const observationFreshness = freshness(observedAtRaw, now);
|
|
32
|
-
const candidates = values
|
|
33
|
-
.map(normalizeBulkSessionResult)
|
|
34
|
-
.filter((value) => Boolean(value))
|
|
35
|
-
.map((value) => retryRow(value, observationFreshness.state === 'stale'))
|
|
36
|
-
.filter((value) => Boolean(value));
|
|
37
|
-
const rows = candidates.slice(0, MAX_BULK_RETRY_ROWS);
|
|
38
|
-
return {
|
|
39
|
-
mode,
|
|
40
|
-
lookupRequests: rows.length,
|
|
41
|
-
rows,
|
|
42
|
-
destinations: mode === 'deep'
|
|
43
|
-
? ['registry RDAP', 'registrar RDAP when advertised', 'WHOIS', 'DNS resolvers', 'target HTTP(S)', 'target TLS']
|
|
44
|
-
: ['registry RDAP', 'authoritative DNS delegation when registration sources are inconclusive'],
|
|
45
|
-
capped: candidates.length > rows.length,
|
|
46
|
-
freshness: observationFreshness,
|
|
47
|
-
limitations: [
|
|
48
|
-
'This repeats the selected lookup profile; the current API does not request only one upstream source.',
|
|
49
|
-
`Evidence is marked stale after ${BULK_REVIEW_STALE_AFTER_DAYS} days for review purposes; this is a local observation age, not an upstream record age.`,
|
|
50
|
-
'Optional third-party intelligence providers are not contacted by Bulk.',
|
|
51
|
-
'Sources deliberately skipped by the selected scan profile or deployment policy do not independently trigger a retry.',
|
|
52
|
-
'A retry can remain partial or fail and does not convert unavailable evidence into absence.',
|
|
53
|
-
],
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
export function preservePriorBulkResult(previousRaw, currentRaw) {
|
|
57
|
-
const previous = normalizeBulkSessionResult(previousRaw);
|
|
58
|
-
const current = normalizeBulkSessionResult(currentRaw);
|
|
59
|
-
if (!previous || !current || previous.domain !== current.domain || previous.status !== 'complete') {
|
|
60
|
-
return { preserve: false, reason: '' };
|
|
61
|
-
}
|
|
62
|
-
if (current.status === 'error') {
|
|
63
|
-
return { preserve: true, reason: 'The retry failed, so the prior settled result was retained.' };
|
|
64
|
-
}
|
|
65
|
-
if (previous.scanDepth === 'deep' && current.scanDepth === 'fast') {
|
|
66
|
-
return { preserve: true, reason: 'The retry returned only Fast evidence, so the prior Deep result was retained.' };
|
|
67
|
-
}
|
|
68
|
-
const currentSources = new Map(current.sourceCoverage.map((item) => [item.source, item.state]));
|
|
69
|
-
const regressed = previous.sourceCoverage
|
|
70
|
-
.filter((item) => item.state === 'complete')
|
|
71
|
-
.map((item) => item.source)
|
|
72
|
-
.filter((source) => currentSources.get(source) !== 'complete');
|
|
73
|
-
return regressed.length
|
|
74
|
-
? {
|
|
75
|
-
preserve: true,
|
|
76
|
-
reason: `The retry reduced complete coverage for ${regressed.slice(0, 6).join(', ')}, so the prior settled result was retained.`,
|
|
77
|
-
}
|
|
78
|
-
: { preserve: false, reason: '' };
|
|
79
|
-
}
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
// Authority-aware interpretation for compact Bulk source observations. Raw
|
|
2
|
-
// source states remain unchanged; this layer only decides whether an observed
|
|
3
|
-
// limitation was unexpected for the registry's published service paths.
|
|
4
|
-
import { registryCapabilityFor } from "../../../../lib/registry-capabilities.mjs";
|
|
5
|
-
const MAX_SOURCE_OBSERVATIONS = 32;
|
|
6
|
-
const MAX_SOURCE_TEXT_LENGTH = 40;
|
|
7
|
-
const CONTROL_RE = /[\u0000-\u001f\u007f]/u;
|
|
8
|
-
const LIMITED_SOURCE_STATES = new Set([
|
|
9
|
-
'error',
|
|
10
|
-
'failed',
|
|
11
|
-
'inconclusive',
|
|
12
|
-
'partial',
|
|
13
|
-
'rate_limited',
|
|
14
|
-
'unavailable',
|
|
15
|
-
'unsupported',
|
|
16
|
-
]);
|
|
17
|
-
function text(value) {
|
|
18
|
-
return typeof value === 'string'
|
|
19
|
-
&& value.length <= MAX_SOURCE_TEXT_LENGTH
|
|
20
|
-
&& !CONTROL_RE.test(value)
|
|
21
|
-
? value.trim().toLowerCase()
|
|
22
|
-
: '';
|
|
23
|
-
}
|
|
24
|
-
function record(value) {
|
|
25
|
-
return value && typeof value === 'object' && !Array.isArray(value)
|
|
26
|
-
? value
|
|
27
|
-
: null;
|
|
28
|
-
}
|
|
29
|
-
function observations(value) {
|
|
30
|
-
return Array.isArray(value)
|
|
31
|
-
? value.slice(0, MAX_SOURCE_OBSERVATIONS).flatMap((entry) => {
|
|
32
|
-
const item = record(entry);
|
|
33
|
-
const source = text(item?.source);
|
|
34
|
-
const state = text(item?.state);
|
|
35
|
-
return source && state ? [{ source, state }] : [];
|
|
36
|
-
})
|
|
37
|
-
: [];
|
|
38
|
-
}
|
|
39
|
-
function expectedUnsupportedSources(domain) {
|
|
40
|
-
const capability = registryCapabilityFor(domain);
|
|
41
|
-
if (!capability)
|
|
42
|
-
return new Set();
|
|
43
|
-
const expected = new Set();
|
|
44
|
-
if (capability.rdapAccessProfile === 'no-iana-service')
|
|
45
|
-
expected.add('rdap');
|
|
46
|
-
if (capability.whoisAccessProfile === 'no-iana-service')
|
|
47
|
-
expected.add('whois');
|
|
48
|
-
return expected;
|
|
49
|
-
}
|
|
50
|
-
export function isExpectedUnsupportedBulkSource(domain, observation) {
|
|
51
|
-
const source = text(observation.source);
|
|
52
|
-
const state = text(observation.state);
|
|
53
|
-
return state === 'unsupported' && expectedUnsupportedSources(domain).has(source);
|
|
54
|
-
}
|
|
55
|
-
export function limitedBulkSources(domain, coverage) {
|
|
56
|
-
const expectedUnsupported = expectedUnsupportedSources(domain);
|
|
57
|
-
return observations(coverage)
|
|
58
|
-
.filter(({ source, state }) => (LIMITED_SOURCE_STATES.has(state)
|
|
59
|
-
&& !(state === 'unsupported' && expectedUnsupported.has(source))))
|
|
60
|
-
.map(({ source }) => source);
|
|
61
|
-
}
|
|
62
|
-
export function classifyBulkSourceCoverage(domain, coverage) {
|
|
63
|
-
const retained = observations(coverage);
|
|
64
|
-
if (!retained.length)
|
|
65
|
-
return 'unrecorded';
|
|
66
|
-
return limitedBulkSources(domain, retained).length ? 'limited' : 'complete';
|
|
67
|
-
}
|
|
68
|
-
export function describeBulkSourceCoverage(domain, observation) {
|
|
69
|
-
const source = text(observation.source);
|
|
70
|
-
const state = text(observation.state);
|
|
71
|
-
if (!source || !state)
|
|
72
|
-
return 'unrecorded source';
|
|
73
|
-
return isExpectedUnsupportedBulkSource(domain, observation)
|
|
74
|
-
? `${source}: unsupported (no IANA-published service)`
|
|
75
|
-
: `${source}: ${state}`;
|
|
76
|
-
}
|
|
@@ -1,253 +0,0 @@
|
|
|
1
|
-
import { CASE_DISPOSITIONS, CASE_IMPORT_VERSIONS, CASE_SCHEMA_VERSION, CASE_SOURCES, CASE_STATUSES, DEFAULT_DISPOSITION, DEFAULT_SOURCE, DEFAULT_STATUS, MAX_CASES, MAX_NOTES_PER_CASE, deterministicId, isoOrNull, normalizeCase, normalizeDomain, normalizeEvidenceHistory, normalizeNotes, normalizeReviewReasonCode, normalizeTags, objectRecord, safeId, } from "./case-record-model.js";
|
|
2
|
-
import { mergeCaseActions, mergeCaseAssertions, mergeCaseDecisions, mergeCaseEvidencePins, mergeCaseManualTrail, mergeCaseSightings, normalizeCaseActions, normalizeCaseAssertions, normalizeCaseDecisions, normalizeCaseEvidencePins, normalizeCaseManualTrail, normalizeCaseSightings, } from "./case-response-model.js";
|
|
3
|
-
const STATUS_VALUES = new Set(CASE_STATUSES.map((item) => item.value));
|
|
4
|
-
const DISPOSITION_VALUES = new Set(CASE_DISPOSITIONS.map((item) => item.value));
|
|
5
|
-
const SOURCE_VALUES = new Set(CASE_SOURCES.map((item) => item.value));
|
|
6
|
-
function assignUniqueIds(cases) {
|
|
7
|
-
const used = new Set();
|
|
8
|
-
for (const record of [...cases].sort((a, b) => a.domain.localeCompare(b.domain))) {
|
|
9
|
-
let id = safeId(record.id) || deterministicId(record.domain);
|
|
10
|
-
if (used.has(id)) {
|
|
11
|
-
const base = deterministicId(record.domain);
|
|
12
|
-
id = base;
|
|
13
|
-
let suffix = 2;
|
|
14
|
-
while (used.has(id))
|
|
15
|
-
id = `${base}-${suffix++}`;
|
|
16
|
-
}
|
|
17
|
-
used.add(id);
|
|
18
|
-
record.id = id;
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* Recovers a clean, bounded store from an arbitrary parsed value. Accepts the
|
|
23
|
-
* versioned envelope or a bare array, drops malformed records, keeps a single
|
|
24
|
-
* case per domain (most recently updated wins), caps to MAX_CASES by recency,
|
|
25
|
-
* and guarantees globally unique safe ids. Never throws.
|
|
26
|
-
* @param {unknown} raw
|
|
27
|
-
* @returns {CaseStore}
|
|
28
|
-
*/
|
|
29
|
-
export function normalizeCaseStore(raw) {
|
|
30
|
-
const now = new Date().toISOString();
|
|
31
|
-
const byDomain = new Map();
|
|
32
|
-
for (const item of asCaseList(raw)) {
|
|
33
|
-
const normalized = normalizeCase(item, undefined, now);
|
|
34
|
-
if (!normalized)
|
|
35
|
-
continue;
|
|
36
|
-
const existing = byDomain.get(normalized.domain);
|
|
37
|
-
if (!existing || Date.parse(normalized.updatedAt) >= Date.parse(existing.updatedAt)) {
|
|
38
|
-
byDomain.set(normalized.domain, normalized);
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
const cases = [...byDomain.values()]
|
|
42
|
-
.sort((a, b) => Date.parse(b.updatedAt) - Date.parse(a.updatedAt))
|
|
43
|
-
.slice(0, MAX_CASES);
|
|
44
|
-
assignUniqueIds(cases);
|
|
45
|
-
return { version: CASE_SCHEMA_VERSION, cases };
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* The schema version declared by a stored/parsed value, or null. The storage
|
|
49
|
-
* wrapper uses this to refuse overwriting data written by a newer, unsupported
|
|
50
|
-
* version instead of silently downgrading it.
|
|
51
|
-
* @param {unknown} raw
|
|
52
|
-
* @returns {number | null}
|
|
53
|
-
*/
|
|
54
|
-
export function parseStoreVersion(raw) {
|
|
55
|
-
const value = objectRecord(raw).version;
|
|
56
|
-
return typeof value === 'number' ? value : null;
|
|
57
|
-
}
|
|
58
|
-
/** @param {unknown} raw @returns {unknown[]} */
|
|
59
|
-
function asCaseList(raw) {
|
|
60
|
-
if (Array.isArray(raw))
|
|
61
|
-
return raw;
|
|
62
|
-
const cases = objectRecord(raw).cases;
|
|
63
|
-
if (Array.isArray(cases))
|
|
64
|
-
return cases;
|
|
65
|
-
return [];
|
|
66
|
-
}
|
|
67
|
-
/**
|
|
68
|
-
* @param {{ domain: unknown, status?: unknown, disposition?: unknown, source?: unknown, tags?: unknown, evidence?: unknown, note?: unknown }} input
|
|
69
|
-
* @param {string} [nowIso]
|
|
70
|
-
* @returns {CaseRecord}
|
|
71
|
-
*/
|
|
72
|
-
function importScalar(value, valid) {
|
|
73
|
-
return typeof value === 'string' && valid.has(value) ? value : undefined;
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* Validates one imported record into a patch. Unlike normalizeCase, absent or
|
|
77
|
-
* invalid scalar fields stay `undefined` (never defaulted) and a missing/invalid
|
|
78
|
-
* updatedAt stays `null` (treated as older than any real local timestamp), so an
|
|
79
|
-
* incomplete import can never win a merge over valid local data. Imported
|
|
80
|
-
* evidence is normalized additively; a snapshot with no captured time falls back
|
|
81
|
-
* only to the imported record's own (older) timestamps, never to "now".
|
|
82
|
-
* @param {unknown} raw
|
|
83
|
-
* @returns {ImportPatch | null}
|
|
84
|
-
*/
|
|
85
|
-
function extractImportPatch(raw) {
|
|
86
|
-
const record = objectRecord(raw);
|
|
87
|
-
const domain = normalizeDomain(record.domain);
|
|
88
|
-
if (!domain)
|
|
89
|
-
return null;
|
|
90
|
-
const importFallback = isoOrNull(record.updatedAt) || isoOrNull(record.createdAt) || null;
|
|
91
|
-
const normalizedFallback = importFallback || '1970-01-01T00:00:00.000Z';
|
|
92
|
-
const rawEvidence = Array.isArray(record.evidenceHistory) ? record.evidenceHistory : [];
|
|
93
|
-
const evidencePins = normalizeCaseEvidencePins(record.evidencePins, normalizedFallback);
|
|
94
|
-
const pinIds = new Set(evidencePins.map((item) => item.id));
|
|
95
|
-
return {
|
|
96
|
-
domain,
|
|
97
|
-
rawId: typeof record.id === 'string' ? record.id : null,
|
|
98
|
-
status: importScalar(record.status, STATUS_VALUES),
|
|
99
|
-
disposition: importScalar(record.disposition, DISPOSITION_VALUES),
|
|
100
|
-
reviewReasonCode: Object.hasOwn(record, 'reviewReasonCode')
|
|
101
|
-
? normalizeReviewReasonCode(record.reviewReasonCode)
|
|
102
|
-
: undefined,
|
|
103
|
-
source: importScalar(record.source, SOURCE_VALUES),
|
|
104
|
-
evidenceHistory: normalizeEvidenceHistory(rawEvidence, { source: 'import', fallback: importFallback }),
|
|
105
|
-
evidencePins,
|
|
106
|
-
decisions: normalizeCaseDecisions(record.decisions, normalizedFallback, pinIds),
|
|
107
|
-
actions: normalizeCaseActions(record.actions, normalizedFallback),
|
|
108
|
-
assertions: normalizeCaseAssertions(record.assertions, normalizedFallback, pinIds),
|
|
109
|
-
manualTrail: normalizeCaseManualTrail(record.manualTrail, normalizedFallback),
|
|
110
|
-
sightings: normalizeCaseSightings(record.sightings, normalizedFallback, pinIds),
|
|
111
|
-
tags: normalizeTags(record.tags),
|
|
112
|
-
// Imported notes fall back only to the imported record's own timestamps
|
|
113
|
-
// (never "now"), so a timestamp-less note gets a stable, deterministic time
|
|
114
|
-
// and id, and re-importing the same file cannot manufacture a duplicate or a
|
|
115
|
-
// spuriously-newer note.
|
|
116
|
-
notes: normalizeNotes(record.notes, importFallback),
|
|
117
|
-
createdAt: isoOrNull(record.createdAt),
|
|
118
|
-
updatedAt: isoOrNull(record.updatedAt),
|
|
119
|
-
};
|
|
120
|
-
}
|
|
121
|
-
/** @param {CaseNote[]} a @param {CaseNote[]} b @returns {CaseNote[]} */
|
|
122
|
-
function unionNotes(a, b) {
|
|
123
|
-
const byId = new Map();
|
|
124
|
-
for (const note of [...a, ...b]) {
|
|
125
|
-
if (!byId.has(note.id))
|
|
126
|
-
byId.set(note.id, note);
|
|
127
|
-
}
|
|
128
|
-
const notes = [...byId.values()].sort((left, right) => Date.parse(left.createdAt) - Date.parse(right.createdAt));
|
|
129
|
-
return notes.slice(Math.max(0, notes.length - MAX_NOTES_PER_CASE));
|
|
130
|
-
}
|
|
131
|
-
// Additive, deduplicated union of two evidence histories. Identical material
|
|
132
|
-
// collapses (earliest firstCapturedAt, latest capturedAt), distinct snapshots
|
|
133
|
-
// are retained subject to the per-case bound, and an older import can never
|
|
134
|
-
// move an existing observation backwards.
|
|
135
|
-
function mergeEvidenceHistories(local, imported) {
|
|
136
|
-
return normalizeEvidenceHistory([...local, ...imported], { source: 'import', fallback: null });
|
|
137
|
-
}
|
|
138
|
-
/** @param {ImportPatch} patch @param {string} now @returns {CaseRecord} */
|
|
139
|
-
function caseFromPatch(patch, now) {
|
|
140
|
-
return {
|
|
141
|
-
id: '', // assigned by mergeCases so it can guarantee uniqueness against locals
|
|
142
|
-
domain: patch.domain,
|
|
143
|
-
status: patch.status ?? DEFAULT_STATUS,
|
|
144
|
-
disposition: patch.disposition ?? DEFAULT_DISPOSITION,
|
|
145
|
-
reviewReasonCode: patch.reviewReasonCode ?? null,
|
|
146
|
-
tags: patch.tags,
|
|
147
|
-
notes: patch.notes,
|
|
148
|
-
source: patch.source ?? DEFAULT_SOURCE,
|
|
149
|
-
evidenceHistory: patch.evidenceHistory,
|
|
150
|
-
evidencePins: patch.evidencePins,
|
|
151
|
-
decisions: patch.decisions,
|
|
152
|
-
actions: patch.actions,
|
|
153
|
-
assertions: patch.assertions,
|
|
154
|
-
manualTrail: patch.manualTrail,
|
|
155
|
-
sightings: patch.sightings,
|
|
156
|
-
createdAt: patch.createdAt || patch.updatedAt || now,
|
|
157
|
-
updatedAt: patch.updatedAt || patch.createdAt || now,
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
/**
|
|
161
|
-
* Merges an imported patch into an existing local case. Notes, tags, and
|
|
162
|
-
* evidence history are merged additively (never destructive); a scalar field is
|
|
163
|
-
* only overwritten when the import provided a valid value AND is strictly newer
|
|
164
|
-
* than the local record. A patch with no/invalid updatedAt is never newer.
|
|
165
|
-
* @param {CaseRecord} local
|
|
166
|
-
* @param {ImportPatch} patch
|
|
167
|
-
* @returns {CaseRecord}
|
|
168
|
-
*/
|
|
169
|
-
function applyImportPatch(local, patch) {
|
|
170
|
-
const importNewer = patch.updatedAt !== null && Date.parse(patch.updatedAt) > Date.parse(local.updatedAt);
|
|
171
|
-
const fallback = patch.updatedAt || local.updatedAt;
|
|
172
|
-
const evidencePins = mergeCaseEvidencePins(local.evidencePins, patch.evidencePins, fallback);
|
|
173
|
-
const pinIds = new Set(evidencePins.map((item) => item.id));
|
|
174
|
-
return {
|
|
175
|
-
...local,
|
|
176
|
-
status: patch.status !== undefined && importNewer ? patch.status : local.status,
|
|
177
|
-
disposition: patch.disposition !== undefined && importNewer ? patch.disposition : local.disposition,
|
|
178
|
-
reviewReasonCode: patch.reviewReasonCode !== undefined && importNewer ? patch.reviewReasonCode : local.reviewReasonCode ?? null,
|
|
179
|
-
source: patch.source !== undefined && importNewer ? patch.source : local.source,
|
|
180
|
-
evidenceHistory: mergeEvidenceHistories(local.evidenceHistory, patch.evidenceHistory),
|
|
181
|
-
evidencePins,
|
|
182
|
-
decisions: mergeCaseDecisions(local.decisions, patch.decisions, fallback, pinIds),
|
|
183
|
-
actions: mergeCaseActions(local.actions, patch.actions, fallback),
|
|
184
|
-
assertions: mergeCaseAssertions(local.assertions, patch.assertions, fallback, pinIds),
|
|
185
|
-
manualTrail: mergeCaseManualTrail(local.manualTrail, patch.manualTrail, fallback),
|
|
186
|
-
sightings: mergeCaseSightings(local.sightings, patch.sightings, fallback, pinIds),
|
|
187
|
-
tags: normalizeTags([...local.tags, ...patch.tags]),
|
|
188
|
-
notes: unionNotes(local.notes, patch.notes),
|
|
189
|
-
createdAt: patch.createdAt && Date.parse(patch.createdAt) < Date.parse(local.createdAt) ? patch.createdAt : local.createdAt,
|
|
190
|
-
updatedAt: importNewer ? (patch.updatedAt ?? local.updatedAt) : local.updatedAt,
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
function pickFreeId(preferred, domain, used) {
|
|
194
|
-
const wanted = safeId(preferred);
|
|
195
|
-
let id = wanted && !used.has(wanted) ? wanted : deterministicId(domain);
|
|
196
|
-
const base = deterministicId(domain);
|
|
197
|
-
let suffix = 2;
|
|
198
|
-
while (used.has(id))
|
|
199
|
-
id = `${base}-${suffix++}`;
|
|
200
|
-
return id;
|
|
201
|
-
}
|
|
202
|
-
/**
|
|
203
|
-
* Merges an imported (already parsed) value into the local cases. Predictable
|
|
204
|
-
* and idempotent: unknown records are skipped, existing domains merge without
|
|
205
|
-
* losing newer local decisions, and new ones are added until the store bound is
|
|
206
|
-
* reached. An imported envelope that declares a schema version newer than we
|
|
207
|
-
* support is rejected up front (before any local data is touched) rather than
|
|
208
|
-
* reinterpreted.
|
|
209
|
-
* @param {CaseRecord[]} localCases
|
|
210
|
-
* @param {unknown} importedRaw
|
|
211
|
-
* @returns {{ cases: CaseRecord[], added: number, updated: number, skipped: number }}
|
|
212
|
-
*/
|
|
213
|
-
export function mergeCases(localCases, importedRaw) {
|
|
214
|
-
const importedVersion = parseStoreVersion(importedRaw);
|
|
215
|
-
if (importedVersion !== null && Number.isInteger(importedVersion) && importedVersion > CASE_SCHEMA_VERSION) {
|
|
216
|
-
throw new Error(`This case file was exported by a newer version of WHOISleuth (schema ${importedVersion}). Update the app before importing it.`);
|
|
217
|
-
}
|
|
218
|
-
if (!CASE_IMPORT_VERSIONS.includes(importedVersion)
|
|
219
|
-
|| !importedRaw || typeof importedRaw !== 'object'
|
|
220
|
-
|| !Array.isArray(objectRecord(importedRaw).cases)) {
|
|
221
|
-
throw new Error(`Expected a WHOISleuth case export using schema ${CASE_IMPORT_VERSIONS.join(' or ')}.`);
|
|
222
|
-
}
|
|
223
|
-
const local = normalizeCaseStore(localCases).cases;
|
|
224
|
-
const byDomain = new Map(local.map((item) => [item.domain, item]));
|
|
225
|
-
const usedIds = new Set(local.map((item) => item.id));
|
|
226
|
-
let added = 0;
|
|
227
|
-
let updated = 0;
|
|
228
|
-
let skipped = 0;
|
|
229
|
-
const now = new Date().toISOString();
|
|
230
|
-
for (const item of asCaseList(importedRaw)) {
|
|
231
|
-
const patch = extractImportPatch(item);
|
|
232
|
-
if (!patch) {
|
|
233
|
-
skipped += 1;
|
|
234
|
-
continue;
|
|
235
|
-
}
|
|
236
|
-
const existing = byDomain.get(patch.domain);
|
|
237
|
-
if (existing) {
|
|
238
|
-
byDomain.set(patch.domain, applyImportPatch(existing, patch));
|
|
239
|
-
updated += 1;
|
|
240
|
-
}
|
|
241
|
-
else if (byDomain.size < MAX_CASES) {
|
|
242
|
-
const record = caseFromPatch(patch, now);
|
|
243
|
-
record.id = pickFreeId(patch.rawId, patch.domain, usedIds);
|
|
244
|
-
usedIds.add(record.id);
|
|
245
|
-
byDomain.set(patch.domain, record);
|
|
246
|
-
added += 1;
|
|
247
|
-
}
|
|
248
|
-
else {
|
|
249
|
-
skipped += 1;
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
return { cases: normalizeCaseStore([...byDomain.values()]).cases, added, updated, skipped };
|
|
253
|
-
}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
// Stable case-record facade. Core validation, evidence history/comparison,
|
|
2
|
-
// and analyst-record operations remain framework-neutral and independently
|
|
3
|
-
// testable while existing consumers keep one compatibility import.
|
|
4
|
-
export * from "./case-record-contracts.js";
|
|
5
|
-
export * from "./case-record-core.js";
|
|
6
|
-
export * from "./case-evidence-model.js";
|
|
7
|
-
export * from "./case-record-operations.js";
|