@slicedearth/whoisleuth-cli 1.45.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +7 -3
- package/NOTICE +5 -4
- package/README.md +74 -124
- package/SECURITY.md +6 -3
- package/bin/whoisleuth.mjs +7 -4
- package/cli/archive-inspect.mjs +28 -10
- package/cli/arguments.mjs +450 -173
- package/cli/artifact-structure.mjs +2507 -0
- package/cli/artifact-verify.mjs +286 -64
- package/cli/bounded-json.mjs +1 -0
- package/cli/bulk-checkpoint.mjs +35 -23
- package/cli/bulk-command-runner.mjs +2 -5
- package/cli/bulk.mjs +9 -2
- package/cli/case-pack.mjs +435 -46
- package/cli/ci-report.mjs +6 -1
- package/cli/collection-preflight.mjs +2 -2
- package/cli/command-catalogue.mjs +12 -3
- package/cli/command-reference.mjs +948 -106
- package/cli/completion.mjs +504 -218
- package/cli/config-profile.mjs +13 -3
- package/cli/ct-event-intake.mjs +32 -2
- package/cli/discover.mjs +7 -1
- package/cli/discovery-observation-snapshot.mjs +24 -28
- package/cli/discovery-scan-command-runner.mjs +13 -14
- package/cli/discovery-scan.mjs +17 -3
- package/cli/discovery-snapshot.mjs +15 -7
- package/cli/discovery-workflow.mjs +2 -3
- package/cli/doctor.mjs +2 -2
- package/cli/domain-control-monitor.mjs +215 -36
- package/cli/domain-control-observations.mjs +34 -17
- package/cli/errors.mjs +8 -3
- package/cli/evidence-command-runner.mjs +17 -11
- package/cli/evidence-signing.mjs +68 -50
- package/cli/export-evidence.mjs +11 -4
- package/cli/external-observation-mapping.mjs +3 -3
- package/cli/formatters/html.mjs +3 -3
- package/cli/formatters/json.mjs +20 -32
- package/cli/formatters/terminal.mjs +586 -22
- package/cli/input.mjs +62 -0
- package/cli/interactive-launcher.mjs +226 -0
- package/cli/interchange-report.mjs +226 -0
- package/cli/investigation-manifest.mjs +16 -17
- package/cli/investigation-plan.mjs +167 -16
- package/cli/investigation-run.mjs +80 -24
- package/cli/local-mmdb-review.mjs +45 -14
- package/cli/lookalike-calibration.mjs +3 -1
- package/cli/lookup-brief.mjs +13 -8
- package/cli/lookup-browser.mjs +844 -0
- package/cli/lookup-command-runner.mjs +96 -33
- package/cli/lookup-diff.mjs +11 -2
- package/cli/lookup-plan.mjs +1 -0
- package/cli/mail-review.mjs +206 -94
- package/cli/manual.mjs +5 -5
- package/cli/network-command-runner.mjs +78 -0
- package/cli/offline-evidence-review.mjs +45 -14
- package/cli/open-asset-model-bridge.mjs +1 -1
- package/cli/output-file.mjs +70 -14
- package/cli/page-compare.mjs +18 -10
- package/cli/progress-events.mjs +2 -1
- package/cli/progress.mjs +3 -3
- package/cli/registry-cohort.mjs +343 -61
- package/cli/registry-doctor.mjs +67 -28
- package/cli/registry-support.mjs +2 -1
- package/cli/retained-artifact-diff.mjs +375 -0
- package/cli/risk-calibration.mjs +89 -40
- package/cli/runner.mjs +322 -246
- package/cli/saved-lookup.mjs +439 -49
- package/cli/sharing-review.mjs +18 -7
- package/cli/source-reliability.mjs +75 -36
- package/cli/terminal-presentation.mjs +43 -18
- package/docs/cli-reference.md +125 -1334
- package/docs/cli.md +87 -236
- package/frontend/src/lib/analysis/domain-control-manifest-core.js +3 -0
- package/frontend/src/lib/analysis/domain-control-records.js +3 -0
- package/lib/application-version.mjs +3 -0
- package/lib/availability.mjs +72 -52
- package/lib/bounded-contract-normalizers.mjs +2 -4
- package/lib/bounded-file.mjs +91 -4
- package/lib/bounded-json.mjs +287 -0
- package/lib/browser-library-profile.mjs +1 -1
- package/lib/certificate-extension-profile.mjs +2 -1
- package/lib/classify.mjs +56 -5
- package/lib/client-behavior-profile.mjs +1 -1
- package/lib/credential-surface-profile.mjs +1 -1
- package/lib/cryptographic-assurance.mjs +173 -0
- package/lib/ct-query.mjs +3 -3
- package/lib/ct-search.mjs +120 -64
- package/lib/distributed-operation-budget.mjs +4 -2
- package/lib/dns-delegation-health.mjs +9 -6
- package/lib/dns-intelligence.mjs +4 -3
- package/lib/dns-resolver-selection.mjs +6 -4
- package/lib/dnssec-chain-validation.mjs +1052 -0
- package/lib/dnssec-evidence-validation.mjs +23 -10
- package/lib/domain-assurance.mjs +12 -12
- package/lib/domain-change-packet.mjs +7 -5
- package/lib/domain-control-flight-recorder.mjs +442 -78
- package/lib/domain-control-manifest.mjs +314 -171
- package/lib/domain-posture-analysis.mjs +5 -2
- package/lib/domain-posture.mjs +7 -4
- package/lib/encrypted-dns-contract.mjs +2 -3
- package/lib/evidence-export.mjs +1243 -88
- package/lib/evidence-report-markdown.mjs +3 -0
- package/lib/evidence-report.mjs +5 -0
- package/lib/external-intelligence-risk.mjs +26 -10
- package/lib/favicon.mjs +8 -5
- package/lib/feature-policy.mjs +1 -1
- package/lib/homepage-metadata-contract.mjs +290 -0
- package/lib/hostname.mjs +1 -1
- package/lib/html-signals.mjs +89 -5
- package/lib/http-evidence-bounds.mjs +3 -0
- package/lib/http-intelligence.mjs +332 -6
- package/lib/interchange-fidelity-registry.mjs +124 -0
- package/lib/local-geoip-evidence.mjs +2 -1
- package/lib/lookup-cache.mjs +6 -4
- package/lib/lookup-network-evidence-bounds.mjs +9 -0
- package/lib/lookup.mjs +27 -4
- package/lib/observed-network-context.mjs +1 -1
- package/lib/outbound-identity.mjs +2 -3
- package/lib/page-fingerprints.mjs +2 -2
- package/lib/page-role-profile.mjs +1 -1
- package/lib/perceptual-hash.mjs +12 -4
- package/lib/portable-generator.mjs +29 -0
- package/lib/provider-policy-admission.mjs +11 -4
- package/lib/rdap-bootstrap.mjs +76 -16
- package/lib/rdap-client.mjs +20 -12
- package/lib/rdap-normalization.mjs +324 -80
- package/lib/rdap-registrar.mjs +5 -1
- package/lib/rdap-transport.mjs +2 -2
- package/lib/rdap-validation.mjs +65 -14
- package/lib/rdap.mjs +2 -2
- package/lib/registrable-domain.mjs +36 -0
- package/lib/registry-capabilities.mjs +3 -0
- package/lib/registry-capability-catalogue.mjs +12 -2
- package/lib/registry-comparison.mjs +28 -11
- package/lib/registry-insights.mjs +7 -0
- package/lib/response-policy.mjs +27 -4
- package/lib/risk-calibration-summary.mjs +340 -0
- package/lib/risk-scoring.mjs +4 -3
- package/lib/rpki-evidence.mjs +9 -8
- package/lib/safe-fetch.mjs +121 -50
- package/lib/security-txt.mjs +27 -4
- package/lib/service-binding-dns.mjs +6 -1
- package/lib/smtp-transport-review.mjs +787 -0
- package/lib/sslbl-certificates.generated.mjs +136 -134
- package/lib/sslbl-intelligence.mjs +6 -8
- package/lib/static-html-analysis.mjs +301 -4
- package/lib/structured-data-identity.mjs +51 -19
- package/lib/threat-intelligence-fixtures.mjs +2 -4
- package/lib/threat-intelligence-manifests.mjs +2 -7
- package/lib/threat-intelligence-runtime.mjs +5 -15
- package/lib/threat-intelligence-types.mjs +21 -1
- package/lib/threatfox-intelligence.mjs +7 -3
- package/lib/tls-intelligence.mjs +15 -9
- package/lib/tlsa-evidence.mjs +37 -7
- package/lib/urlhaus-intelligence.mjs +4 -2
- package/lib/urlscan-intelligence.mjs +6 -5
- package/lib/website-security-posture.mjs +1 -1
- package/lib/website-technology.mjs +1 -1
- package/lib/whois-authority.mjs +43 -8
- package/lib/whois-chain.mjs +25 -3
- package/lib/whois-normalization.mjs +3 -1
- package/lib/whois-transport.mjs +12 -3
- package/lib/zone-intent-review.mjs +1 -1
- package/package.json +12 -4
- package/packages/cases/case-brand-profile-references.mjs +85 -0
- package/{frontend/src/lib/analysis/case-evidence-model.js → packages/cases/case-evidence-model.mjs} +43 -14
- package/packages/cases/case-investigation-branch-model.mjs +132 -0
- package/packages/cases/case-migration-model.mjs +476 -0
- package/{frontend/src/lib/analysis/case-model.js → packages/cases/case-model.mjs} +3 -3
- package/{frontend/src/lib/analysis/case-record-contracts.js → packages/cases/case-record-contracts.mjs} +3 -19
- package/{frontend/src/lib/analysis/case-record-core.js → packages/cases/case-record-core.mjs} +58 -31
- package/packages/cases/case-record-model.mjs +9 -0
- package/{frontend/src/lib/analysis/case-record-operations.js → packages/cases/case-record-operations.mjs} +106 -33
- package/{frontend/src/lib/analysis/case-report.js → packages/cases/case-report.mjs} +143 -16
- package/packages/cases/case-response-model.mjs +1604 -0
- package/packages/cases/case-response-packet.mjs +1312 -0
- package/{frontend/src/lib/analysis/case-storage-model.js → packages/cases/case-storage-model.mjs} +2 -2
- package/{frontend/src/lib/analysis/http-summary.js → packages/cases/http-summary.mjs} +1 -1
- package/packages/cases/opaque-reference-id.mjs +9 -0
- package/packages/comparison/comparison-ledger-bulk.mjs +366 -0
- package/packages/comparison/comparison-ledger-contract.mjs +241 -0
- package/packages/comparison/comparison-ledger-json.mjs +80 -0
- package/packages/comparison/comparison-ledger-serialization.mjs +52 -0
- package/packages/comparison/favicon-similarity.mjs +78 -0
- package/{frontend/src/lib/analysis/page-similarity.js → packages/comparison/page-similarity.mjs} +4 -3
- package/{frontend/src/lib/analysis/relationship-evidence.js → packages/comparison/relationship-evidence.mjs} +6 -7
- package/packages/contracts/analyst-review-state-contract.mjs +61 -0
- package/packages/contracts/analyst-review-state.mjs +50 -0
- package/packages/contracts/case-portability.mjs +1310 -0
- package/packages/contracts/cli-command-catalogue.mjs +2 -0
- package/packages/contracts/cli-lookup.mjs +394 -0
- package/packages/contracts/domain-control-flight-recorder.mjs +461 -0
- package/packages/contracts/domain-control-manifest.mjs +1027 -0
- package/packages/contracts/domain-control-monitor.mjs +310 -0
- package/packages/contracts/domain-control-review.mjs +828 -0
- package/packages/contracts/domain-name.mjs +4 -0
- package/packages/contracts/external-observation-interchange.mjs +224 -0
- package/packages/contracts/extracted-domain-lifecycle.mjs +218 -0
- package/packages/contracts/investigation-portability.mjs +331 -0
- package/packages/contracts/offline-comparison.mjs +160 -0
- package/packages/contracts/risk-calibration.mjs +1288 -0
- package/packages/contracts/schema-compatibility.mjs +71 -0
- package/packages/contracts/schema-lifecycle.mjs +1702 -0
- package/packages/contracts/sslbl-snapshot.mjs +17 -0
- package/packages/contracts/workspace-portability.mjs +1347 -0
- package/packages/evidence/artifact-integrity.mjs +65 -0
- package/packages/evidence/decision-fact.mjs +735 -0
- package/packages/evidence/domain-control-runtime.mjs +547 -0
- package/packages/evidence/domain-name.mjs +41 -0
- package/packages/evidence/observation.mjs +384 -0
- package/{frontend/src/lib/analysis/external-findings-import.js → packages/interchange/external-findings-import.mjs} +86 -20
- package/packages/interchange/retained-text.mjs +8 -0
- package/{frontend/src/lib/analysis/acquisition-decision-packet.js → packages/investigation/acquisition-decision-packet.mjs} +11 -8
- package/{frontend/src/lib/analysis/bulk-domain-comparison.js → packages/investigation/bulk-domain-comparison.mjs} +28 -9
- package/{frontend/src/lib/analysis/bulk-mail-exposure.js → packages/investigation/bulk-mail-exposure.mjs} +67 -18
- package/{frontend/src/lib/analysis/bulk-review-export.js → packages/investigation/bulk-review-export.mjs} +10 -9
- package/packages/investigation/bulk-review-policy.mjs +1 -0
- package/{frontend/src/lib/analysis/investigation-capsule.js → packages/investigation/investigation-capsule.mjs} +50 -18
- package/packages/investigation/lookup-artefact-inputs.mjs +1 -0
- package/{frontend/src/lib/analysis/lookup-asset-graph.js → packages/investigation/lookup-asset-graph.mjs} +4 -2
- package/packages/investigation/lookup-claim-passport.mjs +232 -0
- package/packages/investigation/lookup-claim-readiness.mjs +287 -0
- package/packages/investigation/lookup-investigation-brief.mjs +172 -0
- package/packages/monitoring/analyst-review-state.mjs +680 -0
- package/{frontend/src/lib/analysis/brand-profile-model.js → packages/workspace/brand-profile-model.mjs} +97 -50
- package/{frontend/src/lib/analysis/bulk-review-model.js → packages/workspace/bulk-review-model.mjs} +15 -15
- package/{frontend/src/lib/analysis/bulk-session-model.js → packages/workspace/bulk-session-model.mjs} +179 -39
- package/{frontend/src/lib/analysis/bulk-sort.js → packages/workspace/bulk-sort.mjs} +12 -5
- package/{frontend/src/lib/analysis/campaign-model.js → packages/workspace/campaign-model.mjs} +20 -22
- package/{frontend/src/lib/analysis/detection-rule-model.js → packages/workspace/detection-rule-model.mjs} +15 -17
- package/packages/workspace/hostile-input.mjs +172 -0
- package/{frontend/src/lib/analysis/investigation-guide.js → packages/workspace/investigation-guide.mjs} +51 -46
- package/{frontend/src/lib/analysis/investigation-template-model.js → packages/workspace/investigation-template-model.mjs} +15 -15
- package/{frontend/src/lib/analysis/page-baseline.js → packages/workspace/page-baseline.mjs} +6 -13
- package/{frontend/src/lib/analysis/relationship-observation-model.js → packages/workspace/relationship-observation-model.mjs} +13 -17
- package/{frontend/src/lib/analysis/shortlist-model.js → packages/workspace/shortlist-model.mjs} +20 -18
- package/{frontend/src/lib/analysis/watchlist-history.js → packages/workspace/watchlist-history.mjs} +9 -16
- package/{frontend/src/lib/analysis/watchlist-store.js → packages/workspace/watchlist-store.mjs} +24 -17
- package/{frontend/src/lib/analysis/website-snapshot-model.js → packages/workspace/website-snapshot-model.mjs} +18 -12
- package/{frontend/src/lib/analysis/workspace-archive-crypto.js → packages/workspace/workspace-archive-crypto.mjs} +49 -19
- package/{frontend/src/lib/analysis/workspace-archive.js → packages/workspace/workspace-archive.mjs} +195 -84
- package/third-party-notices.txt +112 -5834
- package/frontend/src/lib/analysis/artifact-integrity.js +0 -21
- package/frontend/src/lib/analysis/bulk-retry-plan.js +0 -79
- package/frontend/src/lib/analysis/bulk-source-coverage.js +0 -76
- package/frontend/src/lib/analysis/case-migration-model.js +0 -253
- package/frontend/src/lib/analysis/case-record-model.js +0 -7
- package/frontend/src/lib/analysis/case-response-model.js +0 -669
- package/frontend/src/lib/analysis/case-response-packet.js +0 -573
- package/frontend/src/lib/analysis/lookup-decision-support.js +0 -487
- package/frontend/src/lib/analysis/lookup-investigation-brief.js +0 -153
- package/frontend/src/lib/analysis/lookup-presentation.js +0 -58
- package/frontend/src/lib/analysis/lookup-response.js +0 -2
- package/frontend/src/lib/analysis/lookup-source-refresh.js +0 -277
- package/frontend/src/lib/analysis/lookup-summary-model.js +0 -398
- package/frontend/src/lib/analysis/scoring.js +0 -55
- package/frontend/src/lib/analysis/utils.js +0 -232
- package/frontend/src/lib/analysis/visualization-bounds.js +0 -26
- package/frontend/src/lib/analysis/visualization-force-graph.js +0 -352
- package/frontend/src/lib/analysis/visualization-matrix.js +0 -88
- package/frontend/src/lib/analysis/visualization-models.js +0 -369
- package/frontend/src/lib/analysis/visualization-time-series.js +0 -144
- package/lib/bounded-json-response.mjs +0 -157
- package/lib/lookup-response-contract.mjs +0 -346
- package/lib/observation.mjs +0 -114
- /package/{frontend/src/lib/analysis/acquisition-due-diligence.js → packages/investigation/acquisition-due-diligence.mjs} +0 -0
- /package/{frontend/src/lib/analysis/evidence-coverage-ledger.js → packages/investigation/evidence-coverage-ledger.mjs} +0 -0
- /package/{frontend/src/lib/analysis/dns-record-normalization.js → packages/workspace/dns-record-normalization.mjs} +0 -0
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
import { ANALYST_REVIEW_REASON_VALUES } from "./analyst-taxonomy.mjs";
|
|
2
|
+
import { scanBoundedJson } from "./bounded-json.mjs";
|
|
3
|
+
import { RISK_REVIEW_THRESHOLD } from "./risk-scoring.mjs";
|
|
4
|
+
import { MAX_RISK_CALIBRATION_RECORDS, MAX_RISK_CALIBRATION_JSON_CONTAINER_ITEMS, MAX_RISK_CALIBRATION_JSON_DEPTH, MAX_RISK_CALIBRATION_JSON_KEYS, MAX_RISK_CALIBRATION_JSON_VALUES, MAX_RISK_CALIBRATION_MODEL_VERSION, MAX_RISK_CALIBRATION_REVIEW_REASON_LENGTH, MAX_RISK_CALIBRATION_SUMMARY_BYTES, MAX_RISK_CALIBRATION_SUMMARY_STRATA, MAX_RISK_CALIBRATION_TIMESTAMP_LENGTH, RISK_CALIBRATION_DATASET_SCHEMA, RISK_CALIBRATION_REPORT_CONFIDENCE_KEYS, RISK_CALIBRATION_REPORT_DATASET_KEYS, RISK_CALIBRATION_REPORT_INTERPRETATION_KEYS, RISK_CALIBRATION_REPORT_INTERVAL_KEYS, RISK_CALIBRATION_REPORT_METRIC_KEYS, RISK_CALIBRATION_REPORT_MODEL_COMPARISON_KEYS, RISK_CALIBRATION_REPORT_PRIVACY_KEYS, RISK_CALIBRATION_REPORT_SCHEMA, RISK_CALIBRATION_REPORT_SCORE_BAND_KEYS, RISK_CALIBRATION_REPORT_STRATUM_KEYS, RISK_CALIBRATION_REPORT_SUMMARY_KEYS, RISK_CALIBRATION_REPORT_VERSION, RISK_CALIBRATION_REPORT_V3_SUMMARY_ROOT_KEYS, SUPPORTED_RISK_CALIBRATION_DATASET_VERSIONS, } from "../packages/contracts/risk-calibration.mjs";
|
|
5
|
+
export const RISK_CALIBRATION_SUMMARY_SCHEMA = RISK_CALIBRATION_REPORT_SCHEMA;
|
|
6
|
+
export const RISK_CALIBRATION_SUMMARY_VERSION = RISK_CALIBRATION_REPORT_VERSION;
|
|
7
|
+
export const RISK_CALIBRATION_SUMMARY_MAX_BYTES = MAX_RISK_CALIBRATION_SUMMARY_BYTES;
|
|
8
|
+
export const RISK_CALIBRATION_SUMMARY_MAX_RECORDS = MAX_RISK_CALIBRATION_RECORDS;
|
|
9
|
+
export const RISK_CALIBRATION_SUMMARY_MIN_STRATUM_SAMPLE = 20;
|
|
10
|
+
export const RISK_CALIBRATION_SUMMARY_THRESHOLDS = Object.freeze([...new Set([40, 50, 60, RISK_REVIEW_THRESHOLD, 80, 90])].sort((left, right) => left - right));
|
|
11
|
+
export const RISK_CALIBRATION_SUMMARY_INTERPRETATION = 'This offline replay compares heuristic Risk scores with analyst dispositions. It does not prove maliciousness or safety.';
|
|
12
|
+
const ROOT_KEYS = new Set(RISK_CALIBRATION_REPORT_V3_SUMMARY_ROOT_KEYS);
|
|
13
|
+
const DATASET_KEYS = new Set(RISK_CALIBRATION_REPORT_DATASET_KEYS);
|
|
14
|
+
const SUMMARY_KEYS = new Set(RISK_CALIBRATION_REPORT_SUMMARY_KEYS);
|
|
15
|
+
const SCORE_BAND_KEYS = new Set(RISK_CALIBRATION_REPORT_SCORE_BAND_KEYS);
|
|
16
|
+
const METRIC_KEYS = new Set(RISK_CALIBRATION_REPORT_METRIC_KEYS);
|
|
17
|
+
const CONFIDENCE_KEYS = new Set(RISK_CALIBRATION_REPORT_CONFIDENCE_KEYS);
|
|
18
|
+
const INTERVAL_KEYS = new Set(RISK_CALIBRATION_REPORT_INTERVAL_KEYS);
|
|
19
|
+
const STRATUM_KEYS = new Set(RISK_CALIBRATION_REPORT_STRATUM_KEYS);
|
|
20
|
+
const MODEL_COMPARISON_KEYS = new Set(RISK_CALIBRATION_REPORT_MODEL_COMPARISON_KEYS);
|
|
21
|
+
const PRIVACY_KEYS = new Set(RISK_CALIBRATION_REPORT_PRIVACY_KEYS);
|
|
22
|
+
const INTERPRETATION_KEYS = new Set(RISK_CALIBRATION_REPORT_INTERPRETATION_KEYS);
|
|
23
|
+
const REVIEW_REASON_VALUES = new Set([...ANALYST_REVIEW_REASON_VALUES, 'not_recorded']);
|
|
24
|
+
const SCAN_DEPTH_VALUES = new Set(['deep', 'fast', 'unknown']);
|
|
25
|
+
const CONTROL_RE = /[\x00-\x1f\x7f]/u;
|
|
26
|
+
function object(value, label) {
|
|
27
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
28
|
+
throw new Error(`${label} must be an object.`);
|
|
29
|
+
return value;
|
|
30
|
+
}
|
|
31
|
+
function exactKeys(value, expected, label) {
|
|
32
|
+
const keys = Object.keys(value);
|
|
33
|
+
if (keys.length !== expected.size || keys.some((key) => !expected.has(key))) {
|
|
34
|
+
throw new Error(`${label} contains unsupported or missing fields.`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function boundedCount(value, label, maximum = RISK_CALIBRATION_SUMMARY_MAX_RECORDS) {
|
|
38
|
+
if (!Number.isSafeInteger(value) || Number(value) < 0 || Number(value) > maximum) {
|
|
39
|
+
throw new Error(`${label} must be a bounded non-negative integer.`);
|
|
40
|
+
}
|
|
41
|
+
return Number(value);
|
|
42
|
+
}
|
|
43
|
+
function modelVersion(value, label) {
|
|
44
|
+
if (!Number.isSafeInteger(value) || Number(value) < 1 || Number(value) > MAX_RISK_CALIBRATION_MODEL_VERSION) {
|
|
45
|
+
throw new Error(`${label} must be a supported positive model version.`);
|
|
46
|
+
}
|
|
47
|
+
return Number(value);
|
|
48
|
+
}
|
|
49
|
+
function canonicalTimestamp(value, label) {
|
|
50
|
+
if (typeof value !== 'string'
|
|
51
|
+
|| value.length > MAX_RISK_CALIBRATION_TIMESTAMP_LENGTH
|
|
52
|
+
|| !Number.isFinite(Date.parse(value))) {
|
|
53
|
+
throw new Error(`${label} must be a valid date and time.`);
|
|
54
|
+
}
|
|
55
|
+
const canonical = new Date(value).toISOString();
|
|
56
|
+
if (canonical !== value)
|
|
57
|
+
throw new Error(`${label} must use canonical UTC ISO-8601 form.`);
|
|
58
|
+
return canonical;
|
|
59
|
+
}
|
|
60
|
+
function ratio(numerator, denominator) {
|
|
61
|
+
return denominator ? Number((numerator / denominator).toFixed(4)) : null;
|
|
62
|
+
}
|
|
63
|
+
function wilson95(successes, total) {
|
|
64
|
+
if (!total)
|
|
65
|
+
return null;
|
|
66
|
+
const z = 1.96;
|
|
67
|
+
const proportion = successes / total;
|
|
68
|
+
const denominator = 1 + (z * z) / total;
|
|
69
|
+
const center = (proportion + (z * z) / (2 * total)) / denominator;
|
|
70
|
+
const margin = (z * Math.sqrt((proportion * (1 - proportion) + (z * z) / (4 * total)) / total)) / denominator;
|
|
71
|
+
return Object.freeze({
|
|
72
|
+
lower: Number(Math.max(0, center - margin).toFixed(4)),
|
|
73
|
+
upper: Number(Math.min(1, center + margin).toFixed(4)),
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
function exactRatio(value, expected, label) {
|
|
77
|
+
if (value !== expected)
|
|
78
|
+
throw new Error(`${label} is inconsistent with the retained confusion counts.`);
|
|
79
|
+
return expected;
|
|
80
|
+
}
|
|
81
|
+
function interval(value, expected, label) {
|
|
82
|
+
if (expected === null) {
|
|
83
|
+
if (value !== null)
|
|
84
|
+
throw new Error(`${label} must be null when its denominator is zero.`);
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
const candidate = object(value, label);
|
|
88
|
+
exactKeys(candidate, INTERVAL_KEYS, label);
|
|
89
|
+
if (candidate.lower !== expected.lower || candidate.upper !== expected.upper) {
|
|
90
|
+
throw new Error(`${label} is inconsistent with the retained confusion counts.`);
|
|
91
|
+
}
|
|
92
|
+
return Object.freeze({ lower: expected.lower, upper: expected.upper });
|
|
93
|
+
}
|
|
94
|
+
function thresholdMetrics(value, label, expectedThreshold, expectedTotal, expectedPositive = null, expectedNegative = null) {
|
|
95
|
+
const metric = object(value, label);
|
|
96
|
+
exactKeys(metric, METRIC_KEYS, label);
|
|
97
|
+
if (metric.threshold !== expectedThreshold)
|
|
98
|
+
throw new Error(`${label} has an unexpected threshold.`);
|
|
99
|
+
const truePositive = boundedCount(metric.truePositive, `${label} true-positive count`, expectedTotal);
|
|
100
|
+
const falseNegative = boundedCount(metric.falseNegative, `${label} false-negative count`, expectedTotal);
|
|
101
|
+
const trueNegative = boundedCount(metric.trueNegative, `${label} true-negative count`, expectedTotal);
|
|
102
|
+
const falsePositive = boundedCount(metric.falsePositive, `${label} false-positive count`, expectedTotal);
|
|
103
|
+
if (truePositive + falsePositive + trueNegative + falseNegative !== expectedTotal
|
|
104
|
+
|| (expectedPositive !== null && truePositive + falseNegative !== expectedPositive)
|
|
105
|
+
|| (expectedNegative !== null && trueNegative + falsePositive !== expectedNegative)) {
|
|
106
|
+
throw new Error(`${label} has inconsistent class counts.`);
|
|
107
|
+
}
|
|
108
|
+
const precision = ratio(truePositive, truePositive + falsePositive);
|
|
109
|
+
const recall = ratio(truePositive, truePositive + falseNegative);
|
|
110
|
+
const specificity = ratio(trueNegative, trueNegative + falsePositive);
|
|
111
|
+
const falsePositiveRate = ratio(falsePositive, falsePositive + trueNegative);
|
|
112
|
+
const f1 = precision === null || recall === null || precision + recall === 0
|
|
113
|
+
? null
|
|
114
|
+
: Number(((2 * precision * recall) / (precision + recall)).toFixed(4));
|
|
115
|
+
const balancedAccuracy = recall === null || specificity === null
|
|
116
|
+
? null
|
|
117
|
+
: Number(((recall + specificity) / 2).toFixed(4));
|
|
118
|
+
const confidence = object(metric.confidence95, `${label} confidence intervals`);
|
|
119
|
+
exactKeys(confidence, CONFIDENCE_KEYS, `${label} confidence intervals`);
|
|
120
|
+
return Object.freeze({
|
|
121
|
+
threshold: expectedThreshold,
|
|
122
|
+
truePositive,
|
|
123
|
+
falsePositive,
|
|
124
|
+
trueNegative,
|
|
125
|
+
falseNegative,
|
|
126
|
+
precision: exactRatio(metric.precision, precision, `${label} precision`),
|
|
127
|
+
recall: exactRatio(metric.recall, recall, `${label} recall`),
|
|
128
|
+
specificity: exactRatio(metric.specificity, specificity, `${label} specificity`),
|
|
129
|
+
falsePositiveRate: exactRatio(metric.falsePositiveRate, falsePositiveRate, `${label} false-positive rate`),
|
|
130
|
+
f1: exactRatio(metric.f1, f1, `${label} F1`),
|
|
131
|
+
balancedAccuracy: exactRatio(metric.balancedAccuracy, balancedAccuracy, `${label} balanced accuracy`),
|
|
132
|
+
confidence95: Object.freeze({
|
|
133
|
+
precision: interval(confidence.precision, wilson95(truePositive, truePositive + falsePositive), `${label} precision interval`),
|
|
134
|
+
recall: interval(confidence.recall, wilson95(truePositive, truePositive + falseNegative), `${label} recall interval`),
|
|
135
|
+
specificity: interval(confidence.specificity, wilson95(trueNegative, trueNegative + falsePositive), `${label} specificity interval`),
|
|
136
|
+
}),
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
export function parseRiskCalibrationSummaryReport(raw) {
|
|
140
|
+
if (typeof raw !== 'string')
|
|
141
|
+
throw new Error('Risk calibration summary must be JSON text.');
|
|
142
|
+
const bytes = new TextEncoder().encode(raw).byteLength;
|
|
143
|
+
if (bytes < 1 || bytes > RISK_CALIBRATION_SUMMARY_MAX_BYTES) {
|
|
144
|
+
throw new Error(`Risk calibration summary must be between 1 byte and ${RISK_CALIBRATION_SUMMARY_MAX_BYTES} bytes.`);
|
|
145
|
+
}
|
|
146
|
+
let parsed;
|
|
147
|
+
try {
|
|
148
|
+
const withoutBom = raw.replace(/^\uFEFF/u, '');
|
|
149
|
+
scanBoundedJson(withoutBom, {
|
|
150
|
+
maximumDepth: MAX_RISK_CALIBRATION_JSON_DEPTH,
|
|
151
|
+
maximumKeys: MAX_RISK_CALIBRATION_JSON_KEYS,
|
|
152
|
+
maximumValues: MAX_RISK_CALIBRATION_JSON_VALUES,
|
|
153
|
+
maximumContainerItems: MAX_RISK_CALIBRATION_JSON_CONTAINER_ITEMS,
|
|
154
|
+
});
|
|
155
|
+
parsed = JSON.parse(withoutBom);
|
|
156
|
+
}
|
|
157
|
+
catch (reason) {
|
|
158
|
+
const message = reason instanceof Error
|
|
159
|
+
? reason.message.replace(/^Artefact (?:input|JSON)/u, 'Risk calibration summary')
|
|
160
|
+
: 'Risk calibration summary is not valid JSON.';
|
|
161
|
+
throw new Error(message);
|
|
162
|
+
}
|
|
163
|
+
const root = object(parsed, 'Risk calibration summary');
|
|
164
|
+
exactKeys(root, ROOT_KEYS, 'Risk calibration summary');
|
|
165
|
+
if (root.schema !== RISK_CALIBRATION_SUMMARY_SCHEMA
|
|
166
|
+
|| root.version !== RISK_CALIBRATION_SUMMARY_VERSION
|
|
167
|
+
|| root.mode !== 'summary') {
|
|
168
|
+
throw new Error('Risk calibration summary has an unsupported schema, version, or mode.');
|
|
169
|
+
}
|
|
170
|
+
const dataset = object(root.dataset, 'Risk calibration dataset metadata');
|
|
171
|
+
exactKeys(dataset, DATASET_KEYS, 'Risk calibration dataset metadata');
|
|
172
|
+
if (dataset.schema !== RISK_CALIBRATION_DATASET_SCHEMA
|
|
173
|
+
|| typeof dataset.version !== 'number'
|
|
174
|
+
|| !SUPPORTED_RISK_CALIBRATION_DATASET_VERSIONS.includes(dataset.version)) {
|
|
175
|
+
throw new Error('Risk calibration dataset metadata has an unsupported schema or version.');
|
|
176
|
+
}
|
|
177
|
+
const recordCount = boundedCount(dataset.recordCount, 'Risk calibration dataset record count');
|
|
178
|
+
if (recordCount < 1)
|
|
179
|
+
throw new Error('Risk calibration dataset metadata must describe at least one record.');
|
|
180
|
+
const riskModelVersion = modelVersion(root.riskModelVersion, 'Risk model version');
|
|
181
|
+
if (!RISK_CALIBRATION_SUMMARY_THRESHOLDS.includes(Number(root.currentReviewThreshold))) {
|
|
182
|
+
throw new Error('Current review threshold is not part of the fixed calibration threshold set.');
|
|
183
|
+
}
|
|
184
|
+
const currentReviewThreshold = Number(root.currentReviewThreshold);
|
|
185
|
+
const rawSummary = object(root.summary, 'Risk calibration summary counts');
|
|
186
|
+
exactKeys(rawSummary, SUMMARY_KEYS, 'Risk calibration summary counts');
|
|
187
|
+
const total = boundedCount(rawSummary.total, 'Total record count');
|
|
188
|
+
const positive = boundedCount(rawSummary.positive, 'Positive label count', total);
|
|
189
|
+
const negative = boundedCount(rawSummary.negative, 'Negative label count', total);
|
|
190
|
+
const excluded = boundedCount(rawSummary.excluded, 'Excluded label count', total);
|
|
191
|
+
if (total !== recordCount || positive + negative + excluded !== total) {
|
|
192
|
+
throw new Error('Risk calibration summary label counts are inconsistent.');
|
|
193
|
+
}
|
|
194
|
+
const rawBands = object(rawSummary.scoreBands, 'Risk calibration score bands');
|
|
195
|
+
exactKeys(rawBands, SCORE_BAND_KEYS, 'Risk calibration score bands');
|
|
196
|
+
const scoreBands = Object.freeze({
|
|
197
|
+
not_scored: boundedCount(rawBands.not_scored, 'Not-scored count', total),
|
|
198
|
+
'0_39': boundedCount(rawBands['0_39'], 'Low-band count', total),
|
|
199
|
+
'40_69': boundedCount(rawBands['40_69'], 'Review-band count', total),
|
|
200
|
+
'70_100': boundedCount(rawBands['70_100'], 'High-band count', total),
|
|
201
|
+
});
|
|
202
|
+
if (Object.values(scoreBands).reduce((sum, count) => sum + count, 0) !== total) {
|
|
203
|
+
throw new Error('Risk calibration score-band counts are inconsistent.');
|
|
204
|
+
}
|
|
205
|
+
if (scoreBands.not_scored > excluded) {
|
|
206
|
+
throw new Error('Risk calibration not-scored count cannot exceed excluded labels.');
|
|
207
|
+
}
|
|
208
|
+
if (!Array.isArray(root.thresholds) || root.thresholds.length !== RISK_CALIBRATION_SUMMARY_THRESHOLDS.length) {
|
|
209
|
+
throw new Error('Risk calibration summary must contain the fixed threshold set exactly once.');
|
|
210
|
+
}
|
|
211
|
+
const thresholds = Object.freeze(root.thresholds.map((value, index) => thresholdMetrics(value, `Threshold ${index + 1}`, RISK_CALIBRATION_SUMMARY_THRESHOLDS[index], positive + negative, positive, negative)));
|
|
212
|
+
if (!Array.isArray(root.strata) || root.strata.length > MAX_RISK_CALIBRATION_SUMMARY_STRATA) {
|
|
213
|
+
throw new Error('Risk calibration strata must be a bounded array.');
|
|
214
|
+
}
|
|
215
|
+
const seenStrata = new Set();
|
|
216
|
+
const strata = Object.freeze(root.strata.map((value, index) => {
|
|
217
|
+
const label = `Stratum ${index + 1}`;
|
|
218
|
+
const stratum = object(value, label);
|
|
219
|
+
exactKeys(stratum, STRATUM_KEYS, label);
|
|
220
|
+
if (stratum.dimension !== 'review_reason' && stratum.dimension !== 'scan_depth') {
|
|
221
|
+
throw new Error(`${label} has an unsupported dimension.`);
|
|
222
|
+
}
|
|
223
|
+
if (typeof stratum.value !== 'string'
|
|
224
|
+
|| !stratum.value
|
|
225
|
+
|| stratum.value.length > MAX_RISK_CALIBRATION_REVIEW_REASON_LENGTH
|
|
226
|
+
|| CONTROL_RE.test(stratum.value)) {
|
|
227
|
+
throw new Error(`${label} has an invalid value.`);
|
|
228
|
+
}
|
|
229
|
+
const allowedValues = stratum.dimension === 'review_reason' ? REVIEW_REASON_VALUES : SCAN_DEPTH_VALUES;
|
|
230
|
+
if (!allowedValues.has(stratum.value))
|
|
231
|
+
throw new Error(`${label} has an unsupported value.`);
|
|
232
|
+
const key = `${stratum.dimension}\u0000${stratum.value}`;
|
|
233
|
+
if (seenStrata.has(key))
|
|
234
|
+
throw new Error(`${label} is duplicated.`);
|
|
235
|
+
seenStrata.add(key);
|
|
236
|
+
const sampleCount = boundedCount(stratum.sampleCount, `${label} sample count`, positive + negative);
|
|
237
|
+
if (stratum.insufficientSample !== (sampleCount < RISK_CALIBRATION_SUMMARY_MIN_STRATUM_SAMPLE)) {
|
|
238
|
+
throw new Error(`${label} has an inconsistent sample-sufficiency state.`);
|
|
239
|
+
}
|
|
240
|
+
return Object.freeze({
|
|
241
|
+
dimension: stratum.dimension,
|
|
242
|
+
value: stratum.value,
|
|
243
|
+
sampleCount,
|
|
244
|
+
insufficientSample: stratum.insufficientSample,
|
|
245
|
+
metrics: thresholdMetrics(stratum.metrics, `${label} metrics`, currentReviewThreshold, sampleCount),
|
|
246
|
+
});
|
|
247
|
+
}));
|
|
248
|
+
for (const dimension of ['review_reason', 'scan_depth']) {
|
|
249
|
+
const dimensionTotal = strata
|
|
250
|
+
.filter((stratum) => stratum.dimension === dimension)
|
|
251
|
+
.reduce((sum, stratum) => sum + stratum.sampleCount, 0);
|
|
252
|
+
if (dimensionTotal !== positive + negative) {
|
|
253
|
+
throw new Error(`Risk calibration ${dimension.replace('_', '-')} strata do not cover every included label exactly once.`);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
const comparison = object(root.modelComparison, 'Risk model comparison');
|
|
257
|
+
exactKeys(comparison, MODEL_COMPARISON_KEYS, 'Risk model comparison');
|
|
258
|
+
if (typeof comparison.available !== 'boolean')
|
|
259
|
+
throw new Error('Risk model comparison availability must be true or false.');
|
|
260
|
+
const currentModelVersion = modelVersion(comparison.currentModelVersion, 'Compared current model version');
|
|
261
|
+
if (currentModelVersion !== riskModelVersion)
|
|
262
|
+
throw new Error('Risk model comparison does not match the report model version.');
|
|
263
|
+
const previousModelVersion = comparison.previousModelVersion === null
|
|
264
|
+
? null
|
|
265
|
+
: modelVersion(comparison.previousModelVersion, 'Compared previous model version');
|
|
266
|
+
const scoresChanged = boundedCount(comparison.scoresChanged, 'Changed-score count', total);
|
|
267
|
+
const bandsChanged = boundedCount(comparison.bandsChanged, 'Changed-band count', total);
|
|
268
|
+
const thresholdClassificationsChanged = boundedCount(comparison.thresholdClassificationsChanged, 'Changed-threshold count', total);
|
|
269
|
+
if ((!comparison.available && (previousModelVersion !== null || scoresChanged || bandsChanged || thresholdClassificationsChanged))
|
|
270
|
+
|| (comparison.available && previousModelVersion === null)
|
|
271
|
+
|| bandsChanged > scoresChanged
|
|
272
|
+
|| thresholdClassificationsChanged > scoresChanged) {
|
|
273
|
+
throw new Error('Risk model comparison availability and counts are inconsistent.');
|
|
274
|
+
}
|
|
275
|
+
const privacy = object(root.privacy, 'Risk calibration privacy declaration');
|
|
276
|
+
exactKeys(privacy, PRIVACY_KEYS, 'Risk calibration privacy declaration');
|
|
277
|
+
if (privacy.targetsRetained !== 0 || privacy.identifiersRetained !== 0 || privacy.rawEvidenceRetained !== 0) {
|
|
278
|
+
throw new Error('Risk calibration summary must retain zero targets, identifiers, and raw evidence.');
|
|
279
|
+
}
|
|
280
|
+
const interpretation = object(root.interpretation, 'Risk calibration interpretation');
|
|
281
|
+
exactKeys(interpretation, INTERPRETATION_KEYS, 'Risk calibration interpretation');
|
|
282
|
+
if (interpretation.authority !== 'analyst_context_only'
|
|
283
|
+
|| interpretation.statement !== RISK_CALIBRATION_SUMMARY_INTERPRETATION
|
|
284
|
+
|| interpretation.automaticTuning !== false
|
|
285
|
+
|| interpretation.networkRequests !== false
|
|
286
|
+
|| interpretation.persisted !== false) {
|
|
287
|
+
throw new Error('Risk calibration interpretation is inconsistent with the offline diagnostic boundary.');
|
|
288
|
+
}
|
|
289
|
+
return Object.freeze({
|
|
290
|
+
schema: RISK_CALIBRATION_SUMMARY_SCHEMA,
|
|
291
|
+
version: RISK_CALIBRATION_SUMMARY_VERSION,
|
|
292
|
+
mode: 'summary',
|
|
293
|
+
generatedAt: canonicalTimestamp(root.generatedAt, 'Risk calibration generation time'),
|
|
294
|
+
dataset: Object.freeze({
|
|
295
|
+
schema: RISK_CALIBRATION_DATASET_SCHEMA,
|
|
296
|
+
version: dataset.version,
|
|
297
|
+
recordCount,
|
|
298
|
+
}),
|
|
299
|
+
riskModelVersion,
|
|
300
|
+
currentReviewThreshold,
|
|
301
|
+
summary: Object.freeze({ total, positive, negative, excluded, scoreBands }),
|
|
302
|
+
thresholds,
|
|
303
|
+
strata,
|
|
304
|
+
modelComparison: Object.freeze({
|
|
305
|
+
available: comparison.available,
|
|
306
|
+
previousModelVersion,
|
|
307
|
+
currentModelVersion,
|
|
308
|
+
scoresChanged,
|
|
309
|
+
bandsChanged,
|
|
310
|
+
thresholdClassificationsChanged,
|
|
311
|
+
}),
|
|
312
|
+
privacy: Object.freeze({ targetsRetained: 0, identifiersRetained: 0, rawEvidenceRetained: 0 }),
|
|
313
|
+
interpretation: Object.freeze({
|
|
314
|
+
authority: 'analyst_context_only',
|
|
315
|
+
statement: RISK_CALIBRATION_SUMMARY_INTERPRETATION,
|
|
316
|
+
automaticTuning: false,
|
|
317
|
+
networkRequests: false,
|
|
318
|
+
persisted: false,
|
|
319
|
+
}),
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
export function buildRiskCalibrationSummaryReport(source) {
|
|
323
|
+
const report = object(source, 'Risk calibration report');
|
|
324
|
+
const projected = {
|
|
325
|
+
schema: RISK_CALIBRATION_SUMMARY_SCHEMA,
|
|
326
|
+
version: RISK_CALIBRATION_SUMMARY_VERSION,
|
|
327
|
+
mode: 'summary',
|
|
328
|
+
generatedAt: report.generatedAt,
|
|
329
|
+
dataset: report.dataset,
|
|
330
|
+
riskModelVersion: report.riskModelVersion,
|
|
331
|
+
currentReviewThreshold: report.currentReviewThreshold,
|
|
332
|
+
summary: report.summary,
|
|
333
|
+
thresholds: report.thresholds,
|
|
334
|
+
strata: report.strata,
|
|
335
|
+
modelComparison: report.modelComparison,
|
|
336
|
+
privacy: { targetsRetained: 0, identifiersRetained: 0, rawEvidenceRetained: 0 },
|
|
337
|
+
interpretation: report.interpretation,
|
|
338
|
+
};
|
|
339
|
+
return parseRiskCalibrationSummaryReport(JSON.stringify(projected));
|
|
340
|
+
}
|
package/lib/risk-scoring.mjs
CHANGED
|
@@ -97,8 +97,9 @@ function scoreQuality(input) {
|
|
|
97
97
|
if ([input.hasPasswordField, input.hasExternalFormAction].some(hasBoolean) || typeof input.phishingLanguageMatch === 'string') {
|
|
98
98
|
observedFamilies.push('credential-lure');
|
|
99
99
|
}
|
|
100
|
-
if (input.threatIntelligence
|
|
100
|
+
if (calibrateExternalIntelligenceRisk(input.threatIntelligence, input.domain).eligibleProviderCount > 0) {
|
|
101
101
|
observedFamilies.push('external-intelligence');
|
|
102
|
+
}
|
|
102
103
|
if ([input.hasMx, input.hasSpf, input.hasDmarc].some(hasBoolean)
|
|
103
104
|
|| typeof input.activityStatus === 'string'
|
|
104
105
|
|| typeof input.domainAgeDays === 'number')
|
|
@@ -156,7 +157,7 @@ function explainRiskScoreInternal(input, excludedFamily) {
|
|
|
156
157
|
else if (input.hasExternalFormAction === true) {
|
|
157
158
|
add('credential-lure', 'External form destination observed without a password field', 0);
|
|
158
159
|
}
|
|
159
|
-
const external = calibrateExternalIntelligenceRisk(input.threatIntelligence);
|
|
160
|
+
const external = calibrateExternalIntelligenceRisk(input.threatIntelligence, input.domain);
|
|
160
161
|
if (external.factor)
|
|
161
162
|
add('external-intelligence', external.factor.label, external.contribution);
|
|
162
163
|
const primaryFamilies = ['domain-resemblance', 'brand-presentation', 'credential-lure', 'external-intelligence']
|
|
@@ -279,7 +280,7 @@ export function explainRiskScoreV6(input) {
|
|
|
279
280
|
}
|
|
280
281
|
if (contextual.size >= 2)
|
|
281
282
|
add('corroboration', `Corroborating context across ${contextual.size} distinct evidence families`, contextual.size >= 3 ? 20 : 10);
|
|
282
|
-
const external = calibrateExternalIntelligenceRisk(input.threatIntelligence);
|
|
283
|
+
const external = calibrateExternalIntelligenceRisk(input.threatIntelligence, input.domain);
|
|
283
284
|
if (external.factor)
|
|
284
285
|
add('external-intelligence', external.factor.label, external.contribution);
|
|
285
286
|
if (input.activityStatus === 'active')
|
package/lib/rpki-evidence.mjs
CHANGED
|
@@ -65,14 +65,13 @@ function reviewRpkiRoute(input) {
|
|
|
65
65
|
limitations: baseLimitations,
|
|
66
66
|
});
|
|
67
67
|
}
|
|
68
|
-
const
|
|
68
|
+
const inputTruncated = rows.length > MAX_RPKI_RECORDS;
|
|
69
69
|
const normalized = rows.slice(0, MAX_RPKI_RECORDS).map(normalizeAuthorization);
|
|
70
70
|
const rejectedCount = normalized.filter((item) => item === null).length + Math.max(0, rows.length - MAX_RPKI_RECORDS);
|
|
71
71
|
const authorizations = normalized.filter((item) => item !== null);
|
|
72
72
|
const covering = authorizations.filter((item) => prefixContains(item.parsedPrefix, route));
|
|
73
|
-
const
|
|
73
|
+
const evaluated = covering
|
|
74
74
|
.sort((left, right) => right.parsedPrefix.length - left.parsedPrefix.length || left.asn - right.asn)
|
|
75
|
-
.slice(0, MAX_RPKI_MATCHES)
|
|
76
75
|
.map((item) => Object.freeze({
|
|
77
76
|
prefix: item.prefix,
|
|
78
77
|
maxLength: item.maxLength,
|
|
@@ -81,13 +80,15 @@ function reviewRpkiRoute(input) {
|
|
|
81
80
|
: route.length > item.maxLength ? 'max_length_exceeded'
|
|
82
81
|
: 'valid',
|
|
83
82
|
}));
|
|
84
|
-
const matchingAuthorizationCount =
|
|
83
|
+
const matchingAuthorizationCount = evaluated.filter((item) => item.state === 'valid').length;
|
|
84
|
+
const matches = evaluated.slice(0, MAX_RPKI_MATCHES);
|
|
85
|
+
const incompleteInput = inputTruncated || rejectedCount > 0;
|
|
85
86
|
const state = covering.length === 0
|
|
86
|
-
? 'not_found'
|
|
87
|
+
? incompleteInput ? 'partial' : 'not_found'
|
|
87
88
|
: matchingAuthorizationCount > 0
|
|
88
|
-
?
|
|
89
|
+
? incompleteInput || covering.length > MAX_RPKI_MATCHES ? 'partial'
|
|
89
90
|
: 'valid'
|
|
90
|
-
: 'invalid';
|
|
91
|
+
: incompleteInput || covering.length > MAX_RPKI_MATCHES ? 'partial' : 'invalid';
|
|
91
92
|
return Object.freeze({
|
|
92
93
|
schema: RPKI_EVIDENCE_SCHEMA,
|
|
93
94
|
version: RPKI_EVIDENCE_VERSION,
|
|
@@ -97,7 +98,7 @@ function reviewRpkiRoute(input) {
|
|
|
97
98
|
coveringAuthorizationCount: covering.length,
|
|
98
99
|
matchingAuthorizationCount,
|
|
99
100
|
rejectedCount,
|
|
100
|
-
truncated,
|
|
101
|
+
truncated: inputTruncated || covering.length > MAX_RPKI_MATCHES,
|
|
101
102
|
matches: Object.freeze(matches),
|
|
102
103
|
limitations: baseLimitations,
|
|
103
104
|
});
|
package/lib/safe-fetch.mjs
CHANGED
|
@@ -25,6 +25,7 @@ import { Agent, fetch as undiciFetch } from 'undici';
|
|
|
25
25
|
import { MAX_OUTBOUND_REDIRECTS } from "./outbound-request-bounds.mjs";
|
|
26
26
|
const MAX_REDIRECTS = MAX_OUTBOUND_REDIRECTS;
|
|
27
27
|
const MAX_SAFE_FETCH_URL_LENGTH = 4096;
|
|
28
|
+
const MAX_SAFE_FETCH_ADDRESS_CANDIDATES = 64;
|
|
28
29
|
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
29
30
|
async function defaultSafeFetchRequest(url, options) {
|
|
30
31
|
// TypeScript's DOM and undici fetch declarations model equivalent web
|
|
@@ -195,6 +196,33 @@ function isPrivateAddress(ip) {
|
|
|
195
196
|
return isPrivateIpv6(ip);
|
|
196
197
|
return true; // not a recognizable IP literal - fail closed
|
|
197
198
|
}
|
|
199
|
+
function normalizePublicAddressCandidates(value, hostname) {
|
|
200
|
+
if (!Array.isArray(value) || value.length === 0)
|
|
201
|
+
throw new Error(`No DNS records for ${hostname}`);
|
|
202
|
+
if (value.length > MAX_SAFE_FETCH_ADDRESS_CANDIDATES) {
|
|
203
|
+
throw new Error(`Refusing to fetch ${hostname}: DNS returned too many address candidates`);
|
|
204
|
+
}
|
|
205
|
+
const normalized = [];
|
|
206
|
+
const seen = new Set();
|
|
207
|
+
for (const item of value) {
|
|
208
|
+
if (!item || typeof item !== 'object' || Array.isArray(item)) {
|
|
209
|
+
throw new Error(`Refusing to fetch ${hostname}: DNS returned an invalid address record`);
|
|
210
|
+
}
|
|
211
|
+
const { address, family } = item;
|
|
212
|
+
if (typeof address !== 'string' || (family !== 4 && family !== 6) || net.isIP(address) !== family) {
|
|
213
|
+
throw new Error(`Refusing to fetch ${hostname}: DNS returned an invalid address record`);
|
|
214
|
+
}
|
|
215
|
+
if (isPrivateAddress(address)) {
|
|
216
|
+
throw new Error(`Refusing to fetch ${hostname}: resolves to a private/reserved address (${address})`);
|
|
217
|
+
}
|
|
218
|
+
const key = `${family}:${address}`;
|
|
219
|
+
if (!seen.has(key)) {
|
|
220
|
+
seen.add(key);
|
|
221
|
+
normalized.push({ address, family });
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return normalized;
|
|
225
|
+
}
|
|
198
226
|
// Resolves the hostname and returns every address it maps to, having
|
|
199
227
|
// verified all of them are public. Throws (fails closed) if any address is
|
|
200
228
|
// private/reserved, or if there are no records at all.
|
|
@@ -206,14 +234,7 @@ async function resolvePublicAddresses(hostname) {
|
|
|
206
234
|
catch (err) {
|
|
207
235
|
throw new Error(`DNS resolution failed for ${hostname}: ${err instanceof Error ? err.message : String(err)}`);
|
|
208
236
|
}
|
|
209
|
-
|
|
210
|
-
throw new Error(`No DNS records for ${hostname}`);
|
|
211
|
-
for (const { address } of records) {
|
|
212
|
-
if (isPrivateAddress(address)) {
|
|
213
|
-
throw new Error(`Refusing to fetch ${hostname}: resolves to a private/reserved address (${address})`);
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
return records;
|
|
237
|
+
return normalizePublicAddressCandidates(records, hostname);
|
|
217
238
|
}
|
|
218
239
|
// A per-request undici Agent whose connections are pinned to the given,
|
|
219
240
|
// already-validated address(es) - overriding the `lookup` function used at
|
|
@@ -261,6 +282,12 @@ function normalizedFetchUrl(value) {
|
|
|
261
282
|
throw new Error('Refusing to fetch an oversized URL');
|
|
262
283
|
return normalized;
|
|
263
284
|
}
|
|
285
|
+
function resolutionHostname(parsed) {
|
|
286
|
+
const hostname = parsed.hostname;
|
|
287
|
+
return hostname.startsWith('[') && hostname.endsWith(']')
|
|
288
|
+
? hostname.slice(1, -1)
|
|
289
|
+
: hostname;
|
|
290
|
+
}
|
|
264
291
|
function boundedDuration(value) {
|
|
265
292
|
return Math.max(0, Math.min(120_000, Math.round(Number(value) || 0)));
|
|
266
293
|
}
|
|
@@ -274,6 +301,24 @@ async function closeDispatcher(dispatcher) {
|
|
|
274
301
|
// Cleanup must never replace the request result or its more useful error.
|
|
275
302
|
}
|
|
276
303
|
}
|
|
304
|
+
function abortError(signal) {
|
|
305
|
+
if (signal.reason instanceof Error)
|
|
306
|
+
return signal.reason;
|
|
307
|
+
const error = new Error('The outbound request was aborted.');
|
|
308
|
+
error.name = 'AbortError';
|
|
309
|
+
return error;
|
|
310
|
+
}
|
|
311
|
+
async function raceWithSignal(promise, signal) {
|
|
312
|
+
if (!signal)
|
|
313
|
+
return promise;
|
|
314
|
+
if (signal.aborted)
|
|
315
|
+
throw abortError(signal);
|
|
316
|
+
return new Promise((resolve, reject) => {
|
|
317
|
+
const abort = () => reject(abortError(signal));
|
|
318
|
+
signal.addEventListener('abort', abort, { once: true });
|
|
319
|
+
void promise.then((value) => { signal.removeEventListener('abort', abort); resolve(value); }, (error) => { signal.removeEventListener('abort', abort); reject(error); });
|
|
320
|
+
});
|
|
321
|
+
}
|
|
277
322
|
// Detailed form of the shared safe request engine. It follows redirects
|
|
278
323
|
// manually under the same DNS validation and connection-pinning policy as
|
|
279
324
|
// safeFetch(), while retaining a bounded hop trace for consumers that need
|
|
@@ -300,7 +345,11 @@ async function safeFetchDetailed(url, options = {}, dependencies = {}) {
|
|
|
300
345
|
let redirectCount = 0;
|
|
301
346
|
while (true) {
|
|
302
347
|
const parsed = new URL(currentUrl);
|
|
303
|
-
const
|
|
348
|
+
const hostname = resolutionHostname(parsed);
|
|
349
|
+
const signal = options.signal ?? null;
|
|
350
|
+
const records = normalizePublicAddressCandidates(await raceWithSignal(resolveAddresses(hostname), signal), hostname);
|
|
351
|
+
if (signal?.aborted)
|
|
352
|
+
throw abortError(signal);
|
|
304
353
|
const dispatcher = makeDispatcher(records);
|
|
305
354
|
const hopStartedAt = now();
|
|
306
355
|
// `dispatcher` is a supported undici extension used to pin the connection.
|
|
@@ -385,34 +434,43 @@ async function readTextCapped(res, maxBytes, options = {}) {
|
|
|
385
434
|
throw new Error('Response body does not expose a bounded readable stream');
|
|
386
435
|
}
|
|
387
436
|
const reader = res.body.getReader();
|
|
388
|
-
const decoder = new TextDecoder();
|
|
437
|
+
const decoder = new TextDecoder('utf-8', { fatal: options.fatalUtf8 === true });
|
|
389
438
|
let text = '';
|
|
390
439
|
let received = 0;
|
|
391
440
|
let truncated = false;
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
441
|
+
try {
|
|
442
|
+
while (received < maxBytes) {
|
|
443
|
+
const { done, value } = await reader.read();
|
|
444
|
+
if (done)
|
|
445
|
+
break;
|
|
446
|
+
const remaining = maxBytes - received;
|
|
447
|
+
const captured = value.subarray(0, remaining);
|
|
448
|
+
hasher?.update(captured);
|
|
449
|
+
received += captured.byteLength;
|
|
450
|
+
text += decoder.decode(captured, { stream: true });
|
|
451
|
+
if (captured.byteLength < value.byteLength) {
|
|
452
|
+
truncated = true;
|
|
453
|
+
break;
|
|
454
|
+
}
|
|
404
455
|
}
|
|
456
|
+
// If the cap was hit, one more read() tells us whether there was actually
|
|
457
|
+
// more data left (truncated) or the body just happened to end right there.
|
|
458
|
+
if (!truncated && received >= maxBytes) {
|
|
459
|
+
const { done } = await reader.read();
|
|
460
|
+
if (!done)
|
|
461
|
+
truncated = true;
|
|
462
|
+
}
|
|
463
|
+
text += decoder.decode();
|
|
464
|
+
return result(text, truncated, received);
|
|
405
465
|
}
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
466
|
+
finally {
|
|
467
|
+
try {
|
|
468
|
+
reader.cancel().catch(() => { });
|
|
469
|
+
}
|
|
470
|
+
finally {
|
|
471
|
+
reader.releaseLock();
|
|
472
|
+
}
|
|
412
473
|
}
|
|
413
|
-
reader.cancel().catch(() => { });
|
|
414
|
-
text += decoder.decode();
|
|
415
|
-
return result(text, truncated, received);
|
|
416
474
|
}
|
|
417
475
|
// Binary-safe counterpart to readTextCapped, for a response body that isn't
|
|
418
476
|
// text (a favicon image) - collects raw bytes into a Buffer instead of
|
|
@@ -427,26 +485,39 @@ async function readBytesCapped(res, maxBytes) {
|
|
|
427
485
|
const chunks = [];
|
|
428
486
|
let received = 0;
|
|
429
487
|
let truncated = false;
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
488
|
+
// Cancel unread body data and release the exclusive reader lock on every
|
|
489
|
+
// exit path, exactly as readTextCapped does. A mid-stream read failure (an
|
|
490
|
+
// upstream reset while fetching an untrusted favicon) must not leave the
|
|
491
|
+
// response locked to this reader.
|
|
492
|
+
try {
|
|
493
|
+
while (received < maxBytes) {
|
|
494
|
+
const { done, value } = await reader.read();
|
|
495
|
+
if (done)
|
|
496
|
+
break;
|
|
497
|
+
const remaining = maxBytes - received;
|
|
498
|
+
const captured = value.subarray(0, remaining);
|
|
499
|
+
received += captured.byteLength;
|
|
500
|
+
chunks.push(Buffer.from(captured));
|
|
501
|
+
if (captured.byteLength < value.byteLength) {
|
|
502
|
+
truncated = true;
|
|
503
|
+
break;
|
|
504
|
+
}
|
|
441
505
|
}
|
|
506
|
+
// Same "one more read() to check for leftover data" trick as readTextCapped.
|
|
507
|
+
if (!truncated && received >= maxBytes) {
|
|
508
|
+
const { done } = await reader.read();
|
|
509
|
+
if (!done)
|
|
510
|
+
truncated = true;
|
|
511
|
+
}
|
|
512
|
+
return { bytes: Buffer.concat(chunks), truncated, bytesRead: received };
|
|
442
513
|
}
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
514
|
+
finally {
|
|
515
|
+
try {
|
|
516
|
+
reader.cancel().catch(() => { });
|
|
517
|
+
}
|
|
518
|
+
finally {
|
|
519
|
+
reader.releaseLock();
|
|
520
|
+
}
|
|
448
521
|
}
|
|
449
|
-
reader.cancel().catch(() => { });
|
|
450
|
-
return { bytes: Buffer.concat(chunks), truncated, bytesRead: received };
|
|
451
522
|
}
|
|
452
|
-
export { MAX_REDIRECTS, MAX_SAFE_FETCH_URL_LENGTH, safeFetch, safeFetchDetailed, readTextCapped, readBytesCapped, isPrivateAddress, resolvePublicAddresses, };
|
|
523
|
+
export { MAX_REDIRECTS, MAX_SAFE_FETCH_URL_LENGTH, MAX_SAFE_FETCH_ADDRESS_CANDIDATES, safeFetch, safeFetchDetailed, readTextCapped, readBytesCapped, isPrivateAddress, resolvePublicAddresses, };
|