@slicedearth/whoisleuth-cli 1.47.4 → 2.0.0

package/cli/risk-calibration.mjs

@@ -1,22 +1,16 @@
 import { Buffer } from 'node:buffer';
+import { decodeBoundedUtf8 } from "../lib/bounded-file.mjs";
 import { isValidAsciiDomainName } from "../lib/hostname.mjs";
+import { scanBoundedJson } from "../lib/bounded-json.mjs";
+import { canonicalRegistrableDomain } from "../lib/registrable-domain.mjs";
 import { ANALYST_REVIEW_REASON_VALUES, analystInteroperabilityTags } from "../lib/analyst-taxonomy.mjs";
-import { RISK_CALIBRATION_SUMMARY_SCHEMA, RISK_CALIBRATION_SUMMARY_VERSION, RISK_CALIBRATION_SUMMARY_INTERPRETATION, RISK_CALIBRATION_SUMMARY_THRESHOLDS, } from "../lib/risk-calibration-summary.mjs";
+import { buildRiskCalibrationSummaryReport, parseRiskCalibrationSummaryReport, RISK_CALIBRATION_SUMMARY_INTERPRETATION, RISK_CALIBRATION_SUMMARY_THRESHOLDS, } from "../lib/risk-calibration-summary.mjs";
+import { MAX_RISK_CALIBRATION_ACTIVITY_LENGTH, MAX_RISK_CALIBRATION_AVAILABILITY_LENGTH, MAX_RISK_CALIBRATION_DISPOSITION_LENGTH, MAX_RISK_CALIBRATION_DOMAIN_AGE_DAYS, MAX_RISK_CALIBRATION_DOMAIN_LENGTH, MAX_RISK_CALIBRATION_FINDING_CATEGORY_LENGTH, MAX_RISK_CALIBRATION_FINDINGS_PER_PROVIDER, MAX_RISK_CALIBRATION_INPUT_BYTES, MAX_RISK_CALIBRATION_JSON_CONTAINER_ITEMS, MAX_RISK_CALIBRATION_JSON_DEPTH, MAX_RISK_CALIBRATION_JSON_KEYS, MAX_RISK_CALIBRATION_JSON_VALUES, MAX_RISK_CALIBRATION_MUTATIONS, MAX_RISK_CALIBRATION_PROVIDER_ID_LENGTH, MAX_RISK_CALIBRATION_PROVIDER_STATE_LENGTH, MAX_RISK_CALIBRATION_PROVIDERS, MAX_RISK_CALIBRATION_RECORD_ID_LENGTH, MAX_RISK_CALIBRATION_RECORDS, MAX_RISK_CALIBRATION_REVIEW_REASON_LENGTH, MAX_RISK_CALIBRATION_SCAN_DEPTH_LENGTH, MAX_RISK_CALIBRATION_STRING_LENGTH, MAX_RISK_CALIBRATION_TIMESTAMP_LENGTH, RISK_CALIBRATION_DATASET_SCHEMA, RISK_CALIBRATION_DATASET_VERSION, RISK_CALIBRATION_REPORT_SCHEMA, RISK_CALIBRATION_REPORT_VERSION, serializeRiskCalibrationSnapshot, snapshotRiskCalibrationReportForSerialization, SUPPORTED_RISK_CALIBRATION_DATASET_VERSIONS, } from "../packages/contracts/risk-calibration.mjs";
 import { CliUsageError } from "./arguments.mjs";
 import { RISK_MUTATION_TYPES } from "../lib/risk-scoring.mjs";
-export const RISK_CALIBRATION_DATASET_SCHEMA = 'whoisleuth.risk-calibration-dataset';
-export const RISK_CALIBRATION_DATASET_VERSION = 2;
-export const SUPPORTED_RISK_CALIBRATION_DATASET_VERSIONS = Object.freeze([1, RISK_CALIBRATION_DATASET_VERSION]);
-export const RISK_CALIBRATION_REPORT_SCHEMA = RISK_CALIBRATION_SUMMARY_SCHEMA;
-export const RISK_CALIBRATION_REPORT_VERSION = RISK_CALIBRATION_SUMMARY_VERSION;
-export const MAX_RISK_CALIBRATION_INPUT_BYTES = 2 * 1024 * 1024;
-export const MAX_RISK_CALIBRATION_RECORDS = 500;
-export const MAX_RISK_CALIBRATION_STRING_LENGTH = 256;
+import { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_ENVELOPE_VERSION, THREAT_INTELLIGENCE_SCHEMA, } from "../lib/threat-intelligence-types.mjs";
+export { MAX_RISK_CALIBRATION_INPUT_BYTES, MAX_RISK_CALIBRATION_RECORDS, MAX_RISK_CALIBRATION_STRING_LENGTH, RISK_CALIBRATION_DATASET_SCHEMA, RISK_CALIBRATION_DATASET_VERSION, RISK_CALIBRATION_REPORT_SCHEMA, RISK_CALIBRATION_REPORT_VERSION, SUPPORTED_RISK_CALIBRATION_DATASET_VERSIONS, } from "../packages/contracts/risk-calibration.mjs";
 export const RISK_CALIBRATION_THRESHOLDS = RISK_CALIBRATION_SUMMARY_THRESHOLDS;
-const MAX_MUTATIONS = 30;
-const MAX_PROVIDERS = 10;
-const MAX_FINDINGS_PER_PROVIDER = 100;
-const MAX_TIMESTAMP_LENGTH = 64;
 const CONTROL_RE = /[\x00-\x1f\x7f]/;
 const DISPOSITIONS = new Set([
     'unreviewed', 'suspicious', 'confirmed_abuse', 'false_positive', 'expected', 'closed_no_action',
@@ -49,7 +43,7 @@ function boundedString(value, field, maximum = MAX_RISK_CALIBRATION_STRING_LENGT
 function optionalTimestamp(value, field) {
     if (value === null || value === undefined)
         return undefined;
-    return boundedString(value, field, MAX_TIMESTAMP_LENGTH);
+    return boundedString(value, field, MAX_RISK_CALIBRATION_TIMESTAMP_LENGTH);
 }
 function projectThreatIntelligence(value, field) {
     if (value === null || value === undefined)
@@ -57,8 +51,8 @@ function projectThreatIntelligence(value, field) {
     const envelope = object(value, field);
     if (!Array.isArray(envelope.providers))
         throw new CliUsageError(`${field}.providers must be an array.`);
-    if (envelope.providers.length > MAX_PROVIDERS)
-        throw new CliUsageError(`${field}.providers exceeds the ${MAX_PROVIDERS}-provider limit.`);
+    if (envelope.providers.length > MAX_RISK_CALIBRATION_PROVIDERS)
+        throw new CliUsageError(`${field}.providers exceeds the ${MAX_RISK_CALIBRATION_PROVIDERS}-provider limit.`);
     return {
         providers: envelope.providers.map((item, providerIndex) => {
             const prefix = `${field}.providers[${providerIndex}]`;
@@ -69,15 +63,15 @@ function projectThreatIntelligence(value, field) {
                 : object(provider.observation, `${prefix}.observation`);
             if (!Array.isArray(provider.findings))
                 throw new CliUsageError(`${prefix}.findings must be an array.`);
-            if (provider.findings.length > MAX_FINDINGS_PER_PROVIDER) {
-                throw new CliUsageError(`${prefix}.findings exceeds the ${MAX_FINDINGS_PER_PROVIDER}-finding limit.`);
+            if (provider.findings.length > MAX_RISK_CALIBRATION_FINDINGS_PER_PROVIDER) {
+                throw new CliUsageError(`${prefix}.findings exceeds the ${MAX_RISK_CALIBRATION_FINDINGS_PER_PROVIDER}-finding limit.`);
             }
             const observedAt = observation
                 ? optionalTimestamp(observation.observedAt, `${prefix}.observation.observedAt`)
                 : undefined;
             return {
-                provider: { id: boundedString(identity.id, `${prefix}.provider.id`, 64) },
-                state: boundedString(provider.state, `${prefix}.state`, 32),
+                provider: { id: boundedString(identity.id, `${prefix}.provider.id`, MAX_RISK_CALIBRATION_PROVIDER_ID_LENGTH) },
+                state: boundedString(provider.state, `${prefix}.state`, MAX_RISK_CALIBRATION_PROVIDER_STATE_LENGTH),
                 ...(observation ? { observation: observedAt ? { observedAt } : {} } : {}),
                 findings: provider.findings.map((findingValue, findingIndex) => {
                     const findingPrefix = `${prefix}.findings[${findingIndex}]`;
@@ -85,7 +79,7 @@ function projectThreatIntelligence(value, field) {
                     const firstObservedAt = optionalTimestamp(finding.firstObservedAt, `${findingPrefix}.firstObservedAt`);
                     const lastObservedAt = optionalTimestamp(finding.lastObservedAt, `${findingPrefix}.lastObservedAt`);
                     return {
-                        category: boundedString(finding.category, `${findingPrefix}.category`, 64),
+                        category: boundedString(finding.category, `${findingPrefix}.category`, MAX_RISK_CALIBRATION_FINDING_CATEGORY_LENGTH),
                         ...(firstObservedAt ? { firstObservedAt } : {}),
                         ...(lastObservedAt ? { lastObservedAt } : {}),
                     };
@@ -96,10 +90,12 @@ function projectThreatIntelligence(value, field) {
 }
 function projectEvidence(value, field) {
     const source = object(value, field);
-    const availability = boundedString(source.availability ?? source.state, `${field}.availability`, 32);
+    const availability = boundedString(source.availability ?? source.state, `${field}.availability`, MAX_RISK_CALIBRATION_AVAILABILITY_LENGTH);
     if (!AVAILABILITY_STATES.has(availability))
         throw new CliUsageError(`${field}.availability is unsupported.`);
-    const result = { availability };
+    const result = {
+        availability: availability,
+    };
     for (const name of BOOLEAN_FIELDS) {
         const candidate = source[name];
         if (candidate === null || candidate === undefined)
@@ -109,13 +105,13 @@ function projectEvidence(value, field) {
         result[name] = candidate;
     }
     if (source.activityStatus !== null && source.activityStatus !== undefined) {
-        const activity = boundedString(source.activityStatus, `${field}.activityStatus`, 32);
+        const activity = boundedString(source.activityStatus, `${field}.activityStatus`, MAX_RISK_CALIBRATION_ACTIVITY_LENGTH);
         if (!ACTIVITY_STATES.has(activity))
             throw new CliUsageError(`${field}.activityStatus is unsupported.`);
         result.activityStatus = activity;
     }
     if (source.scanDepth !== null && source.scanDepth !== undefined) {
-        const depth = boundedString(source.scanDepth, `${field}.scanDepth`, 16);
+        const depth = boundedString(source.scanDepth, `${field}.scanDepth`, MAX_RISK_CALIBRATION_SCAN_DEPTH_LENGTH);
         if (depth !== 'fast' && depth !== 'deep')
             throw new CliUsageError(`${field}.scanDepth must be fast or deep.`);
         Object.assign(result, { scanDepth: depth });
@@ -130,7 +126,7 @@ function projectEvidence(value, field) {
     }
     if (source.domainAgeDays !== null && source.domainAgeDays !== undefined) {
         if (typeof source.domainAgeDays !== 'number' || !Number.isFinite(source.domainAgeDays)
-            || source.domainAgeDays < 0 || source.domainAgeDays > 100_000) {
+            || source.domainAgeDays < 0 || source.domainAgeDays > MAX_RISK_CALIBRATION_DOMAIN_AGE_DAYS) {
             throw new CliUsageError(`${field}.domainAgeDays must be a finite number from 0 to 100000.`);
         }
         result.domainAgeDays = source.domainAgeDays;
@@ -138,10 +134,10 @@ function projectEvidence(value, field) {
     if (source.mutationTypes !== null && source.mutationTypes !== undefined) {
         if (!Array.isArray(source.mutationTypes))
             throw new CliUsageError(`${field}.mutationTypes must be an array.`);
-        if (source.mutationTypes.length > MAX_MUTATIONS)
-            throw new CliUsageError(`${field}.mutationTypes exceeds the ${MAX_MUTATIONS}-item limit.`);
+        if (source.mutationTypes.length > MAX_RISK_CALIBRATION_MUTATIONS)
+            throw new CliUsageError(`${field}.mutationTypes exceeds the ${MAX_RISK_CALIBRATION_MUTATIONS}-item limit.`);
         result.mutationTypes = [...new Set(source.mutationTypes.map((item, index) => {
-                const mutation = boundedString(item, `${field}.mutationTypes[${index}]`, 64);
+                const mutation = boundedString(item, `${field}.mutationTypes[${index}]`, MAX_RISK_CALIBRATION_REVIEW_REASON_LENGTH);
                 if (!MUTATION_TYPES.has(mutation))
                     throw new CliUsageError(`${field}.mutationTypes[${index}] is unsupported.`);
                 return mutation;
@@ -164,7 +160,12 @@ export async function readRiskCalibrationInputBounded(stream, limit = MAX_RISK_C
             throw new CliUsageError(`Risk calibration input is limited to ${limit} bytes.`);
         chunks.push(buffer);
     }
-    return Buffer.concat(chunks).toString('utf8');
+    try {
+        return decodeBoundedUtf8(Buffer.concat(chunks), 'Risk calibration input');
+    }
+    catch (cause) {
+        throw new CliUsageError(cause instanceof Error ? cause.message : 'Risk calibration input must contain valid UTF-8 text.');
+    }
 }
 export function parseRiskCalibrationDataset(text) {
     if (typeof text !== 'string')
@@ -172,12 +173,19 @@ export function parseRiskCalibrationDataset(text) {
     if (Buffer.byteLength(text, 'utf8') > MAX_RISK_CALIBRATION_INPUT_BYTES) {
         throw new CliUsageError(`Risk calibration input is limited to ${MAX_RISK_CALIBRATION_INPUT_BYTES} bytes.`);
     }
+    const normalized = text.replace(/^\uFEFF/u, '');
     let parsed;
     try {
-        parsed = JSON.parse(text.replace(/^\uFEFF/, ''));
+        scanBoundedJson(normalized, {
+            maximumDepth: MAX_RISK_CALIBRATION_JSON_DEPTH,
+            maximumKeys: MAX_RISK_CALIBRATION_JSON_KEYS,
+            maximumValues: MAX_RISK_CALIBRATION_JSON_VALUES,
+            maximumContainerItems: MAX_RISK_CALIBRATION_JSON_CONTAINER_ITEMS,
+        });
+        parsed = JSON.parse(normalized);
     }
     catch {
-        throw new CliUsageError('Risk calibration input must be valid JSON.');
+        throw new CliUsageError('Risk calibration input must be valid bounded JSON without duplicate keys.');
     }
     const document = object(parsed, 'Risk calibration input');
     const documentVersion = typeof document.version === 'number' && Number.isInteger(document.version)
@@ -198,19 +206,19 @@ export function parseRiskCalibrationDataset(text) {
     const records = document.records.map((value, index) => {
         const prefix = `records[${index}]`;
         const record = object(value, prefix);
-        const id = boundedString(record.id, `${prefix}.id`, 128);
+        const id = boundedString(record.id, `${prefix}.id`, MAX_RISK_CALIBRATION_RECORD_ID_LENGTH);
         if (ids.has(id))
             throw new CliUsageError(`${prefix}.id must be unique.`);
         ids.add(id);
-        const domain = boundedString(record.domain, `${prefix}.domain`, 253).toLowerCase().replace(/\.$/, '');
+        const domain = boundedString(record.domain, `${prefix}.domain`, MAX_RISK_CALIBRATION_DOMAIN_LENGTH).toLowerCase().replace(/\.$/, '');
         if (!isValidAsciiDomainName(domain, { requireDot: true }))
             throw new CliUsageError(`${prefix}.domain must be a valid ASCII DNS hostname, not an IP address.`);
-        const analystDisposition = boundedString(record.analystDisposition, `${prefix}.analystDisposition`, 32);
+        const analystDisposition = boundedString(record.analystDisposition, `${prefix}.analystDisposition`, MAX_RISK_CALIBRATION_DISPOSITION_LENGTH);
         if (!DISPOSITIONS.has(analystDisposition))
             throw new CliUsageError(`${prefix}.analystDisposition is unsupported.`);
         let reviewReasonCode;
         if (record.reviewReasonCode !== null && record.reviewReasonCode !== undefined) {
-            reviewReasonCode = boundedString(record.reviewReasonCode, `${prefix}.reviewReasonCode`, 64);
+            reviewReasonCode = boundedString(record.reviewReasonCode, `${prefix}.reviewReasonCode`, MAX_RISK_CALIBRATION_REVIEW_REASON_LENGTH);
             if (!REVIEW_REASON_CODES.has(reviewReasonCode))
                 throw new CliUsageError(`${prefix}.reviewReasonCode is unsupported.`);
         }
@@ -297,9 +305,32 @@ function scoreBand(score) {
         return '40_69';
     return '0_39';
 }
+function scoringEvidence(record) {
+    const threatIntelligence = record.evidence.threatIntelligence;
+    const scoringDomain = canonicalRegistrableDomain(record.domain);
+    return {
+        ...record.evidence,
+        domain: scoringDomain ?? record.domain,
+        ...(threatIntelligence && scoringDomain ? {
+            threatIntelligence: {
+                version: THREAT_INTELLIGENCE_ENVELOPE_VERSION,
+                providers: threatIntelligence.providers.map((provider) => ({
+                    ...provider,
+                    schema: THREAT_INTELLIGENCE_SCHEMA,
+                    version: THREAT_INTELLIGENCE_CONTRACT_VERSION,
+                    target: {
+                        type: 'domain',
+                        value: scoringDomain,
+                        exposure: 'registrable_domain',
+                    },
+                })),
+            },
+        } : {}),
+    };
+}
 export function buildRiskCalibrationReport(dataset, explainRiskScore, options) {
     const records = dataset.records.map((record) => {
-        const explained = explainRiskScore(record.evidence);
+        const explained = explainRiskScore(scoringEvidence(record));
         const classification = metricClass(record.analystDisposition);
         const includedInMetrics = classification !== 'excluded' && explained !== null;
         return {
@@ -363,7 +394,7 @@ export function buildRiskCalibrationReport(dataset, explainRiskScore, options) {
             const current = records[index];
             if (!source || !current)
                 continue;
-            const previous = options.explainPreviousRiskScore(source.evidence);
+            const previous = options.explainPreviousRiskScore(scoringEvidence(source));
             if (previous?.score !== current.score)
                 scoresChanged += 1;
             if (scoreBand(previous?.score ?? null) !== current.band)
@@ -402,3 +433,19 @@ export function buildRiskCalibrationReport(dataset, explainRiskScore, options) {
         },
     };
 }
+export function serializeRiskCalibrationReport(report) {
+    try {
+        const snapshot = snapshotRiskCalibrationReportForSerialization(report);
+        const serialised = serializeRiskCalibrationSnapshot(snapshot);
+        if (snapshot.mode === 'summary')
+            parseRiskCalibrationSummaryReport(serialised);
+        else
+            buildRiskCalibrationSummaryReport(snapshot.document);
+        return serialised;
+    }
+    catch (cause) {
+        throw new CliUsageError(cause instanceof Error
+            ? cause.message
+            : 'Risk calibration output must use the current detailed or summary report contract.');
+    }
+}