@aiwg/cli 2026.9.20 → 2026.9.21

package/dist/src/decision/qualification/index.js

@@ -0,0 +1,12 @@
+export * from './types.js';
+export * from './manifest.js';
+export * from './gates.js';
+export * from './gate-evidence.js';
+export * from './runner.js';
+export * from './quality.js';
+export * from './drift.js';
+export * from './privacy.js';
+export * from './capture.js';
+export * from './release.js';
+export * from './load.js';
+//# sourceMappingURL=index.js.map

package/dist/src/decision/qualification/load.js

@@ -0,0 +1,275 @@
+import { createHash } from 'node:crypto';
+import { canonicalJson } from '../../security/artifact-trust.js';
+import { AdmissionError, DecisionAdmissionRegistry } from '../admission.js';
+export function decisionLoadManifestDigest(bytes) {
+    return `sha256:${createHash('sha256').update(bytes).digest('hex')}`;
+}
+/** mulberry32: small, seedable, and stable across Node versions. */
+function seeded(seed) {
+    let state = seed >>> 0;
+    return () => {
+        state = (state + 0x6d2b79f5) >>> 0;
+        let value = Math.imul(state ^ (state >>> 15), 1 | state);
+        value = (value + Math.imul(value ^ (value >>> 7), 61 | value)) ^ value;
+        return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296;
+    };
+}
+/** Seeded bursty Poisson arrivals for the manifest's load, spike, and soak phases. */
+export function decisionLoadArrivals(manifest, windowSeconds) {
+    const model = manifest.arrivalModel;
+    const random = seeded(model.seed);
+    const total = manifest.duration.loadSeconds + manifest.duration.spikeSeconds + manifest.duration.soakSeconds;
+    const endMs = Math.min(total, windowSeconds ?? total) * 1000;
+    const spikeStart = manifest.duration.loadSeconds * 1000;
+    const spikeEnd = spikeStart + model.spikeDurationSeconds * 1000;
+    const weights = manifest.principals.reduce((sum, principal) => sum + principal.weight, 0);
+    const providers = Object.keys(manifest.providers).sort();
+    const arrivals = [];
+    let at = 0;
+    for (;;) {
+        const rate = (at >= spikeStart && at < spikeEnd ? model.spikeMultiplier : 1) * model.averageRequestsPerSecond / 1000;
+        at += -Math.log(1 - random()) / rate;
+        if (at >= endMs)
+            break;
+        let pick = random() * weights;
+        const principal = manifest.principals.find(candidate => (pick -= candidate.weight) < 0) ?? manifest.principals.at(-1);
+        const provider = providers[Math.floor(random() * providers.length)];
+        const huge = random() < model.hugeTokenShare;
+        const service = model.serviceTimeMs.minimum + Math.floor(random() * (model.serviceTimeMs.maximum - model.serviceTimeMs.minimum + 1));
+        const fails = random() < model.retryableFailureRate;
+        const cancels = random() < model.cancellationRate;
+        const atMs = Math.round(at);
+        arrivals.push({ ordinal: arrivals.length, atMs, principalId: principal.id, principalClass: principal.class,
+            providerId: provider, tokens: huge ? model.hugeTokens : model.smallTokens, serviceMs: service,
+            failsFirstAttempt: fails, cancelAtMs: cancels ? atMs + model.cancelAfterMs : null });
+    }
+    return arrivals;
+}
+/** Scheduler policy the manifest declares for one principal. */
+export function decisionLoadPolicy(manifest, workspaceId, principalId) {
+    const principal = manifest.principals.find(candidate => candidate.id === principalId);
+    return {
+        enabled: true, profileVersion: manifest.profileVersion,
+        workspace: { id: workspaceId, limits: structuredClone(manifest.workspace) },
+        principal: { id: principal.id, limits: structuredClone(principal.limits) },
+        providers: structuredClone(manifest.providers),
+    };
+}
+/** Drives each arrival straight through a registry-owned `DecisionAdmissionController`. */
+export function admissionControllerLoadDriver(manifest, clock, registry = new DecisionAdmissionRegistry()) {
+    const workspaceId = 'load-harness';
+    for (const principal of manifest.principals)
+        registry.register(decisionLoadPolicy(manifest, workspaceId, principal.id), clock.now);
+    const controller = registry.controller(workspaceId);
+    const driver = async (arrival, signal, context) => {
+        const queueDelaysMs = [];
+        let attempts = 0;
+        for (let retry = 0; retry <= manifest.arrivalModel.maxRetries; retry += 1) {
+            attempts += 1;
+            let lease;
+            try {
+                lease = await controller.acquire({ budgetId: `load-${arrival.ordinal}`, principalId: arrival.principalId,
+                    workspaceId, providerId: arrival.providerId, signal,
+                    estimate: { tokens: arrival.tokens, attempts: 1, batchSize: 1, items: 1, costUsd: 0, retainedWork: 1 },
+                    deadlineEpochMs: clock.now() + manifest.workspace.maxQueueWaitMs + arrival.serviceMs });
+            }
+            catch (error) {
+                if (!(error instanceof AdmissionError))
+                    throw error;
+                controller.releaseBudget(`load-${arrival.ordinal}`);
+                return { attempts, outcome: error.evidence.reason === 'cancelled' ? 'cancelled' : 'shed',
+                    admissionReason: error.evidence.reason, queueDelaysMs, settledAtMs: clock.now() };
+            }
+            queueDelaysMs.push(lease.evidence.queueDelayMs);
+            const dispatched = await context.dispatch(arrival.serviceMs, signal);
+            const failed = arrival.failsFirstAttempt && retry === 0;
+            lease.release({ success: dispatched === 'completed' && !failed });
+            if (dispatched === 'aborted') {
+                controller.releaseBudget(`load-${arrival.ordinal}`);
+                return { attempts, outcome: 'cancelled', queueDelaysMs, settledAtMs: clock.now() };
+            }
+            if (!failed) {
+                controller.releaseBudget(`load-${arrival.ordinal}`);
+                return { attempts, outcome: 'completed', queueDelaysMs, settledAtMs: clock.now() };
+            }
+            if (retry < manifest.arrivalModel.maxRetries) {
+                await new Promise(done => setTimeout(done, manifest.arrivalModel.retryBackoffMs));
+            }
+        }
+        controller.releaseBudget(`load-${arrival.ordinal}`);
+        return { attempts, outcome: 'failed', queueDelaysMs, settledAtMs: clock.now() };
+    };
+    return { driver, sample: () => controller.snapshot() };
+}
+/**
+ * Offline load harness. Arrivals are released on a virtual timeline supplied by
+ * the caller's fake clock. Deterministic observations are separated from
+ * host-dependent memory and CPU measurements.
+ */
+export async function runDecisionLoadHarness(options) {
+    const { manifest, clock, driver, sample } = options;
+    const arrivals = decisionLoadArrivals(manifest, options.windowSeconds);
+    const start = clock.now();
+    const threadCpu = process.threadCpuUsage?.bind(process);
+    const cpuStart = threadCpu?.();
+    const wallStart = process.hrtime.bigint();
+    let rss = process.memoryUsage().rss;
+    let heap = process.memoryUsage().heapUsed;
+    const sampleMemory = () => {
+        const usage = process.memoryUsage();
+        rss = Math.max(rss, usage.rss);
+        heap = Math.max(heap, usage.heapUsed);
+    };
+    let active = 0;
+    let maximumActive = 0;
+    let maximumQueued = 0;
+    const observe = () => {
+        const snapshot = sample();
+        maximumQueued = Math.max(maximumQueued, snapshot.queued);
+        maximumActive = Math.max(maximumActive, snapshot.active, active);
+    };
+    const context = {
+        clock, manifest,
+        dispatch: (serviceMs, signal) => new Promise(resolve => {
+            active += 1;
+            observe();
+            const finish = (result) => {
+                clearTimeout(timer);
+                signal.removeEventListener('abort', aborted);
+                active -= 1;
+                resolve(result);
+            };
+            const aborted = () => finish('aborted');
+            const timer = setTimeout(() => finish('completed'), serviceMs);
+            if (signal.aborted)
+                aborted();
+            else
+                signal.addEventListener('abort', aborted, { once: true });
+        }),
+    };
+    const outcomes = [];
+    for (const arrival of arrivals) {
+        const due = start + arrival.atMs - clock.now();
+        if (due > 0)
+            await clock.advance(due);
+        const controller = new AbortController();
+        const cancellation = { abortedAtMs: null };
+        if (arrival.cancelAtMs !== null) {
+            setTimeout(() => { cancellation.abortedAtMs = clock.now(); controller.abort(); }, arrival.cancelAtMs - arrival.atMs);
+        }
+        outcomes.push(driver(arrival, controller.signal, context)
+            .then(result => ({ arrival, result, abortedAtMs: cancellation.abortedAtMs })));
+        observe();
+        if (arrival.ordinal % 500 === 0)
+            sampleMemory();
+    }
+    let settled = false;
+    const all = Promise.all(outcomes).then(values => { settled = true; return values; });
+    const drainStep = manifest.arrivalModel.serviceTimeMs.maximum + manifest.arrivalModel.retryBackoffMs;
+    while (!settled)
+        await clock.advance(drainStep);
+    const results = await all;
+    sampleMemory();
+    const threadUsage = cpuStart && threadCpu ? threadCpu(cpuStart) : undefined;
+    const cpuMicros = threadUsage ? threadUsage.user + threadUsage.system : Number(process.hrtime.bigint() - wallStart) / 1000;
+    const virtualDurationMs = Math.max(1, clock.now() - start);
+    const shedReasons = {};
+    let attempts = 0;
+    let completed = 0;
+    let failed = 0;
+    let shed = 0;
+    let cancelled = 0;
+    let laneWait = 0;
+    let cancellationLatency = 0;
+    const admitted = { quiet: 0, noisy: 0 };
+    const submitted = { quiet: 0, noisy: 0 };
+    for (const { arrival, result, abortedAtMs } of results) {
+        attempts += result.attempts;
+        submitted[arrival.principalClass] += 1;
+        if (result.queueDelaysMs.length)
+            admitted[arrival.principalClass] += 1;
+        for (const delay of result.queueDelaysMs)
+            laneWait = Math.max(laneWait, delay);
+        if (result.outcome === 'completed')
+            completed += 1;
+        else if (result.outcome === 'failed')
+            failed += 1;
+        else if (result.outcome === 'cancelled')
+            cancelled += 1;
+        else {
+            shed += 1;
+            shedReasons[result.admissionReason ?? 'unknown'] = (shedReasons[result.admissionReason ?? 'unknown'] ?? 0) + 1;
+        }
+        if (result.outcome === 'cancelled' && abortedAtMs !== null) {
+            cancellationLatency = Math.max(cancellationLatency, result.settledAtMs - abortedAtMs);
+        }
+    }
+    const ratio = (value, total) => total ? Number((value / total).toFixed(6)) : 1;
+    return {
+        observations: {
+            arrivals: arrivals.length, attempts, completed, failed, shed, cancelled,
+            shedReasons: Object.fromEntries(Object.entries(shedReasons).sort(([left], [right]) => left.localeCompare(right))),
+            maximumActiveCalls: maximumActive, maximumQueuedCalls: maximumQueued, maximumEligibleLaneWaitMs: laneWait,
+            retryAmplificationRatio: ratio(attempts, arrivals.length), maximumCancellationLatencyMs: cancellationLatency,
+            quietPrincipalAdmissionRatio: ratio(admitted.quiet, submitted.quiet),
+            noisyPrincipalAdmissionRatio: ratio(admitted.noisy, submitted.noisy),
+            virtualDurationMs,
+        },
+        resources: {
+            heapUsedMiB: Number((heap / 1_048_576).toFixed(1)),
+            residentMemoryMiB: Number((rss / 1_048_576).toFixed(1)),
+            cpuPercentOneCore: Number(((cpuMicros / 1000) / virtualDurationMs * 100).toFixed(3)),
+            cpuSource: threadUsage ? 'thread' : 'wall-clock-upper-bound',
+        },
+    };
+}
+export function compareDecisionLoadBounds(manifest, observations, resources) {
+    const bounds = manifest.bounds;
+    const atMost = (bound, observed) => ({ bound, limit: bounds[bound], observed, pass: observed <= bounds[bound] });
+    return [
+        atMost('maximumActiveCalls', observations.maximumActiveCalls),
+        atMost('maximumQueuedCalls', observations.maximumQueuedCalls),
+        atMost('maximumHeapUsedMiB', resources.heapUsedMiB),
+        atMost('maximumCpuPercentOneCore', resources.cpuPercentOneCore),
+        atMost('maximumEligibleLaneWaitMs', observations.maximumEligibleLaneWaitMs),
+        atMost('maximumRetryAmplificationRatio', observations.retryAmplificationRatio),
+        atMost('maximumCancellationLatencyMs', observations.maximumCancellationLatencyMs),
+        { bound: 'minimumQuietPrincipalAdmissionRatio', limit: bounds.minimumQuietPrincipalAdmissionRatio,
+            observed: observations.quietPrincipalAdmissionRatio,
+            pass: observations.quietPrincipalAdmissionRatio >= bounds.minimumQuietPrincipalAdmissionRatio },
+    ];
+}
+/** Build a digest-bound result record. The manifest digest must be computed from its exact bytes. */
+export function decisionLoadResultRecord(input) {
+    const comparisons = compareDecisionLoadBounds(input.manifest, input.observations, input.resources);
+    const body = {
+        schema: 'decision-load-result/v1',
+        manifest: { schema: input.manifest.schema, version: input.manifest.version, digest: input.manifestDigest },
+        mode: input.manifest.mode, target: input.target, window: { seconds: input.windowSeconds },
+        environment: { node: process.version, platform: process.platform, arch: process.arch },
+        observations: input.observations, resources: input.resources, comparisons,
+        passed: comparisons.every(comparison => comparison.pass),
+    };
+    return { ...body, digest: decisionLoadManifestDigest(canonicalJson(body)) };
+}
+export function verifyDecisionLoadResultRecord(record, manifestDigest) {
+    const { digest, ...body } = record;
+    return record.schema === 'decision-load-result/v1' && record.manifest.digest === manifestDigest
+        && digest === decisionLoadManifestDigest(canonicalJson(body));
+}
+/**
+ * G5 evidence flags. Offline fake-provider records can prove the harness and
+ * the admission path, but only a passing staged-provider record bound to its
+ * manifest may set `load-manifest-qualified`.
+ */
+export function decisionLoadEvidenceFlags(records, manifestDigest) {
+    const bound = records.filter(record => verifyDecisionLoadResultRecord(record, manifestDigest));
+    const offline = bound.filter(record => record.mode === 'offline-fake-provider');
+    return {
+        'load-manifest-offline-passed': offline.length > 0 && offline.length === records.filter(record => record.mode === 'offline-fake-provider').length
+            && offline.every(record => record.passed),
+        'load-manifest-qualified': bound.some(record => record.mode === 'staged-provider' && record.passed)
+            && bound.length === records.length && bound.every(record => record.passed),
+    };
+}
+//# sourceMappingURL=load.js.map

package/dist/src/decision/qualification/manifest.js

@@ -0,0 +1,37 @@
+import { createHash } from 'node:crypto';
+const ids = (prefix, count) => Array.from({ length: count }, (_, index) => `${prefix}${String(index + 1).padStart(2, '0')}`);
+export const REQUIRED_BASELINE_CASE_IDS = ids('C', 42);
+export const REQUIRED_VENDOR_CASE_IDS = ids('TV', 25);
+export function expectedQualificationCaseIds() {
+    return [...REQUIRED_BASELINE_CASE_IDS, ...REQUIRED_VENDOR_CASE_IDS];
+}
+export function validateCaseInventory(cases) {
+    const actual = new Set();
+    const errors = [];
+    for (const item of cases) {
+        if (actual.has(item.id))
+            errors.push(`duplicate:${item.id}`);
+        actual.add(item.id);
+    }
+    for (const id of expectedQualificationCaseIds()) {
+        if (!actual.has(id))
+            errors.push(`missing:${id}`);
+    }
+    for (const id of actual) {
+        if (!expectedQualificationCaseIds().includes(id))
+            errors.push(`unexpected:${id}`);
+    }
+    return errors.sort();
+}
+export function stableManifestDigest(manifest) {
+    const canonical = JSON.stringify({
+        ...manifest,
+        cases: [...manifest.cases].sort((a, b) => a.id.localeCompare(b.id)),
+        evidence: [...manifest.evidence].sort((a, b) => a.caseId.localeCompare(b.caseId)),
+        evidenceFlags: Object.fromEntries(Object.entries(manifest.evidenceFlags).sort(([a], [b]) => a.localeCompare(b))),
+        ...(manifest.gateArtifacts ? { gateArtifacts: Object.fromEntries(Object.entries(manifest.gateArtifacts)
+                .sort(([a], [b]) => a.localeCompare(b))) } : {}),
+    });
+    return `sha256:${createHash('sha256').update(canonical).digest('hex')}`;
+}
+//# sourceMappingURL=manifest.js.map

package/dist/src/decision/qualification/privacy.js

@@ -0,0 +1,41 @@
+/** Qualification canary scan of sanitized text captures. Never put the canary value in diagnostics. */
+export const QUALIFICATION_PRIVACY_SURFACES = [
+    'stdout', 'stderr', 'test-report', 'trace', 'receipt', 'snapshot', 'export', 'thrown-error',
+    // Host-visible runtime activity: evidence callbacks, admission/prefix evidence and audit records.
+    'activity-record',
+];
+/**
+ * Scans raw and JSON-escaped spellings on each observed surface. An omitted
+ * surface is not assumed empty, and an empty canary list cannot prove privacy.
+ * The caller must collect streams/errors at their actual source; this function
+ * cannot infer that a supplied capture covers the whole process lifetime.
+ */
+export function scanQualificationPrivacy(captures, canaries) {
+    const missing = [];
+    const affected = [];
+    if (!canaries.length || canaries.some(value => typeof value !== 'string' || !value.length)) {
+        throw new Error('qualification privacy scan requires nonempty synthetic canaries');
+    }
+    const known = new Set(QUALIFICATION_PRIVACY_SURFACES);
+    const bySurface = new Map();
+    for (const item of captures) {
+        if (!known.has(item.surface) || bySurface.has(item.surface)) {
+            throw new Error('qualification privacy capture has unknown or duplicate surface');
+        }
+        if (typeof item.content !== 'string' && !(item.content instanceof Uint8Array)) {
+            throw new Error('qualification privacy capture requires text or bytes');
+        }
+        bySurface.set(item.surface, typeof item.content === 'string' ? item.content : new TextDecoder().decode(item.content));
+    }
+    for (const surface of QUALIFICATION_PRIVACY_SURFACES) {
+        const content = bySurface.get(surface);
+        if (content === undefined) {
+            missing.push(surface);
+        }
+        else if (canaries.some(canary => content.includes(canary) || content.includes(JSON.stringify(canary).slice(1, -1)))) {
+            affected.push(surface);
+        }
+    }
+    return { clean: !missing.length && !affected.length, missing, affected };
+}
+//# sourceMappingURL=privacy.js.map

package/dist/src/decision/qualification/quality.js

@@ -0,0 +1,213 @@
+import { createHash } from 'node:crypto';
+const sha256 = (value) => `sha256:${createHash('sha256').update(JSON.stringify(value)).digest('hex')}`;
+/** Freeze thresholds and all three label sets before acquiring test predictions. */
+export function freezeBinaryBenchmarkPlan(splits, labels, limits) {
+    verifyQualificationSplits(splits);
+    const ids = splits.flatMap(split => split.ids).sort();
+    if (labels.length !== ids.length || new Set(labels.map(label => label.id)).size !== ids.length
+        || labels.some(label => !ids.includes(label.id) || ![0, 1].includes(label.label) || !label.slice?.trim())
+        || !Number.isSafeInteger(limits.minimumOverallN) || limits.minimumOverallN < 1
+        || !Number.isSafeInteger(limits.minimumSliceN) || limits.minimumSliceN < 1
+        || [limits.maximumSelectiveRisk, limits.maximumReviewRate, limits.maximumBrier]
+            .some(value => !Number.isFinite(value) || value < 0 || value > 1)) {
+        throw new Error('invalid preregistered benchmark plan');
+    }
+    const fields = {
+        schemaVersion: 'decision-binary-benchmark-plan/v1',
+        splits: ['tuning', 'calibration', 'test'].map(name => splits.find(split => split.name === name)),
+        datasetDigest: sha256([...labels].sort((a, b) => a.id.localeCompare(b.id))),
+        ...limits,
+    };
+    return { ...fields, digest: sha256(fields) };
+}
+/** A separately anchored digest is required: a caller-created plan cannot attest its own preregistration. */
+export function evaluatePreregisteredBinaryBenchmark(plan, trustedPlanDigest, labels, samples) {
+    const { digest, ...fields } = plan;
+    if (digest !== trustedPlanDigest || digest !== sha256(fields)
+        || freezeBinaryBenchmarkPlan(plan.splits, labels, {
+            minimumOverallN: plan.minimumOverallN, minimumSliceN: plan.minimumSliceN,
+            maximumSelectiveRisk: plan.maximumSelectiveRisk, maximumReviewRate: plan.maximumReviewRate,
+            maximumBrier: plan.maximumBrier,
+        }).digest !== digest)
+        throw new Error('benchmark preregistration or dataset mismatch');
+    const testLabels = new Map(labels.filter(label => plan.splits.find(split => split.name === 'test').ids.includes(label.id))
+        .map(label => [label.id, label]));
+    if (samples.some(sample => sample.label !== testLabels.get(sample.id)?.label || sample.slice !== testLabels.get(sample.id)?.slice)) {
+        throw new Error('held-out label or slice mismatch');
+    }
+    const metrics = evaluateBinaryHeldout(plan.splits, samples);
+    const insufficient = metrics.overall.sampleN < plan.minimumOverallN
+        || Object.values(metrics.slices).some(slice => slice.sampleN < plan.minimumSliceN || slice.selectiveRisk === null);
+    const reasons = [];
+    if (insufficient)
+        reasons.push('insufficient-heldout-evidence');
+    if (metrics.overall.selectiveRisk === null)
+        reasons.push('zero-accepted-samples');
+    if (metrics.overall.selectiveRisk !== null && metrics.overall.selectiveRisk > plan.maximumSelectiveRisk)
+        reasons.push('selective-risk');
+    if (metrics.overall.reviewRate > plan.maximumReviewRate)
+        reasons.push('review-rate');
+    if (metrics.overall.brier > plan.maximumBrier)
+        reasons.push('brier');
+    const exceeded = reasons.some(reason => reason === 'selective-risk' || reason === 'review-rate' || reason === 'brier');
+    return { decision: exceeded ? 'fail' : insufficient || metrics.overall.selectiveRisk === null ? 'insufficient-evidence'
+            : 'pass', reasons, metrics };
+}
+function digestIds(ids) {
+    return `sha256:${createHash('sha256').update(JSON.stringify([...ids].sort())).digest('hex')}`;
+}
+/** Hash the exact membership set; duplicate, empty and overlapping IDs fail closed. */
+export function freezeQualificationSplit(name, ids) {
+    if (!ids.length || ids.some(id => !id.trim()) || new Set(ids).size !== ids.length) {
+        throw new Error('qualification split requires unique nonempty IDs');
+    }
+    return { name, ids: [...ids].sort(), digest: digestIds(ids) };
+}
+export function verifyQualificationSplits(splits) {
+    if (splits.length !== 3 || new Set(splits.map(split => split.name)).size !== 3
+        || splits.some(split => !['tuning', 'calibration', 'test'].includes(split.name))) {
+        throw new Error('qualification requires tuning, calibration and test splits');
+    }
+    const all = new Set();
+    for (const split of splits) {
+        if (!split.ids.length || split.ids.some(id => typeof id !== 'string' || !id.trim())
+            || new Set(split.ids).size !== split.ids.length || split.digest !== digestIds(split.ids)) {
+            throw new Error('qualification split digest or membership mismatch');
+        }
+        for (const id of split.ids) {
+            if (all.has(id))
+                throw new Error('qualification splits overlap');
+            all.add(id);
+        }
+    }
+}
+/** Refuses to score unregistered or non-held-out samples. */
+export function evaluateBinaryHeldout(splits, samples) {
+    verifyQualificationSplits(splits);
+    const test = splits.find(split => split.name === 'test');
+    if (samples.length !== test.ids.length || new Set(samples.map(sample => sample.id)).size !== samples.length
+        || samples.some(sample => !test.ids.includes(sample.id))) {
+        throw new Error('held-out sample membership mismatch');
+    }
+    for (const sample of samples) {
+        if (typeof sample.slice !== 'string' || !sample.slice.trim() || typeof sample.accepted !== 'boolean'
+            || ![0, 1].includes(sample.label)
+            || !Number.isFinite(sample.probability) || sample.probability < 0 || sample.probability > 1
+            || !Number.isFinite(sample.latencyMs) || sample.latencyMs < 0
+            || [sample.calls, sample.retries, sample.fallbacks].some(n => !Number.isSafeInteger(n) || n < 0)
+            || [sample.inputTokens, sample.outputTokens].some(n => n !== null && (!Number.isSafeInteger(n) || n < 0))
+            || (sample.costUsd !== null && (!Number.isFinite(sample.costUsd) || sample.costUsd < 0))) {
+            throw new Error('invalid held-out sample');
+        }
+    }
+    const groups = new Map();
+    for (const sample of samples)
+        groups.set(sample.slice, [...(groups.get(sample.slice) ?? []), sample]);
+    return {
+        overall: scoreBinary(samples),
+        slices: Object.fromEntries([...groups].sort(([a], [b]) => a.localeCompare(b)).map(([key, group]) => [key, scoreBinary(group)])),
+    };
+}
+/** Mean absolute level error and exact match, scored on a frozen held-out split. */
+export function evaluateOrdinalHeldout(splits, samples) {
+    verifyQualificationSplits(splits);
+    const testIds = splits.find(split => split.name === 'test').ids;
+    if (samples.length !== testIds.length || new Set(samples.map(s => s.id)).size !== samples.length
+        || samples.some(s => !testIds.includes(s.id)))
+        throw new Error('held-out ordinal membership mismatch');
+    if (samples.some(s => !Number.isSafeInteger(s.levels) || s.levels < 2
+        || !Number.isSafeInteger(s.trueLevel) || !Number.isSafeInteger(s.predictedLevel)
+        || s.trueLevel < 0 || s.trueLevel >= s.levels || s.predictedLevel < 0 || s.predictedLevel >= s.levels)) {
+        throw new Error('invalid held-out ordinal sample');
+    }
+    return {
+        sampleN: samples.length,
+        exactRate: samples.filter(s => s.trueLevel === s.predictedLevel).length / samples.length,
+        meanAbsoluteError: samples.reduce((sum, s) => sum + Math.abs(s.trueLevel - s.predictedLevel), 0) / samples.length,
+        normalizedAbsoluteError: samples.reduce((sum, s) => sum + Math.abs(s.trueLevel - s.predictedLevel) / (s.levels - 1), 0) / samples.length,
+    };
+}
+/** Counts gold-comparable option pairs; ties in the prediction are errors, not wins. */
+export function evaluateRankingHeldout(splits, samples) {
+    verifyQualificationSplits(splits);
+    const testIds = splits.find(split => split.name === 'test').ids;
+    if (samples.length !== testIds.length || new Set(samples.map(s => s.id)).size !== samples.length
+        || samples.some(s => !testIds.includes(s.id)))
+        throw new Error('held-out ranking membership mismatch');
+    let pairs = 0;
+    let correct = 0;
+    for (const sample of samples) {
+        const options = Object.keys(sample.gold).sort();
+        if (options.length < 2 || options.length !== Object.keys(sample.predicted).length
+            || options.some(option => !Object.hasOwn(sample.predicted, option))) {
+            throw new Error('ranking option mismatch');
+        }
+        for (const option of options) {
+            if (!Number.isFinite(sample.gold[option]) || !Number.isFinite(sample.predicted[option])) {
+                throw new Error('invalid ranking score');
+            }
+        }
+        for (let i = 0; i < options.length; i++)
+            for (let j = i + 1; j < options.length; j++) {
+                const goldDifference = sample.gold[options[i]] - sample.gold[options[j]];
+                if (goldDifference === 0)
+                    continue;
+                pairs++;
+                const predictedDifference = sample.predicted[options[i]] - sample.predicted[options[j]];
+                if (Math.sign(predictedDifference) === Math.sign(goldDifference))
+                    correct++;
+            }
+    }
+    return { sampleN: samples.length, comparablePairs: pairs, concordance: pairs ? correct / pairs : null };
+}
+/** Measures categorical movement without claiming that either output is correct. */
+export function measurePairedMovement(pairs) {
+    if (!pairs.length || new Set(pairs.map(pair => pair.id)).size !== pairs.length
+        || pairs.some(pair => !pair.id.trim() || !pair.control.trim() || !pair.observed.trim())) {
+        throw new Error('paired qualification requires unique nonempty IDs and outcomes');
+    }
+    const changedN = pairs.filter(pair => pair.control !== pair.observed).length;
+    const [low, high] = wilson95(changedN, pairs.length);
+    return { sampleN: pairs.length, changedN, changedRate: changedN / pairs.length, changedWilson95: [low, high] };
+}
+function wilson95(errors, n) {
+    // 95% normal quantile; finite-sample Wilson interval for binomial events.
+    const z = 1.959963984540054;
+    const rate = errors / n;
+    const denominator = 1 + z * z / n;
+    const center = (rate + z * z / (2 * n)) / denominator;
+    const margin = z * Math.sqrt(rate * (1 - rate) / n + z * z / (4 * n * n)) / denominator;
+    return [Math.max(0, center - margin), Math.min(1, center + margin)];
+}
+function scoreBinary(samples) {
+    const n = samples.length;
+    const errors = samples.filter(s => (s.probability >= 0.5 ? 1 : 0) !== s.label).length;
+    const accepted = samples.filter(s => s.accepted);
+    const acceptedErrors = accepted.filter(s => (s.probability >= 0.5 ? 1 : 0) !== s.label).length;
+    const rate = errors / n;
+    const sorted = samples.map(s => s.latencyMs).sort((a, b) => a - b);
+    const quantile = (q) => sorted[Math.ceil(q * n) - 1];
+    // Deciles are a reporting convention, not a calibrated decision threshold.
+    const buckets = Array.from({ length: 10 }, () => []);
+    for (const sample of samples)
+        buckets[Math.min(9, Math.floor(sample.probability * 10))].push(sample);
+    const sumKnown = (key) => samples.some(s => s[key] === null) ? null : samples.reduce((sum, s) => sum + (s[key] ?? 0), 0);
+    return {
+        sampleN: n, errorRate: rate,
+        brier: samples.reduce((sum, s) => sum + (s.probability - s.label) ** 2, 0) / n,
+        logLoss: samples.reduce((sum, s) => sum - (s.label ? Math.log(Math.max(s.probability, Number.EPSILON))
+            : Math.log(Math.max(1 - s.probability, Number.EPSILON))), 0) / n,
+        expectedCalibrationError: buckets.reduce((sum, bucket) => bucket.length ? sum + bucket.length / n
+            * Math.abs(bucket.reduce((a, s) => a + s.probability, 0) / bucket.length
+                - bucket.reduce((a, s) => a + s.label, 0) / bucket.length) : sum, 0),
+        coverage: accepted.length / n, selectiveRisk: accepted.length ? acceptedErrors / accepted.length : null,
+        reviewRate: (n - accepted.length) / n,
+        errorWilson95: wilson95(errors, n),
+        latencyMs: { p50: quantile(0.5), p95: quantile(0.95), p99: quantile(0.99) },
+        inputTokens: sumKnown('inputTokens'), outputTokens: sumKnown('outputTokens'), costUsd: sumKnown('costUsd'),
+        calls: samples.reduce((sum, s) => sum + s.calls, 0),
+        retries: samples.reduce((sum, s) => sum + s.retries, 0),
+        fallbacks: samples.reduce((sum, s) => sum + s.fallbacks, 0),
+    };
+}
+//# sourceMappingURL=quality.js.map