@aiwg/cli 2026.9.21 → 2026.10.0

package/dist/src/decision/ensemble-study/score.js

@@ -0,0 +1,462 @@
+import { heldoutDigest, validateHeldoutAttempt, validateHeldoutInputs } from '../heldout/contract.js';
+import { artifactPin } from '../validate.js';
+import { aggregateEnsembleResults } from '../ensemble/aggregate.js';
+import { validateChampionChallenger, pairedThresholdsDigest } from '../ensemble/contract.js';
+import { buildEnsembleIntegrityReport } from '../ensemble/report.js';
+import { championChallengerInputSetDigest, championChallengerShadowBaseline, runChampionChallengerShadow } from '../ensemble/runtime.js';
+import { qualificationIntegrityAllowlistProblems } from '../qualification/release.js';
+import { evaluateBinaryHeldout, freezeQualificationSplit } from '../qualification/quality.js';
+import { D17_ANALYSIS } from './protocol.js';
+import { d17Statistics } from './statistics.js';
+import { validateD17Artifact } from './artifacts.js';
+import { applyD17Mapping, assertD17Staged, d17AttemptsByRow, d17RawProbabilities, verifyD17CalibrationSet } from './calibration.js';
+import { calibrationIdentityDigest } from '../calibration/registry.js';
+import { canonicalJson } from '../../security/artifact-trust.js';
+function policy(prepared, rowAttempts) {
+    const first = rowAttempts.find(attempt => attempt.result?.receipt)?.result?.receipt;
+    if (!first)
+        return null;
+    const definition = artifactPin(prepared.corpus.definitions[0]);
+    return { schemaVersion: 'decision-ensemble-policy/v1', id: 'd17-synthetic-repeated-sample', version: '1.0.0',
+        mode: 'offline-shadow', riskTiers: ['synthetic-diagnostic'], definition, primitive: 'choice', options: ['no', 'yes'],
+        compatibleUncertaintyProfiles: ['typesafe-distribution-v1'], requiredCapabilities: ['choice', 'probability-distribution'],
+        members: [{ id: 'jev', memberType: 'repeated-sample', definition, binding: first.spec.binding,
+                adapter: { id: 'jev', version: '1.0.0' }, model: { provider: 'jev', backend: 'api', requested: 'jev-1.13.0', pinnedVersion: 'jev-1.13.0' },
+                primitive: 'choice', uncertaintyProfile: 'typesafe-distribution-v1', requiredCapabilities: ['choice', 'probability-distribution'],
+                capabilities: ['choice', 'probability-distribution'], calibration: null, samples: 3, fallbackDepth: 0,
+                estimate: { attemptsPerSample: 2, tokensPerAttempt: 4000, costMicrosPerAttempt: 400, deadlineMsPerAttempt: prepared.preregistration.requestTimeoutMs },
+                approvalReference: null }], aggregation: { algorithm: 'mean-probability-v1', tieRule: D17_ANALYSIS.acceptance.tieRule },
+        disagreement: { metric: D17_ANALYSIS.acceptance.disagreementMetric, thresholdBps: D17_ANALYSIS.acceptance.maximumDisagreementBps, onExceeded: 'defer' },
+        acceptance: { minimumSuccessfulMembers: D17_ANALYSIS.acceptance.minimumSuccessfulMembers, onInsufficientMembers: 'defer',
+            highAgreementWarningBps: D17_ANALYSIS.acceptance.highAgreementWarningBps }, calibration: { requirement: 'advisory' },
+        ceilings: { members: 1, attempts: 6, deadlineMs: prepared.preregistration.requestTimeoutMs * 6, tokens: 24000, costMicros: 2400,
+            concurrency: 1, fallbackDepth: 0, unknownCost: { rule: 'reject' } } };
+}
+/** Maps retained provider distributions only; confidence and agreement never become calibrated probability. */
+export async function scoreD17Study(input, prepared) {
+    validateHeldoutInputs(input.corpus, input.preregistration);
+    if (input.preregistration.calibration.scope !== 'uncalibrated-diagnostic'
+        || !input.approvedCalibration || input.approvedCalibration.mode !== 'uncalibrated-diagnostic'
+        || Object.keys(input.approvedCalibration).join(',') !== 'mode' || input.calibrated !== false) {
+        throw new Error('D17 uncalibrated diagnostic scope');
+    }
+    if (heldoutDigest(input.corpus) !== heldoutDigest(prepared.corpus)
+        || heldoutDigest(input.preregistration) !== heldoutDigest(prepared.preregistration)
+        || heldoutDigest(input.gold) !== input.corpus.provenance.goldDigest
+        || heldoutDigest(prepared.gold) !== input.corpus.provenance.goldDigest)
+        throw new Error('D17 frozen scoring pins');
+    const integrityProblems = qualificationIntegrityAllowlistProblems(input.integrity);
+    if (integrityProblems.includes('integrity-invalid'))
+        throw new Error('D17 invalid integrity');
+    const rows = new Map(input.corpus.rows.map(row => [row.id, row]));
+    const seen = new Set();
+    for (const attempt of input.attempts) {
+        validateHeldoutAttempt(attempt);
+        const row = rows.get(attempt.rowId), key = `${attempt.rowId}/${attempt.requestId}/${attempt.ordinal}`;
+        if (!row || !row.requests.some(request => request.id === attempt.requestId) || seen.has(key)
+            || attempt.corpusDigest !== input.preregistration.corpusDigest || attempt.study !== 'D17'
+            || attempt.preregistrationDigest !== heldoutDigest(input.preregistration))
+            throw new Error('D17 attempt membership');
+        seen.add(key);
+    }
+    const armAttempts = (row, candidate) => input.attempts.filter(attempt => attempt.rowId === row.id
+        && (candidate ? attempt.requestId.startsWith('member_') : attempt.requestId === 'champion'));
+    const member = (row, requestId) => {
+        const attempts = input.attempts.filter(attempt => attempt.rowId === row.id && attempt.requestId === requestId).sort((a, b) => a.ordinal - b.ordinal);
+        const terminal = attempts.at(-1)?.result;
+        const decision = terminal?.receipt?.spec.evaluations.q0;
+        const uncertainty = decision?.spec.uncertainty;
+        const ok = terminal?.disposition === 'success' && terminal.servedModel === 'jev-1.13.0'
+            && decision?.spec.status === 'success' && uncertainty?.source === 'provider'
+            && uncertainty.profile === 'typesafe-distribution-v1' && uncertainty.distribution !== null;
+        if (ok && (Object.keys(uncertainty.distribution).sort().join(',') !== 'no,yes'
+            || !['yes', 'no'].includes(String(decision.spec.value))))
+            throw new Error('D17 native distribution');
+        return { memberId: 'jev', sampleIndex: requestId === 'champion' ? 0 : Number(requestId.slice(-1)) - 1,
+            status: ok ? 'succeeded' : 'failed', resultDigest: heldoutDigest(attempts), value: ok ? decision.spec.value : null,
+            distribution: ok ? uncertainty.distribution : null, uncertaintyProfile: ok ? uncertainty.profile : null };
+    };
+    const measured = (probability, accepted, value, gold, attempts) => ({
+        probability, accepted, correct: accepted && value === gold, brier: (probability - Number(gold === 'yes')) ** 2,
+        latencyMs: attempts.reduce((sum, attempt) => sum + (attempt.result?.latencyMs ?? 0), 0),
+        tokens: attempts.some(attempt => !attempt.result || attempt.result.inputTokens === null || attempt.result.outputTokens === null) ? null
+            : attempts.reduce((sum, attempt) => sum + attempt.result.inputTokens + attempt.result.outputTokens, 0),
+        reservedCostMicros: attempts.reduce((sum, attempt) => sum + Math.max(attempt.reservedUsdMicros, attempt.result?.accountedUsdMicros ?? 0), 0),
+    });
+    const pairs = [], observations = [];
+    let championCorrect = 0, challengerCorrect = 0;
+    const test = input.corpus.rows.filter(row => row.split === 'test');
+    for (const row of test) {
+        const single = member(row, 'champion'), members = [1, 2, 3].map(n => member(row, `member_${n}`));
+        const candidateAttempts = armAttempts(row, true), championAttempts = armAttempts(row, false);
+        const ensemblePolicy = policy(prepared, candidateAttempts);
+        const aggregate = ensemblePolicy ? aggregateEnsembleResults(ensemblePolicy, members) : null;
+        const gold = prepared.gold.labels[row.id];
+        const champion = single.status === 'succeeded' ? measured(single.distribution.yes, single.distribution.yes !== single.distribution.no, single.value, gold, championAttempts) : null;
+        const challenger = members.every(m => m.status === 'succeeded') && aggregate?.statistics.meanDistribution
+            ? measured(aggregate.statistics.meanDistribution.yes, aggregate.outcome.disposition === 'accept', aggregate.outcome.value, gold, candidateAttempts) : null;
+        championCorrect += Number(champion?.correct ?? false);
+        challengerCorrect += Number(challenger?.correct ?? false);
+        if (champion && challenger)
+            pairs.push({ id: row.id, slice: row.slice, champion, challenger });
+        const observation = (arm, attempts) => arm ? {
+            inputDigest: heldoutDigest(row.input), receiptDigest: heldoutDigest(attempts), quality: Number(arm.correct), calibration: arm.brier,
+            riskCoverage: Number(arm.accepted && !arm.correct), abstention: Number(!arm.accepted), latencyMs: arm.latencyMs,
+            tokens: arm.tokens, costMicros: arm.reservedCostMicros, slice: Number(arm.correct),
+        } : null;
+        observations.push({ id: row.id, champion: observation(champion, championAttempts), challenger: observation(challenger, candidateAttempts), aggregate });
+    }
+    const statistics = d17Statistics(pairs, test.map(row => row.id));
+    statistics.worstCaseFailureAsError.championCorrect = championCorrect;
+    statistics.worstCaseFailureAsError.challengerCorrect = challengerCorrect;
+    const splits = ['tuning', 'calibration', 'test'].map(split => freezeQualificationSplit(split, input.corpus.rows.filter(row => row.split === split).map(row => row.id)));
+    const binary = (role) => pairs.length === test.length ? evaluateBinaryHeldout(splits, pairs.map(pair => {
+        const attempts = armAttempts(rows.get(pair.id), role === 'challenger');
+        const known = (key) => attempts.some(a => !a.result || a.result[key] === null)
+            ? null : attempts.reduce((sum, a) => sum + a.result[key], 0);
+        return { id: pair.id, slice: pair.slice, label: Number(prepared.gold.labels[pair.id] === 'yes'),
+            probability: pair[role].probability, accepted: pair[role].accepted, latencyMs: pair[role].latencyMs,
+            inputTokens: known('inputTokens'), outputTokens: known('outputTokens'), costUsd: known('providerCostUsd'),
+            calls: attempts.length, retries: attempts.filter(a => a.ordinal > 1).length, fallbacks: 0 };
+    })) : null;
+    const reservation = (attempts) => attempts.reduce((sum, attempt) => sum + Math.max(attempt.reservedUsdMicros, attempt.result?.accountedUsdMicros ?? 0), 0);
+    const definitionPin = artifactPin(input.corpus.definitions[0]);
+    const successful = input.attempts.filter(attempt => attempt.result?.disposition === 'success');
+    const bindingPin = successful[0]?.result?.receipt?.spec.binding ?? null;
+    if (successful.some(attempt => !attempt.result?.receipt
+        || heldoutDigest(attempt.result.receipt.spec.binding) !== heldoutDigest(bindingPin)
+        || Object.values(attempt.result.receipt.spec.evaluations).some(result => heldoutDigest(result.spec.decision) !== heldoutDigest(definitionPin)))) {
+        throw new Error('D17 mixed execution pins');
+    }
+    const testIds = new Set(test.map(row => row.id));
+    const testAttempts = input.attempts.filter(attempt => testIds.has(attempt.rowId));
+    const championReservations = reservation(testAttempts.filter(attempt => attempt.requestId === 'champion'));
+    const challengerReservations = reservation(testAttempts.filter(attempt => attempt.requestId !== 'champion'));
+    const report = { schemaVersion: 'decision-d17-study-report/v1', scope: 'synthetic-diagnostic-only',
+        calibrated: false, d09Qualified: false, calibratedGate: false,
+        corpusDigest: heldoutDigest(input.corpus), preregistrationDigest: heldoutDigest(input.preregistration),
+        executionPins: { definition: definitionPin, binding: bindingPin, model: 'jev-1.13.0', adapter: { id: 'jev', version: '1.0.0' },
+            aggregatePolicyDigest: observations.find(row => row.aggregate)?.aggregate?.policy.digest ?? null },
+        integrityDigest: heldoutDigest(input.integrity), integrity: structuredClone(input.integrity), integrityProblems,
+        statistics, statisticsDigest: heldoutDigest(statistics), pairs, observations,
+        accounting: { allSplits: { attempts: input.attempts.length, reservedCostMicros: reservation(input.attempts) },
+            test: { championReservedCostMicros: championReservations, challengerReservedCostMicros: challengerReservations,
+                netSavingsMicros: championReservations - challengerReservations, complete: pairs.length === test.length } },
+        calibrationDiagnostics: { champion: binary('champion'), challenger: binary('challenger'), calibrated: false },
+        pending: ['D09 compatible member and aggregate calibration', 'blind gold audit and delayed repeats', 'separate extra-cost tradeoff approval',
+            'protected native report approval and eligibility'],
+        decision: input.integrity.release_gate.decision === 'ROLLBACK' || input.integrity.compromise_labels.length ? 'ROLLBACK' : 'HOLD' };
+    validateD17Artifact('report', report);
+    return report;
+}
+/** Native serialization requires a separately anchored record. It cannot create approval or eligibility. */
+export async function buildD17NativeReport(input) {
+    const staged = input.report.schemaVersion === 'decision-d17-study-report/v2';
+    validateD17Artifact(staged ? 'reportV2' : 'report', input.report);
+    if (!staged && (input.eligibility !== undefined || input.tradeoff !== undefined))
+        throw new Error('D17 diagnostic native report has no promotion route');
+    if (input.eligibility !== undefined) {
+        const stored = input.registry?.promotionEligibility(input.record.eligibilityId) ?? null;
+        if (!stored || canonicalJson(stored) !== canonicalJson(input.eligibility))
+            throw new Error('D17 native eligibility');
+    }
+    if (staged) {
+        // AC7/AC14: each role must cite the exact D09 artifact the calibrated report applied to it.
+        const calibration = input.report.calibration, record = input.record;
+        const cites = (role, pin) => record[role]?.calibration?.artifactId === pin.artifactId && record[role]?.calibration?.artifactDigest === pin.artifactDigest;
+        assertD17Staged(input.prepared);
+        if (!cites('champion', calibration.member) || !cites('challenger', calibration.aggregate))
+            throw new Error('D17 native calibration pins');
+    }
+    const { record } = validateChampionChallenger(input.record);
+    const problems = qualificationIntegrityAllowlistProblems(input.integrity);
+    if (problems.includes('integrity-invalid') || heldoutDigest(input.integrity) !== input.trustedIntegrityDigest
+        || record.evaluationIntegrityReport.digest !== input.trustedIntegrityDigest)
+        throw new Error('D17 native integrity pin');
+    if (heldoutDigest(input.report) !== input.trustedReportDigest || input.report.integrityDigest !== heldoutDigest(input.report.integrity)
+        || input.report.corpusDigest !== heldoutDigest(input.prepared.corpus)
+        || input.report.preregistrationDigest !== heldoutDigest(input.prepared.preregistration)
+        || input.report.statisticsDigest !== heldoutDigest(input.report.statistics)
+        || record.preregistration.thresholdsDigest !== pairedThresholdsDigest([...D17_ANALYSIS.native])
+        || input.integrity.release_gate.decision === 'PROMOTE'
+            && (!staged || !d17PromotionSupported(input.report, input.integrity, input.tradeoff, input.trustedTradeoffDigest))
+        || input.report.decision === 'ROLLBACK' && input.integrity.release_gate.decision !== 'ROLLBACK')
+        throw new Error('D17 native report needs anchored HOLD or ROLLBACK');
+    const pins = input.report.executionPins;
+    if (!pins.binding || !pins.aggregatePolicyDigest || record.champion.ensemblePolicy !== null
+        || record.challenger.ensemblePolicy?.digest !== pins.aggregatePolicyDigest
+        || [record.champion, record.challenger].some(role => role.actualModel !== pins.model
+            || heldoutDigest(role.binding) !== heldoutDigest(pins.binding) || heldoutDigest(role.adapter) !== heldoutDigest(pins.adapter))) {
+        throw new Error('D17 native execution pins');
+    }
+    const items = input.prepared.corpus.rows.filter(row => row.split === 'test').map(row => ({ id: row.id, input: row.input, slice: row.slice }));
+    if (record.inputSet.digest !== championChallengerInputSetDigest(items))
+        throw new Error('D17 native membership');
+    const shadow = await runChampionChallengerShadow(record, { enabled: true, invocationId: 'd17-recorded-replay', items,
+        evaluate: async ({ role, item }) => {
+            const observation = input.report.observations.find(row => row.id === item.id)?.[role];
+            if (!observation || observation.tokens === null)
+                throw new Error('D17 incomplete native endpoints');
+            return observation;
+        } });
+    const baseline = championChallengerShadowBaseline(record, shadow);
+    const upstream = input.integrity.paired_baseline;
+    if (!upstream || heldoutDigest(upstream.championChallengerShadow ?? null) !== heldoutDigest(baseline)
+        || upstream.d17StatisticsDigest !== input.report.statisticsDigest)
+        throw new Error('D17 unbound native statistics');
+    return buildEnsembleIntegrityReport({ record, integrity: input.integrity, pairedDeltas: shadow.pairedDeltas, eligibility: input.eligibility ?? null });
+}
+/**
+ * Calibrated test-phase scorer (#2611 staged D09). The collector passes only test rows and `calibrated: null`.
+ * The approved calibration set, both approved artifacts, their mappings and the completed development review
+ * are verified and D09-qualified at the scoring clock before any row is scored. The champion's native P(yes)
+ * passes through the member calibrator; the challenger's raw three-member mean passes through the aggregate
+ * calibrator, and it is accepted only when the native aggregate accepts and the calibrated value is not a tie.
+ * The frozen D17 native gates are evaluated on those calibrated measurements; the uncalibrated statistics are
+ * retained as a descriptive comparison. The decision is HOLD or a preserved ROLLBACK, never PROMOTE.
+ */
+export async function scoreD17StagedStudy(input, prepared, context) {
+    assertD17Staged(prepared);
+    if (input.preregistration?.calibration?.scope !== 'calibrated' || input.calibrated !== null)
+        throw new Error('D17 staged calibration scope');
+    const testRows = prepared.corpus.rows.filter(row => row.split === 'test');
+    if (heldoutDigest(input.corpus) !== heldoutDigest({ ...prepared.corpus, rows: testRows })
+        || heldoutDigest(input.preregistration) !== heldoutDigest(prepared.preregistration)
+        || heldoutDigest(input.gold) !== prepared.corpus.provenance.goldDigest
+        || heldoutDigest(prepared.gold) !== prepared.corpus.provenance.goldDigest)
+        throw new Error('D17 frozen scoring pins');
+    if (!context || typeof context.testPhaseAccessAt !== 'string')
+        throw new Error('D17 staged test access');
+    const verified = verifyD17CalibrationSet(prepared, input.approvedCalibration, context);
+    const integrityProblems = qualificationIntegrityAllowlistProblems(input.integrity);
+    if (integrityProblems.includes('integrity-invalid'))
+        throw new Error('D17 invalid integrity');
+    const rows = new Map(testRows.map(row => [row.id, row]));
+    const seen = new Set();
+    for (const attempt of input.attempts) {
+        validateHeldoutAttempt(attempt);
+        const row = rows.get(attempt.rowId), key = `${attempt.rowId}/${attempt.requestId}/${attempt.ordinal}`;
+        if (!row || !row.requests.some(request => request.id === attempt.requestId) || seen.has(key)
+            || attempt.corpusDigest !== input.preregistration.corpusDigest || attempt.study !== 'D17'
+            || attempt.preregistrationDigest !== heldoutDigest(input.preregistration))
+            throw new Error('D17 attempt membership');
+        seen.add(key);
+    }
+    const grouped = d17AttemptsByRow(input.attempts), rowAttempts = (row) => grouped.get(row.id) ?? [];
+    const of = (row, candidate) => rowAttempts(row).filter(attempt => candidate ? attempt.requestId.startsWith('member_') : attempt.requestId === 'champion');
+    const member = (row, requestId) => {
+        const attempts = rowAttempts(row).filter(attempt => attempt.requestId === requestId).sort((a, b) => a.ordinal - b.ordinal);
+        const terminal = attempts.at(-1)?.result, decision = terminal?.receipt?.spec.evaluations.q0, uncertainty = decision?.spec.uncertainty;
+        const ok = terminal?.disposition === 'success' && terminal.servedModel === 'jev-1.13.0' && decision?.spec.status === 'success'
+            && uncertainty?.source === 'provider' && uncertainty.profile === 'typesafe-distribution-v1' && uncertainty.distribution !== null;
+        if (ok && (Object.keys(uncertainty.distribution).sort().join(',') !== 'no,yes' || !['yes', 'no'].includes(String(decision.spec.value))))
+            throw new Error('D17 native distribution');
+        return { memberId: 'jev', sampleIndex: requestId === 'champion' ? 0 : Number(requestId.slice(-1)) - 1,
+            status: ok ? 'succeeded' : 'failed', resultDigest: heldoutDigest(attempts), value: ok ? decision.spec.value : null,
+            distribution: ok ? uncertainty.distribution : null, uncertaintyProfile: ok ? uncertainty.profile : null };
+    };
+    const arm = (probability, accepted, value, gold, attempts) => ({
+        probability, accepted, correct: accepted && value === gold, brier: (probability - Number(gold === 'yes')) ** 2,
+        latencyMs: attempts.reduce((sum, attempt) => sum + (attempt.result?.latencyMs ?? 0), 0),
+        tokens: attempts.some(attempt => !attempt.result || attempt.result.inputTokens === null || attempt.result.outputTokens === null) ? null
+            : attempts.reduce((sum, attempt) => sum + attempt.result.inputTokens + attempt.result.outputTokens, 0),
+        reservedCostMicros: attempts.reduce((sum, attempt) => sum + Math.max(attempt.reservedUsdMicros, attempt.result?.accountedUsdMicros ?? 0), 0),
+    });
+    const calibratedValue = (p) => p > 0.5 ? 'yes' : p < 0.5 ? 'no' : null;
+    const pairs = [], rawPairs = [], observations = [];
+    let championCorrect = 0, challengerCorrect = 0;
+    for (const row of testRows) {
+        const single = member(row, 'champion'), members = [1, 2, 3].map(n => member(row, `member_${n}`));
+        const candidateAttempts = of(row, true), championAttempts = of(row, false);
+        const ensemblePolicy = policy(prepared, candidateAttempts);
+        const aggregate = ensemblePolicy ? aggregateEnsembleResults(ensemblePolicy, members) : null;
+        const gold = prepared.gold.labels[row.id], raw = d17RawProbabilities(rowAttempts(row), row);
+        let champion = null, challenger = null;
+        if (single.status === 'succeeded' && raw.champion !== null) {
+            const p = applyD17Mapping(verified.member.mapping, raw.champion);
+            champion = arm(p, calibratedValue(p) !== null, calibratedValue(p), gold, championAttempts);
+        }
+        if (members.every(m => m.status === 'succeeded') && aggregate?.statistics.meanDistribution && raw.mean !== null) {
+            const p = applyD17Mapping(verified.aggregate.mapping, raw.mean);
+            challenger = arm(p, aggregate.outcome.disposition === 'accept' && calibratedValue(p) !== null, calibratedValue(p), gold, candidateAttempts);
+        }
+        // Uncalibrated comparison: the original v1 measurements, descriptive only.
+        const rawChampion = single.status === 'succeeded' ? arm(single.distribution.yes, single.distribution.yes !== single.distribution.no, single.value, gold, championAttempts) : null;
+        const rawChallenger = members.every(m => m.status === 'succeeded') && aggregate?.statistics.meanDistribution ? arm(aggregate.statistics.meanDistribution.yes, aggregate.outcome.disposition === 'accept', aggregate.outcome.value, gold, candidateAttempts) : null;
+        if (rawChampion && rawChallenger)
+            rawPairs.push({ id: row.id, slice: row.slice, champion: rawChampion, challenger: rawChallenger });
+        championCorrect += Number(champion?.correct ?? false);
+        challengerCorrect += Number(challenger?.correct ?? false);
+        if (champion && challenger)
+            pairs.push({ id: row.id, slice: row.slice, champion, challenger });
+        const observation = (measured, attempts) => measured ? {
+            inputDigest: heldoutDigest(row.input), receiptDigest: heldoutDigest(attempts), quality: Number(measured.correct), calibration: measured.brier,
+            riskCoverage: Number(measured.accepted && !measured.correct), abstention: Number(!measured.accepted), latencyMs: measured.latencyMs,
+            tokens: measured.tokens, costMicros: measured.reservedCostMicros, slice: Number(measured.correct),
+        } : null;
+        observations.push({ id: row.id, champion: observation(champion, championAttempts), challenger: observation(challenger, candidateAttempts), aggregate });
+    }
+    const ids = testRows.map(row => row.id);
+    const statistics = d17Statistics(pairs, ids);
+    statistics.worstCaseFailureAsError.championCorrect = championCorrect;
+    statistics.worstCaseFailureAsError.challengerCorrect = challengerCorrect;
+    const uncalibratedStatistics = d17Statistics(rawPairs, ids);
+    const splits = ['tuning', 'calibration', 'test'].map(split => freezeQualificationSplit(split, prepared.corpus.rows.filter(row => row.split === split).map(row => row.id)));
+    const binary = (role) => pairs.length === testRows.length ? evaluateBinaryHeldout(splits, pairs.map(pair => {
+        const attempts = of(rows.get(pair.id), role === 'challenger');
+        const known = (key) => attempts.some(a => !a.result || a.result[key] === null)
+            ? null : attempts.reduce((sum, a) => sum + a.result[key], 0);
+        return { id: pair.id, slice: pair.slice, label: Number(prepared.gold.labels[pair.id] === 'yes'),
+            probability: pair[role].probability, accepted: pair[role].accepted, latencyMs: pair[role].latencyMs,
+            inputTokens: known('inputTokens'), outputTokens: known('outputTokens'), costUsd: known('providerCostUsd'),
+            calls: attempts.length, retries: attempts.filter(a => a.ordinal > 1).length, fallbacks: 0 };
+    })) : null;
+    const reservation = (attempts) => attempts.reduce((sum, attempt) => sum + Math.max(attempt.reservedUsdMicros, attempt.result?.accountedUsdMicros ?? 0), 0);
+    const definitionPin = artifactPin(prepared.corpus.definitions[0]);
+    const successful = input.attempts.filter(attempt => attempt.result?.disposition === 'success');
+    const bindingPin = successful[0]?.result?.receipt?.spec.binding ?? null;
+    if (successful.some(attempt => !attempt.result?.receipt || heldoutDigest(attempt.result.receipt.spec.binding) !== heldoutDigest(bindingPin)
+        || Object.values(attempt.result.receipt.spec.evaluations).some(result => heldoutDigest(result.spec.decision) !== heldoutDigest(definitionPin)))) {
+        throw new Error('D17 mixed execution pins');
+    }
+    const championReservations = reservation(input.attempts.filter(attempt => attempt.requestId === 'champion'));
+    const challengerReservations = reservation(input.attempts.filter(attempt => attempt.requestId !== 'champion'));
+    const pin = (role) => ({ artifactId: verified[role].artifact.id, artifactDigest: verified[role].artifact.digest,
+        mappingDigest: heldoutDigest(verified[role].mapping), compatibility: { action: 'allow',
+            state: verified[role].compatibility.state, pinId: verified[role].compatibility.pinId } });
+    const approved = input.approvedCalibration;
+    const decision = input.integrity.release_gate.decision === 'ROLLBACK' || input.integrity.compromise_labels.length ? 'ROLLBACK' : 'HOLD';
+    const report = { schemaVersion: 'decision-d17-study-report/v2', scope: 'synthetic-staged-calibrated',
+        calibrated: true, d09Qualified: true, calibratedGate: true,
+        corpusDigest: heldoutDigest(prepared.corpus), preregistrationDigest: heldoutDigest(prepared.preregistration),
+        executionPins: { definition: definitionPin, binding: bindingPin, model: 'jev-1.13.0', adapter: { id: 'jev', version: '1.0.0' },
+            aggregatePolicyDigest: observations.find(row => row.aggregate)?.aggregate?.policy.digest ?? null },
+        integrityDigest: heldoutDigest(input.integrity), integrity: structuredClone(input.integrity), integrityProblems,
+        calibration: { mode: 'staged-test', calibrationSetDigest: verified.setDigest,
+            calibrationPhaseRecordDigest: approved.calibrationPhaseRecordDigest, priorApprovalDigest: approved.priorApprovalDigest,
+            member: pin('member'), aggregate: pin('aggregate'), qualifiedAt: verified.qualifiedAt,
+            applied: { champion: 'member', challenger: 'aggregate' } },
+        developmentReviewDigest: verified.developmentReviewDigest, calibrationReviewDigest: verified.calibrationReviewDigest,
+        statistics, statisticsDigest: heldoutDigest(statistics), uncalibratedStatistics, pairs, observations,
+        accounting: { allSplits: { attempts: input.attempts.length, reservedCostMicros: reservation(input.attempts) },
+            test: { championReservedCostMicros: championReservations, challengerReservedCostMicros: challengerReservations,
+                netSavingsMicros: championReservations - challengerReservations, complete: pairs.length === testRows.length } },
+        calibrationDiagnostics: { champion: binary('champion'), challenger: binary('challenger'), calibrated: true },
+        gates: { source: 'd17-native-protocol', gateBinding: null, statisticalGate: statistics.statisticalGate,
+            findings: [...statistics.findings], promotion: 'not-authorized: requires separate extra-cost tradeoff approval and D09 promotion eligibility' },
+        pending: ['blind gold audit and delayed repeats', 'separate extra-cost tradeoff approval', 'D09 promotion eligibility',
+            'protected native report approval'],
+        decision };
+    validateD17Artifact('reportV2', report);
+    return report;
+}
+/**
+ * Builds the real champion/challenger record and the shadow baseline for a calibrated v2 report. Each role
+ * cites the D09 artifact applied to it; identities are the calibrated decision identities. The caller binds
+ * the returned baseline and statistics digest into eval-integrity metadata, anchors that digest externally,
+ * and only then serializes the native report with `buildD17NativeReport`.
+ */
+export async function d17NativeHandoff(input) {
+    validateD17Artifact('reportV2', input.report);
+    assertD17Staged(input.prepared);
+    const { report, operator, artifacts } = input;
+    if (heldoutDigest(report) !== input.trustedReportDigest || report.corpusDigest !== heldoutDigest(input.prepared.corpus)
+        || report.preregistrationDigest !== heldoutDigest(input.prepared.preregistration))
+        throw new Error('D17 native report pin');
+    for (const role of ['member', 'aggregate']) {
+        if (artifacts[role].id !== report.calibration[role].artifactId || artifacts[role].digest !== report.calibration[role].artifactDigest) {
+            throw new Error('D17 native calibration pins');
+        }
+    }
+    const pins = report.executionPins;
+    if (!pins.binding || !pins.aggregatePolicyDigest || !report.accounting.test.complete)
+        throw new Error('D17 incomplete native endpoints');
+    const policyPin = report.observations.find(row => row.aggregate)?.aggregate?.policy;
+    if (!policyPin || policyPin.digest !== pins.aggregatePolicyDigest)
+        throw new Error('D17 native execution pins');
+    const template = input.prepared.nativeTemplates.comparison;
+    const items = input.prepared.corpus.rows.filter(row => row.split === 'test').map(row => ({ id: row.id, input: row.input, slice: row.slice }));
+    const championIdentity = calibrationIdentityDigest(artifacts.member.identity);
+    const role = (calibration, identityDigest) => ({ identityDigest, actualModel: pins.model,
+        binding: structuredClone(pins.binding), adapter: structuredClone(pins.adapter),
+        calibration: { artifactId: calibration.id, artifactDigest: calibration.digest } });
+    const record = { ...structuredClone(template), alias: operator.alias,
+        champion: { ...role(artifacts.member, championIdentity), aliasRevision: operator.aliasRevision, ensemblePolicy: null },
+        challenger: { ...role(artifacts.aggregate, calibrationIdentityDigest(artifacts.aggregate.identity)), ensemblePolicy: structuredClone(policyPin) },
+        inputSet: { ...structuredClone(template.inputSet), digest: championChallengerInputSetDigest(items) },
+        preregistration: { ...structuredClone(template.preregistration), thresholdsDigest: pairedThresholdsDigest([...D17_ANALYSIS.native]),
+            holdoutAccessedAt: operator.holdoutAccessedAt },
+        eligibilityId: operator.eligibilityId, evaluationIntegrityReport: { id: operator.integrityReportId, digest: report.integrityDigest },
+        approval: { reference: operator.approvalReference, approvedAt: operator.approvedAt },
+        rollbackTarget: { aliasRevision: operator.aliasRevision, identityDigest: championIdentity } };
+    validateChampionChallenger(record);
+    const shadow = await runChampionChallengerShadow(record, { enabled: true, invocationId: 'd17-recorded-replay', items,
+        evaluate: async ({ role: side, item }) => {
+            const observation = report.observations.find(row => row.id === item.id)?.[side];
+            if (!observation || observation.tokens === null)
+                throw new Error('D17 incomplete native endpoints');
+            return observation;
+        } });
+    return { record, pairedBaseline: { championChallengerShadow: championChallengerShadowBaseline(record, shadow),
+            d17StatisticsDigest: report.statisticsDigest } };
+}
+/**
+ * The preregistered extra-cost tradeoff: an anchored approval of this exact report and statistics that accepts at least the
+ * measured additional reservation cost and records the observed quality lower bound. It approves cost, not quality.
+ */
+export function validateD17Tradeoff(report, tradeoff, trustedDigest) {
+    try {
+        validateD17Artifact('calibration', tradeoff);
+    }
+    catch {
+        throw new Error('D17 tradeoff approval');
+    }
+    const value = tradeoff;
+    if (value.schemaVersion !== 'decision-d17-tradeoff-approval/v1' || typeof trustedDigest !== 'string' || heldoutDigest(value) !== trustedDigest
+        || value.reportDigest !== heldoutDigest(report) || value.statisticsDigest !== report.statisticsDigest
+        || value.netSavingsMicros !== report.statistics.netSavingsMicros || value.acceptedAdditionalCostMicros < Math.max(0, -report.statistics.netSavingsMicros)
+        || value.qualityLowerBps !== report.statistics.quality?.lowerBps)
+        throw new Error('D17 tradeoff approval');
+    return value;
+}
+/** A calibrated report supports PROMOTE only with a passing study gate, a positive quality bound and the bound tradeoff. */
+function d17PromotionSupported(report, integrity, tradeoff, trustedDigest) {
+    if (report.gates.statisticalGate !== 'pass' || !report.statistics.benefitSupported)
+        return false;
+    try {
+        validateD17Tradeoff(report, tradeoff, trustedDigest);
+    }
+    catch {
+        return false;
+    }
+    const upstream = integrity.paired_baseline;
+    return upstream?.d17TradeoffApprovalDigest === trustedDigest;
+}
+/**
+ * Eval-integrity metadata for the native D17 record. The release gate comes from the locked-snapshot evidence (`base`); a
+ * PROMOTE gate survives only when the calibrated study gate passes, the quality lower bound is positive and an anchored
+ * extra-cost tradeoff approval binds this report. Otherwise PROMOTE is held, with the missing pieces as reasons.
+ */
+export function d17NativeIntegrity(input) {
+    const missing = [];
+    if (input.report.gates.statisticalGate !== 'pass')
+        missing.push('D17 statistical gate did not pass');
+    if (!input.report.statistics.benefitSupported)
+        missing.push('D17 quality lower bound is not positive');
+    let tradeoffDigest = null;
+    if (input.tradeoff === undefined || input.tradeoff === null)
+        missing.push('no anchored extra-cost tradeoff approval');
+    else {
+        validateD17Tradeoff(input.report, input.tradeoff, input.trustedTradeoffDigest);
+        tradeoffDigest = input.trustedTradeoffDigest;
+    }
+    // The ensemble integrity contract carries only the gate decision and reasons (eval-tool thresholds are dropped).
+    const gate = { decision: input.base.release_gate.decision, reasons: [...input.base.release_gate.reasons] };
+    const release_gate = gate.decision === 'PROMOTE' && missing.length
+        ? { decision: 'HOLD', reasons: [...gate.reasons, ...missing.map(reason => `promotion withheld: ${reason}`)] } : gate;
+    const { sample_n, uncertainty, integrity_mode, fresh_workspace_required, fresh_workspace_verified, integrity_state, trusted_score_source, compromise_labels, weak_signal_reason } = input.base;
+    return { sample_n, uncertainty, paired_baseline: { ...input.pairedBaseline, d17TradeoffApprovalDigest: tradeoffDigest },
+        integrity_mode, fresh_workspace_required, fresh_workspace_verified, integrity_state, trusted_score_source,
+        compromise_labels: [...compromise_labels], weak_signal_reason, release_gate };
+}
+//# sourceMappingURL=score.js.map

package/dist/src/decision/ensemble-study/staged.js

@@ -0,0 +1,87 @@
+import { heldoutDigest, heldoutRequestSize, heldoutReservationMicros, heldoutReservationTokens, validateHeldoutInputs } from '../heldout/contract.js';
+import { heldoutCollectionAllowance } from '../heldout/journal.js';
+import { d17DryRun, prepareD17Study } from './corpus.js';
+import { D17_ANALYSIS, D17_CALIBRATION } from './protocol.js';
+import { validateD17Artifact } from './artifacts.js';
+/**
+ * Staged D09 calibration preparation (#2611). The corpus, gold, split manifest, review template and native
+ * templates are exactly the diagnostic ones for the same seed (the corpus module, and therefore the corpus
+ * pin, is unchanged); only the preregistration, the v2 analysis, the calibration-phase approval form and the
+ * per-phase dry run differ. The v1 diagnostic preparation remains available and unchanged.
+ */
+export function prepareD17StagedStudy(seed, moduleDigest, sourceDigests) {
+    const base = prepareD17Study(seed, moduleDigest, sourceDigests);
+    const analysis = { schemaVersion: 'decision-d17-analysis/v2', protocol: D17_ANALYSIS, scope: 'staged-calibrated',
+        calibration: D17_CALIBRATION, sourceDigests: base.analysis.sourceDigests, splitManifestDigest: base.analysis.splitManifestDigest,
+        nativeTemplatesDigest: base.analysis.nativeTemplatesDigest };
+    validateD17Artifact('analysisV2', analysis);
+    const preregistration = { ...base.preregistration, studyAnalysisDigest: heldoutDigest(analysis),
+        calibration: { scope: 'calibrated', allowedModes: ['staged'], calibrationPhaseSplits: [...D17_CALIBRATION.phaseSplits] } };
+    validateHeldoutInputs(base.corpus, preregistration);
+    // The study starts with the calibration-phase approval; the test-phase form comes from the reviewed calibration set.
+    // Operator-authorized staged budget: 24,000 calls and 96M tokens give a 19,200-call allowance at the 80% stop, 4,800 calls
+    // of headroom over the 14,400-call two-phase worst case. The USD 8 study cap is unchanged.
+    const approvalTemplate = { ...base.approvalTemplate, preregistrationDigest: heldoutDigest(preregistration),
+        budget: { ...D17_STAGED_BUDGET }, calibration: { mode: 'staged', phase: 'calibration' } };
+    const dryRun = d17StagedDryRun();
+    validateD17Artifact('dryRunV2', dryRun);
+    return { ...base, preregistration, analysis, approvalTemplate, dryRun };
+}
+export const D17_STAGED_BUDGET = Object.freeze({ calls: 24000, tokens: 96000000, usd: 8 });
+/** Staged planning bounds: the same 7,200 first attempts, split into the calibration and test phases. */
+export function d17StagedDryRun() {
+    return { ...d17DryRun(), schemaVersion: 'decision-d17-dry-run/v2', approvalCeilings: { ...D17_STAGED_BUDGET },
+        phases: { calibration: { splits: ['tuning', 'calibration'], rows: 600, firstAttempts: 2400, worstCaseAttempts: 4800 },
+            test: { splits: ['test'], rows: 1200, firstAttempts: 4800, worstCaseAttempts: 9600 } },
+        missingInputs: ['completed 40-item development review', 'calibration-phase operator approval', 'exact-source CI evidence',
+            'provider terms', 'prior study and portfolio spend', 'region and credential resolver pin', 'calibration-phase live observations',
+            'fitted and operator-reviewed D09 calibration set', 'test-phase operator approval', 'test-phase live observations', 'blind human review'] };
+}
+/**
+ * Worst-case reservations for both staged phases from the actual projected request bytes and the approval's price bound,
+ * with every preregistered retry. The collector plans one phase at a time; this bounds the whole study before any spend.
+ */
+export async function d17TwoPhasePlan(corpus, preregistration, approval) {
+    const sizing = { ...approval, calibration: { mode: 'uncalibrated-diagnostic' } };
+    const attemptsPerRequest = 1 + preregistration.providerFailurePolicy.maxRetries;
+    const phase = async (splits) => {
+        const plan = { rows: 0, firstAttempts: 0, maximumAttempts: 0, maximumRequestEstimateTokens: 0, reservedTokens: 0, reservedUsdMicros: 0 };
+        for (const row of corpus.rows.filter(item => splits.includes(item.split))) {
+            plan.rows++;
+            for (const request of row.requests) {
+                const { estimatedTokens } = await heldoutRequestSize(corpus, preregistration, sizing, row, request);
+                plan.firstAttempts++;
+                plan.maximumAttempts += attemptsPerRequest;
+                plan.maximumRequestEstimateTokens = Math.max(plan.maximumRequestEstimateTokens, estimatedTokens);
+                plan.reservedTokens += attemptsPerRequest * heldoutReservationTokens(preregistration, estimatedTokens);
+                plan.reservedUsdMicros += attemptsPerRequest * heldoutReservationMicros(sizing, estimatedTokens);
+            }
+        }
+        return plan;
+    };
+    const calibration = await phase(D17_CALIBRATION.phaseSplits), test = await phase(['test']);
+    return { calibration, test, combined: { maximumAttempts: calibration.maximumAttempts + test.maximumAttempts,
+            reservedTokens: calibration.reservedTokens + test.reservedTokens, reservedUsdMicros: calibration.reservedUsdMicros + test.reservedUsdMicros } };
+}
+/** Both phases must fit the remaining collector allowance (the 80% stop thresholds net of prior charges). */
+export function d17TwoPhaseFit(plan, allowance) {
+    const headroom = { calls: allowance.calls - plan.combined.maximumAttempts, tokens: allowance.tokens - plan.combined.reservedTokens,
+        usdMicros: allowance.usdMicros - plan.combined.reservedUsdMicros };
+    return { allowance: { ...allowance }, headroom, fits: headroom.calls >= 0 && headroom.tokens >= 0 && headroom.usdMicros >= 0 };
+}
+/** A fresh ledger's allowance for an approval: the first approval's thresholds net of its attested prior spend floors. */
+export function d17FreshAllowance(approval) {
+    return heldoutCollectionAllowance(approval, { studyUsdMicros: Math.ceil(approval.priorStudySpendUsd * 1_000_000),
+        portfolioUsdMicros: Math.ceil(approval.priorPortfolioSpendUsd * 1_000_000), studyCalls: 0, studyReservedTokens: 0,
+        counterBlocked: false, attempts: [], journalDigests: [], runs: [] });
+}
+/**
+ * The staged corpus is the diagnostic corpus for the same seed. Any attempt in the ledger on this corpus under another
+ * preregistration (for example a diagnostic run that saw test rows) means the seed is spent: use a fresh seed.
+ */
+export function assertD17SeedUnused(attempts, approval) {
+    if (attempts.some(attempt => attempt.corpusDigest === approval.corpusDigest && attempt.preregistrationDigest !== approval.preregistrationDigest)) {
+        throw new Error('D17 seed reused: this corpus already has observations under another preregistration');
+    }
+}
+//# sourceMappingURL=staged.js.map