@aiwg/cli 2026.9.24 → 2026.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agentic/code/providers/capability-matrix.yaml +2 -1
- package/dist/src/artifacts/discovery-shadow-evaluator.js +177 -0
- package/dist/src/artifacts/discovery-shadow.js +195 -0
- package/dist/src/artifacts/index-builder.js +47 -9
- package/dist/src/artifacts/query-engine.js +94 -1
- package/dist/src/artifacts/types.js +1 -0
- package/dist/src/cli/handlers/decision.js +88 -0
- package/dist/src/cli/handlers/gates.js +154 -0
- package/dist/src/cli/handlers/index.js +5 -1
- package/dist/src/cli/handlers/sessions.js +10 -7
- package/dist/src/cli/handlers/utilities.js +20 -0
- package/dist/src/config/aiwg-config.js +15 -2
- package/dist/src/decision/acceptance.js +25 -0
- package/dist/src/decision/adapters/jev.js +24 -2
- package/dist/src/decision/calibration/registry.js +9 -0
- package/dist/src/decision/compose.js +74 -15
- package/dist/src/decision/context-live-promotion.js +344 -0
- package/dist/src/decision/context-live-qualification.js +591 -0
- package/dist/src/decision/context-pruning.js +847 -0
- package/dist/src/decision/driver.js +430 -0
- package/dist/src/decision/egress-live-qualification.js +874 -0
- package/dist/src/decision/ensemble/contract.js +64 -1
- package/dist/src/decision/ensemble/index.js +1 -0
- package/dist/src/decision/ensemble/report.js +8 -42
- package/dist/src/decision/ensemble/runtime.js +571 -0
- package/dist/src/decision/ensemble-study/artifacts.js +85 -0
- package/dist/src/decision/ensemble-study/calibration.js +443 -0
- package/dist/src/decision/ensemble-study/corpus.js +229 -0
- package/dist/src/decision/ensemble-study/multifact-artifacts.js +23 -0
- package/dist/src/decision/ensemble-study/multifact-study.js +596 -0
- package/dist/src/decision/ensemble-study/multifact.js +241 -0
- package/dist/src/decision/ensemble-study/protocol.js +77 -0
- package/dist/src/decision/ensemble-study/score.js +462 -0
- package/dist/src/decision/ensemble-study/staged.js +87 -0
- package/dist/src/decision/ensemble-study/statistics.js +84 -0
- package/dist/src/decision/evaluate.js +227 -14
- package/dist/src/decision/export.js +4 -0
- package/dist/src/decision/feature-export.js +1074 -0
- package/dist/src/decision/graph-index.js +16 -0
- package/dist/src/decision/graph-live-qualification.js +1041 -0
- package/dist/src/decision/graph-live-workload.js +245 -0
- package/dist/src/decision/graph-skill-bridge.js +21 -3
- package/dist/src/decision/heldout/calibration.js +140 -0
- package/dist/src/decision/heldout/collector.js +430 -0
- package/dist/src/decision/heldout/contract.js +309 -0
- package/dist/src/decision/heldout/d29-generator-ids.js +12 -0
- package/dist/src/decision/heldout/d29-passage-baseline-v2.js +93 -0
- package/dist/src/decision/heldout/d29-passage-baseline-v3.js +100 -0
- package/dist/src/decision/heldout/d29-passage-baseline.js +66 -0
- package/dist/src/decision/heldout/d29-pools-v8.js +575 -0
- package/dist/src/decision/heldout/d29-pools.js +288 -0
- package/dist/src/decision/heldout/d29-v6.js +169 -0
- package/dist/src/decision/heldout/d29-v7.js +202 -0
- package/dist/src/decision/heldout/d29-v8.js +586 -0
- package/dist/src/decision/heldout/generator-registry.js +67 -0
- package/dist/src/decision/heldout/generators.js +696 -0
- package/dist/src/decision/heldout/index.js +8 -0
- package/dist/src/decision/heldout/journal.js +346 -0
- package/dist/src/decision/heldout/study-generators.js +64 -0
- package/dist/src/decision/heldout/types.js +2 -0
- package/dist/src/decision/host-policy.js +61 -0
- package/dist/src/decision/index.js +11 -0
- package/dist/src/decision/interop-profile.v1.json +112 -0
- package/dist/src/decision/interop.js +653 -0
- package/dist/src/decision/issue-triage/index.js +3 -0
- package/dist/src/decision/issue-triage/runtime.js +1069 -0
- package/dist/src/decision/issue-triage/types.js +2 -0
- package/dist/src/decision/patterns/catalog.js +28 -8
- package/dist/src/decision/patterns/playground.js +28 -11
- package/dist/src/decision/patterns/specs.js +19 -10
- package/dist/src/decision/preprocessed-evidence.js +724 -0
- package/dist/src/decision/qualification/capture.js +8 -2
- package/dist/src/decision/qualification/gate-evidence.js +19 -4
- package/dist/src/decision/qualification/quality.js +57 -49
- package/dist/src/decision/qualification/release.js +76 -3
- package/dist/src/decision/qualification/runner.js +5 -2
- package/dist/src/decision/routing/contract.js +175 -0
- package/dist/src/decision/routing/index.js +5 -0
- package/dist/src/decision/routing/report.js +216 -0
- package/dist/src/decision/routing/runtime.js +685 -0
- package/dist/src/decision/routing/types.js +2 -0
- package/dist/src/decision/sdlc-screening.js +1157 -0
- package/dist/src/decision/sensitivity/comparative.js +154 -0
- package/dist/src/decision/sensitivity/contract.js +218 -0
- package/dist/src/decision/sensitivity/index.js +5 -0
- package/dist/src/decision/sensitivity/runtime.js +912 -0
- package/dist/src/decision/sensitivity/types.js +2 -0
- package/dist/src/decision/validate.js +26 -0
- package/dist/src/extensions/commands/definitions.js +38 -0
- package/dist/src/extensions/manifest.js +44 -0
- package/dist/src/gates/discovery.js +315 -0
- package/dist/src/gates/driver.js +388 -0
- package/dist/src/gates/evaluate.js +719 -0
- package/dist/src/gates/floors.js +255 -0
- package/dist/src/gates/index.js +17 -0
- package/dist/src/gates/providers/evidence.js +38 -0
- package/dist/src/gates/providers/index.js +21 -0
- package/dist/src/gates/providers/isolated-child-source.js +446 -0
- package/dist/src/gates/providers/loader.js +1069 -0
- package/dist/src/gates/providers/paired.js +52 -0
- package/dist/src/gates/providers/proportion.js +60 -0
- package/dist/src/gates/providers/registry.js +40 -0
- package/dist/src/gates/providers/scalar.js +41 -0
- package/dist/src/gates/providers/screening.js +53 -0
- package/dist/src/gates/providers/types.js +25 -0
- package/dist/src/gates/registry.js +1003 -0
- package/dist/src/gates/report.js +102 -0
- package/dist/src/gates/schema.js +56 -0
- package/dist/src/gates/stats/binomial.js +26 -0
- package/dist/src/gates/stats/bootstrap.js +62 -0
- package/dist/src/gates/stats/clopper-pearson.js +95 -0
- package/dist/src/gates/stats/digest.js +26 -0
- package/dist/src/gates/stats/error.js +5 -0
- package/dist/src/gates/stats/index.js +9 -0
- package/dist/src/gates/stats/normal.js +35 -0
- package/dist/src/gates/stats/paired.js +120 -0
- package/dist/src/gates/stats/splits.js +30 -0
- package/dist/src/gates/types.js +8 -0
- package/dist/src/governance/redaction.js +14 -4
- package/dist/src/mcp/muse-native-config.mjs +123 -0
- package/dist/src/mcp/tools/decision.mjs +188 -0
- package/dist/src/mcp/tools/subsystems.mjs +2 -0
- package/dist/src/providers/capability-matrix.yaml +2 -1
- package/dist/src/providers/provider-definitions.js +2 -1
- package/dist/src/sessions/adapters/muse.js +336 -39
- package/dist/src/sessions/batch-import.js +7 -0
- package/dist/src/sessions/workspace-discovery.js +13 -2
- package/package.json +1 -1
- package/schemas/decision/ConformalFrozenOpenData.v2.schema.json +371 -0
- package/schemas/decision/ConformalOpenDataReport.v2.schema.json +421 -0
- package/schemas/decision/ConformalPreregistration.v2.schema.json +529 -0
- package/schemas/decision/ContextPruningCandidate.v1.schema.json +47 -0
- package/schemas/decision/ContextPruningEvaluationReport.v1.schema.json +758 -0
- package/schemas/decision/ContextPruningPreregistration.v1.schema.json +175 -0
- package/schemas/decision/ContextPruningReceipt.v1.schema.json +72 -0
- package/schemas/decision/D17Multifact.v1.schema.json +1851 -0
- package/schemas/decision/D17StudyAnalysis.v1.schema.json +42 -0
- package/schemas/decision/D17StudyAnalysis.v2.schema.json +51 -0
- package/schemas/decision/D17StudyCalibration.v1.schema.json +819 -0
- package/schemas/decision/D17StudyDryRun.v1.schema.json +169 -0
- package/schemas/decision/D17StudyDryRun.v2.schema.json +239 -0
- package/schemas/decision/D17StudyGold.v1.schema.json +108 -0
- package/schemas/decision/D17StudyGuide.v1.schema.json +63 -0
- package/schemas/decision/D17StudyNativeTemplates.v1.schema.json +619 -0
- package/schemas/decision/D17StudyProtocol.v1.schema.json +241 -0
- package/schemas/decision/D17StudyReport.v1.schema.json +902 -0
- package/schemas/decision/D17StudyReport.v2.schema.json +1319 -0
- package/schemas/decision/D17StudyReview.v1.schema.json +175 -0
- package/schemas/decision/D17StudySplits.v1.schema.json +217 -0
- package/schemas/decision/D29Study.v1.schema.json +1903 -0
- package/schemas/decision/D29Study.v2.schema.json +1344 -0
- package/schemas/decision/D29Study.v3.schema.json +609 -0
- package/schemas/decision/D29Study.v4.schema.json +1133 -0
- package/schemas/decision/D29Study.v6.schema.json +1205 -0
- package/schemas/decision/D29Study.v7.schema.json +1471 -0
- package/schemas/decision/D29Study.v8.schema.json +671 -0
- package/schemas/decision/D29Study.v9.schema.json +778 -0
- package/schemas/decision/DecisionComparativeReplayAuditSample.v1.schema.json +137 -0
- package/schemas/decision/DecisionComparativeReplayCorpus.v1.schema.json +258 -0
- package/schemas/decision/DecisionComparativeReplayFreeze.v1.schema.json +74 -0
- package/schemas/decision/DecisionComparativeReplayGold.v1.schema.json +50 -0
- package/schemas/decision/DecisionComparativeReplayGoldAsset.v1.schema.json +37 -0
- package/schemas/decision/DecisionComparativeReplayIntegrity.v1.schema.json +81 -0
- package/schemas/decision/DecisionComparativeReplayMembers.v1.schema.json +52 -0
- package/schemas/decision/DecisionComparativeReplayPreregistration.v1.schema.json +113 -0
- package/schemas/decision/DecisionComparativeReplayReport.v1.schema.json +407 -0
- package/schemas/decision/DecisionComparativeReplayReviewPackets.v1.schema.json +260 -0
- package/schemas/decision/DecisionComparativeReplaySplit.v1.schema.json +84 -0
- package/schemas/decision/DecisionFeatureObservation.v1.schema.json +564 -0
- package/schemas/decision/DecisionFeatureSet.v1.schema.json +155 -0
- package/schemas/decision/DecisionInteropMapping.v1.schema.json +85 -0
- package/schemas/decision/DecisionIssueTriageEvaluationManifest.v1.schema.json +174 -0
- package/schemas/decision/DecisionIssueTriageEvaluationReport.v1.schema.json +280 -0
- package/schemas/decision/DecisionIssueTriagePilotPack.v1.schema.json +114 -0
- package/schemas/decision/DecisionRouteReceipt.v1.schema.json +155 -0
- package/schemas/decision/DecisionRoutingPolicy.v1.schema.json +132 -0
- package/schemas/decision/DecisionRoutingShadowReport.v1.schema.json +170 -0
- package/schemas/decision/DecisionRoutingTask.v1.schema.json +49 -0
- package/schemas/decision/DecisionRuleset.schema.json +8 -2
- package/schemas/decision/DecisionRuleset.v1alpha2.schema.json +8 -2
- package/schemas/decision/DecisionSensitivityPlan.v1.schema.json +191 -0
- package/schemas/decision/DecisionSensitivityReport.v1.schema.json +163 -0
- package/schemas/decision/HeldoutApproval.v1.schema.json +312 -0
- package/schemas/decision/HeldoutAttempt.v1.schema.json +211 -0
- package/schemas/decision/HeldoutBaseline.v1.schema.json +90 -0
- package/schemas/decision/HeldoutCalibrationPhase.v1.schema.json +87 -0
- package/schemas/decision/HeldoutCorpus.v1.schema.json +167 -0
- package/schemas/decision/HeldoutEvent.v1.schema.json +41 -0
- package/schemas/decision/HeldoutFrozen.v1.schema.json +142 -0
- package/schemas/decision/HeldoutPreregistration.v1.schema.json +222 -0
- package/schemas/decision/HeldoutSpendEvent.v1.schema.json +122 -0
- package/schemas/decision/HeldoutSpendHead.v1.schema.json +25 -0
- package/schemas/decision/HeldoutSpendHead.v2.schema.json +46 -0
- package/schemas/decision/HeldoutSummary.v1.schema.json +101 -0
- package/schemas/decision/OpaDecisionLogExport.v1.schema.json +65 -0
- package/schemas/decision/PreprocessedEvidence.v1.schema.json +306 -0
- package/schemas/decision/RulesetResult.v1alpha2.schema.json +175 -0
- package/schemas/decision/SdlcEvidenceScreening.v1.schema.json +499 -0
- package/schemas/decision/SdlcScreeningPreregistration.v1.schema.json +132 -0
- package/schemas/decision/SdlcScreeningPreregistration.v2.schema.json +138 -0
- package/schemas/decision/SdlcScreeningRelease.v1.schema.json +654 -0
- package/schemas/decision/SdlcScreeningRelease.v2.schema.json +654 -0
- package/tools/agents/deploy-agents.mjs +1 -1
- package/tools/agents/providers/base.mjs +26 -7
- package/tools/agents/providers/factory.mjs +10 -2
- package/tools/agents/providers/muse-hooks.mjs +1 -29
- package/tools/agents/providers/muse.mjs +6 -3
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TV-12 promotion steps after collection (#2681): record the reviewer-approved profile and margin chosen
|
|
3
|
+
* by the preregistered rule, then run an enforce-mode canary with an exercised rollback to single calls.
|
|
4
|
+
* Neither step changes a default: enforcement still needs a caller to pass the recorded qualification.
|
|
5
|
+
*/
|
|
6
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
7
|
+
import { join, resolve } from 'node:path';
|
|
8
|
+
import { canonicalJson } from '../security/artifact-trust.js';
|
|
9
|
+
import { admitEntry } from './entry.js';
|
|
10
|
+
import { CanonicalJsonByteEstimator, ContextPlanError, planDecisionContext } from './context-plan.js';
|
|
11
|
+
import { assertContextQualified, compareContextUsage } from './context-qualification.js';
|
|
12
|
+
import { assertContextArtifactRoot, assertContextLiveSource, compileContextLiveCase, contextLiveBinding, contextLiveDigest, contextLiveRequestCapacity, contextLiveRuleset, contextLiveTarget, ContextLiveBudget, validateContextLiveApproval, withContextLiveSpendLedger, TV12_ISSUE_USD_CAP, CREDENTIAL_REF, } from './context-live-qualification.js';
|
|
13
|
+
import { JevDecisionAdapter } from './adapters/jev.js';
|
|
14
|
+
import { decisionBatchQuestionId } from './batch.js';
|
|
15
|
+
import { contextResultEnvelopeFits, evaluateDecisionRuleset } from './evaluate.js';
|
|
16
|
+
const estimator = new CanonicalJsonByteEstimator();
|
|
17
|
+
const subject = 'synthetic-tv12';
|
|
18
|
+
const nonblank = (value) => typeof value === 'string' && value.trim().length > 0;
|
|
19
|
+
const sha256 = /^sha256:[a-f0-9]{64}$/;
|
|
20
|
+
const closed = (value, keys) => !!value && typeof value === 'object' && !Array.isArray(value)
|
|
21
|
+
&& Object.keys(value).sort().join(',') === [...keys].sort().join(',');
|
|
22
|
+
export const contextLiveRecordsDigest = (records) => contextLiveDigest([...records]
|
|
23
|
+
.sort((a, b) => `${a.caseId}\0${a.partitionId}`.localeCompare(`${b.caseId}\0${b.partitionId}`, 'en')));
|
|
24
|
+
/** Rebinds every retained comparison to the frozen corpus and returns exactly one complete provider request per partition. */
|
|
25
|
+
async function boundComparisons(approval, corpus, records) {
|
|
26
|
+
const samples = [];
|
|
27
|
+
const used = new Set();
|
|
28
|
+
for (const item of corpus.cases) {
|
|
29
|
+
const { input } = await compileContextLiveCase(item, approval.model, approval.region);
|
|
30
|
+
let plan;
|
|
31
|
+
try {
|
|
32
|
+
plan = planDecisionContext(input, corpus.profile, estimator);
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
if (error instanceof ContextPlanError && ['oversized-state', 'oversized-question'].includes(error.reason))
|
|
36
|
+
continue;
|
|
37
|
+
throw error;
|
|
38
|
+
}
|
|
39
|
+
for (const partition of plan.partitions) {
|
|
40
|
+
const partitionInput = { ...input, questions: input.questions.filter(q => partition.questionIds.includes(q.id)) };
|
|
41
|
+
const checked = planDecisionContext(partitionInput, corpus.profile, estimator);
|
|
42
|
+
const matches = records.filter(r => r.caseId === item.id && r.partitionId === partition.id);
|
|
43
|
+
const record = matches[0];
|
|
44
|
+
const usage = record && { source: record.source, servedModel: record.servedModel, requestId: record.requestId,
|
|
45
|
+
inputTokens: record.inputTokens, outputTokens: record.outputTokens, costUsd: record.costUsd };
|
|
46
|
+
if (matches.length !== 1 || !record || checked.partitions.length !== 1 || record.source !== 'provider' || record.servedModel !== approval.model
|
|
47
|
+
|| record.planDigest !== checked.planDigest || record.profileDigest !== checked.providerProfile.digest
|
|
48
|
+
|| record.estimator.id !== estimator.id || record.estimator.version !== estimator.version
|
|
49
|
+
|| record.estimatedInputTokens !== checked.partitions[0].estimate.aggregateTokens
|
|
50
|
+
|| record.actualInputTokens !== record.inputTokens || record.usageArtifactDigest !== contextLiveDigest(usage)
|
|
51
|
+
|| record.usageRef !== `usage:${record.usageArtifactDigest}`) {
|
|
52
|
+
throw new ContextPlanError('invalid-input', 'retained comparison does not bind to the frozen corpus partition');
|
|
53
|
+
}
|
|
54
|
+
used.add(record);
|
|
55
|
+
samples.push({ caseId: `${item.id}:${partition.id}`, input: partitionInput, actualInputTokens: record.actualInputTokens, source: 'provider', usageRef: record.usageRef });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if (!samples.length || used.size !== records.length)
|
|
59
|
+
throw new ContextPlanError('invalid-input', 'retained comparisons must cover exactly the planned partitions');
|
|
60
|
+
return samples;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Applies the preregistered rule (worst observed undercount plus the frozen reserve, capped by the frozen
|
|
64
|
+
* maximum) and records the reviewer's matching approval. Synthetic, partial or unbound evidence throws.
|
|
65
|
+
*/
|
|
66
|
+
export async function recordContextQualification(options) {
|
|
67
|
+
const approval = structuredClone(options.approval), corpus = structuredClone(options.corpus);
|
|
68
|
+
const records = structuredClone(options.records), review = structuredClone(options.review);
|
|
69
|
+
validateContextLiveApproval(approval, corpus);
|
|
70
|
+
admitEntry(review);
|
|
71
|
+
const { marginRule } = approval;
|
|
72
|
+
// A selected margin above the collection margin could split a request that was collected whole.
|
|
73
|
+
if (corpus.profile.safetyMarginBps < marginRule.maximumMarginBps)
|
|
74
|
+
throw new ContextPlanError('invalid-profile', 'collection margin must cover the preregistered maximum');
|
|
75
|
+
const samples = await boundComparisons(approval, corpus, records);
|
|
76
|
+
const observed = compareContextUsage(samples, corpus.profile, estimator);
|
|
77
|
+
for (const row of observed.cases) {
|
|
78
|
+
const [caseId, partitionId] = row.caseId.split(':');
|
|
79
|
+
if (records.find(r => r.caseId === caseId && r.partitionId === partitionId).undercountBps !== row.undercountBps) {
|
|
80
|
+
throw new ContextPlanError('invalid-input', 'retained undercount differs from recomputation');
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const candidateMarginBps = observed.worstUndercountBps + marginRule.extraReserveBps;
|
|
84
|
+
if (candidateMarginBps > marginRule.maximumMarginBps) {
|
|
85
|
+
throw new ContextPlanError('rollout-unqualified', 'observed undercount exceeds the preregistered maximum margin');
|
|
86
|
+
}
|
|
87
|
+
const recordsDigest = contextLiveRecordsDigest(records);
|
|
88
|
+
if (!closed(review, ['schemaVersion', 'approved', 'reviewer', 'approvalReference', 'runId', 'recordsDigest', 'selectedMarginBps', 'qualifiedProfile'])
|
|
89
|
+
|| !closed(review.qualifiedProfile, ['id', 'version']) || review.schemaVersion !== 'context-margin-review/v1' || review.approved !== true
|
|
90
|
+
|| !nonblank(review.reviewer) || !nonblank(review.approvalReference) || review.runId !== approval.runId || review.recordsDigest !== recordsDigest
|
|
91
|
+
|| review.selectedMarginBps !== candidateMarginBps || !nonblank(review.qualifiedProfile.id) || !nonblank(review.qualifiedProfile.version)
|
|
92
|
+
|| review.qualifiedProfile.id === corpus.profile.id && review.qualifiedProfile.version === corpus.profile.version) {
|
|
93
|
+
throw new ContextPlanError('rollout-unqualified', 'margin review does not approve the preregistered selection for these records');
|
|
94
|
+
}
|
|
95
|
+
const profile = { ...structuredClone(corpus.profile), id: review.qualifiedProfile.id,
|
|
96
|
+
version: review.qualifiedProfile.version, safetyMarginBps: candidateMarginBps };
|
|
97
|
+
const qualification = compareContextUsage(samples, profile, estimator);
|
|
98
|
+
assertContextQualified(qualification, profile, estimator);
|
|
99
|
+
return { schemaVersion: 'context-qualification-record/v1', runId: approval.runId, sourceCommit: approval.sourceCommit,
|
|
100
|
+
model: approval.model, region: approval.region, corpusDigest: approval.corpusDigest, preregistrationDigest: approval.preregistrationDigest,
|
|
101
|
+
approvalDigest: contextLiveDigest(approval), recordsDigest, collectionProfileDigest: contextLiveDigest(corpus.profile), marginRule,
|
|
102
|
+
selection: { worstUndercountBps: observed.worstUndercountBps, extraReserveBps: marginRule.extraReserveBps, candidateMarginBps,
|
|
103
|
+
maximumMarginBps: marginRule.maximumMarginBps }, review, profile, profileDigest: contextLiveDigest(profile), qualification };
|
|
104
|
+
}
|
|
105
|
+
/** Enforcement gate for a stored record. Any profile change, including only its version, invalidates it. */
|
|
106
|
+
export function verifyContextQualificationRecord(record, profile = record?.profile) {
|
|
107
|
+
const s = record?.selection;
|
|
108
|
+
if (record?.schemaVersion !== 'context-qualification-record/v1' || !s || s.candidateMarginBps !== s.worstUndercountBps + s.extraReserveBps
|
|
109
|
+
|| s.candidateMarginBps > s.maximumMarginBps || record.qualification?.worstUndercountBps !== s.worstUndercountBps
|
|
110
|
+
|| record.review?.approved !== true || !nonblank(record.review.reviewer) || record.review.selectedMarginBps !== s.candidateMarginBps
|
|
111
|
+
|| record.review.qualifiedProfile?.id !== profile?.id || record.review.qualifiedProfile?.version !== profile?.version
|
|
112
|
+
|| profile.safetyMarginBps !== s.candidateMarginBps || record.profileDigest !== contextLiveDigest(profile)) {
|
|
113
|
+
throw new ContextPlanError('rollout-unqualified', 'qualification record does not match the enforced profile');
|
|
114
|
+
}
|
|
115
|
+
assertContextQualified(record.qualification, profile, estimator);
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* The evaluator preflights the worst-case completion envelope before dispatch (#2797): every
|
|
119
|
+
* evaluation carries every usage entry with the binding's full retry attempts, so the 24-question
|
|
120
|
+
* `many-short` case rejects with `invalid-input` before any provider work instead of exceeding
|
|
121
|
+
* the default entry limits (`property-count`) after dispatch. The canary covers that shape as a
|
|
122
|
+
* rejected-before-dispatch case with zero dispatches in both phases.
|
|
123
|
+
*/
|
|
124
|
+
export function validateContextCanaryApproval(approval, corpus, record) {
|
|
125
|
+
admitEntry(approval);
|
|
126
|
+
const plan = approval?.plan;
|
|
127
|
+
if (!closed(approval, ['schemaVersion', 'approved', 'reviewer', 'stagingWorkspace', 'runId', 'sourceCommit', 'exactHeadCi', 'model', 'apiRevision', 'region',
|
|
128
|
+
'secretServiceReference', 'credentialResolverDigest', 'corpusDigest', 'qualificationRecordDigest', 'canaryPlanDigest', 'plan'])
|
|
129
|
+
|| !closed(plan, ['schemaVersion', 'caseIds', 'rollback', 'budget', 'perRequestBound']) || !closed(plan.budget, ['requests', 'tokens', 'usd', 'wallClockMs'])
|
|
130
|
+
|| !closed(plan.perRequestBound, ['totalTokens', 'outputTokens', 'usd', 'approvalReference'])
|
|
131
|
+
|| approval.schemaVersion !== 'context-canary-approval/v1' || approval.approved !== true || plan.schemaVersion !== 'context-canary-plan/v1' || plan.rollback !== 'observe-only'
|
|
132
|
+
|| ![approval.reviewer, approval.stagingWorkspace, approval.exactHeadCi, approval.model, approval.region, approval.secretServiceReference, plan.perRequestBound.approvalReference].every(nonblank)
|
|
133
|
+
|| !/^[a-zA-Z0-9_-]+$/.test(approval.runId) || !/^[a-f0-9]{40}$/.test(approval.sourceCommit) || approval.apiRevision !== 'v1' || /latest|unknown/i.test(approval.model)
|
|
134
|
+
|| !CREDENTIAL_REF.test(approval.secretServiceReference)
|
|
135
|
+
|| ![approval.credentialResolverDigest, approval.corpusDigest, approval.qualificationRecordDigest, approval.canaryPlanDigest].every(d => sha256.test(d))
|
|
136
|
+
|| approval.canaryPlanDigest !== contextLiveDigest(plan) || approval.corpusDigest !== contextLiveDigest(corpus) || approval.corpusDigest !== record.corpusDigest
|
|
137
|
+
|| approval.qualificationRecordDigest !== contextLiveDigest(record) || approval.model !== record.model || approval.region !== record.region
|
|
138
|
+
|| !Array.isArray(plan.caseIds) || !plan.caseIds.length || new Set(plan.caseIds).size !== plan.caseIds.length
|
|
139
|
+
|| plan.caseIds.some(id => !corpus.cases.some(c => c.id === id))
|
|
140
|
+
|| plan.budget.usd > TV12_ISSUE_USD_CAP
|
|
141
|
+
|| [plan.budget.requests, plan.budget.tokens, plan.budget.wallClockMs, plan.perRequestBound.totalTokens, plan.perRequestBound.outputTokens].some(n => !Number.isSafeInteger(n) || n < 1)
|
|
142
|
+
|| plan.perRequestBound.outputTokens > plan.perRequestBound.totalTokens
|
|
143
|
+
|| [plan.budget.usd, plan.perRequestBound.usd].some(n => !Number.isFinite(n) || n <= 0 || n > Number.MAX_SAFE_INTEGER / 1_000_000)) {
|
|
144
|
+
throw new Error('Incomplete or mismatched TV-12 canary approval');
|
|
145
|
+
}
|
|
146
|
+
verifyContextQualificationRecord(record);
|
|
147
|
+
}
|
|
148
|
+
/** Dispatches each phase would need, derived from the qualified plan; rejected phases need none. */
|
|
149
|
+
export async function contextCanaryDispatches(caseIds, corpus, record) {
|
|
150
|
+
const rows = [];
|
|
151
|
+
const adapterVersion = new JevDecisionAdapter({ region: record.region }).version;
|
|
152
|
+
for (const id of caseIds) {
|
|
153
|
+
const item = corpus.cases.find(c => c.id === id);
|
|
154
|
+
const { input } = await compileContextLiveCase(item, record.model, record.region);
|
|
155
|
+
const aliases = item.definitions.map((_, index) => `q${index}`);
|
|
156
|
+
// Mirror of the dispatch binding the canary builds per phase (single target without retries,
|
|
157
|
+
// one attempt budget per alias); the probe reads only retry ceilings and identity, never
|
|
158
|
+
// credentials, so no credential reference is carried here.
|
|
159
|
+
const envelopeBinding = { spec: { maxAttempts: aliases.length,
|
|
160
|
+
evaluations: Object.fromEntries(aliases.map(alias => [alias, { targets: [{ adapter: 'jev', adapterVersion,
|
|
161
|
+
model: record.model, retry: { maxRetries: 0, initialDelayMs: 0, maxDelayMs: 0 } }] }])) } };
|
|
162
|
+
const envelopeItems = aliases.map((alias, index) => ({ alias, definition: item.definitions[index] }));
|
|
163
|
+
let enforce = 0, rollback = 0;
|
|
164
|
+
let enforceRejection = null;
|
|
165
|
+
let rollbackRejection = null;
|
|
166
|
+
try {
|
|
167
|
+
const planned = planDecisionContext(input, record.profile, estimator);
|
|
168
|
+
// Both phases evaluate every alias in one invocation, so both check the same worst-case
|
|
169
|
+
// envelope (#2797) — each phase with its own call. The worst-case single-dispatch shape
|
|
170
|
+
// bounds the native enforce batches as well as the single rollback dispatches, so a
|
|
171
|
+
// rejected envelope needs no dispatch in that phase.
|
|
172
|
+
for (const phase of ['enforce', 'rollback']) {
|
|
173
|
+
if (!contextResultEnvelopeFits(planned, envelopeItems, envelopeBinding)) {
|
|
174
|
+
if (phase === 'enforce')
|
|
175
|
+
enforceRejection = 'invalid-input';
|
|
176
|
+
else
|
|
177
|
+
rollbackRejection = 'invalid-input';
|
|
178
|
+
}
|
|
179
|
+
else if (phase === 'enforce')
|
|
180
|
+
enforce = planned.partitions.length;
|
|
181
|
+
else
|
|
182
|
+
rollback = input.questions.length;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
catch (error) {
|
|
186
|
+
if (!(error instanceof ContextPlanError && ['oversized-state', 'oversized-question'].includes(error.reason)))
|
|
187
|
+
throw error;
|
|
188
|
+
enforceRejection = error.reason;
|
|
189
|
+
rollbackRejection = error.reason;
|
|
190
|
+
}
|
|
191
|
+
rows.push({ caseId: id, enforce, rollback, enforceRejection, rollbackRejection });
|
|
192
|
+
}
|
|
193
|
+
return rows;
|
|
194
|
+
}
|
|
195
|
+
function inspect(caseId, phase, expected, expectedRejection, result, aliases, model, bound, outputBound) {
|
|
196
|
+
const plan = result.spec.context?.plan ?? null;
|
|
197
|
+
const usage = result.spec.context?.actualUsage ?? [];
|
|
198
|
+
const evaluations = Object.values(result.spec.evaluations ?? {});
|
|
199
|
+
const attempts = evaluations.flatMap(e => e.spec.attempts ?? []);
|
|
200
|
+
const partitions = usage.map(u => {
|
|
201
|
+
const partition = plan?.partitions.find(p => p.id === u.partitionId);
|
|
202
|
+
const withinEffectiveLimits = !!plan && !!partition && u.questionIds.every(id => partition.questionIds.includes(id))
|
|
203
|
+
&& partition.estimate.aggregateTokens <= plan.limits.effectiveAggregateTokens
|
|
204
|
+
&& partition.estimate.stateAndLongestQuestionTokens <= plan.limits.effectiveStateAndLongestQuestionTokens;
|
|
205
|
+
const withinDocumentedLimits = !!plan && Number.isSafeInteger(u.actualInputTokens) && u.actualInputTokens <= plan.limits.documentedAggregateTokens && u.actualInputTokens <= bound;
|
|
206
|
+
return { partitionId: u.partitionId, questions: u.questionIds.length, estimatedInputTokens: u.estimatedInputTokens, actualInputTokens: u.actualInputTokens, withinEffectiveLimits, withinDocumentedLimits };
|
|
207
|
+
});
|
|
208
|
+
const oversizedDispatches = partitions.filter(p => !p.withinEffectiveLimits || !p.withinDocumentedLimits).length;
|
|
209
|
+
const nativeDispatches = new Set(attempts.filter(a => a.batch?.mode === 'native').map(a => a.batch.groupId)).size;
|
|
210
|
+
const requestIds = [...new Set(attempts.map(a => a.requestId).filter((id) => typeof id === 'string'))].sort();
|
|
211
|
+
const rejected = expected === 0;
|
|
212
|
+
// The provider is the only source of per-call output; null means unknown, so it fails closed.
|
|
213
|
+
// A native batch keeps one shared usage copy per request on the ruleset result while its answer
|
|
214
|
+
// attempts carry null usage by design, so requests are read there and singles from success attempts.
|
|
215
|
+
const outputReports = new Map();
|
|
216
|
+
for (const entry of result.spec.batchRequests ?? []) {
|
|
217
|
+
const key = typeof entry.requestId === 'string' && entry.requestId ? `id:${entry.requestId}` : `batch:${entry.groupId}:${entry.ordinal}`;
|
|
218
|
+
if (!outputReports.has(key))
|
|
219
|
+
outputReports.set(key, entry.usage?.outputTokens ?? null);
|
|
220
|
+
}
|
|
221
|
+
attempts.forEach((attempt, index) => {
|
|
222
|
+
if (attempt.batch?.mode === 'native' || attempt.status !== 'success')
|
|
223
|
+
return;
|
|
224
|
+
const key = typeof attempt.requestId === 'string' && attempt.requestId ? `id:${attempt.requestId}` : `single:${index}`;
|
|
225
|
+
if (!outputReports.has(key))
|
|
226
|
+
outputReports.set(key, attempt.usage?.outputTokens ?? null);
|
|
227
|
+
});
|
|
228
|
+
const outputOversizedDispatches = rejected ? 0
|
|
229
|
+
: [...outputReports.values()].filter(n => !Number.isSafeInteger(n) || n < 0 || n > outputBound).length;
|
|
230
|
+
const covered = new Set(usage.flatMap(u => u.questionIds));
|
|
231
|
+
const pass = rejected
|
|
232
|
+
? expectedRejection !== null && usage.length === 0 && attempts.length === 0 && result.spec.contextFailure?.reason === expectedRejection
|
|
233
|
+
: oversizedDispatches === 0 && outputOversizedDispatches === 0 && usage.length === expected && covered.size === aliases.length
|
|
234
|
+
&& aliases.every(alias => covered.has(decisionBatchQuestionId(alias))) && evaluations.length === aliases.length
|
|
235
|
+
&& evaluations.every(e => e.spec.status === 'success') && attempts.every(a => a.status === 'success' && a.actualModel === model)
|
|
236
|
+
&& (phase === 'rollback' ? nativeDispatches === 0 && usage.every(u => u.questionIds.length === 1) : true);
|
|
237
|
+
return { caseId, phase, expectedDispatches: expected, dispatches: usage.length, rejectedBeforeDispatch: rejected && usage.length === 0,
|
|
238
|
+
planDigest: plan?.planDigest ?? null, profileDigest: plan?.providerProfile.digest ?? null, requestIds, partitions, oversizedDispatches, nativeDispatches, pass, outputOversizedDispatches };
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Enforce-mode canary. Each case runs once with the recorded qualification (native batching split by the
|
|
242
|
+
* qualified plan), then once rolled back to `observe-only`, which evaluates every question singly. All
|
|
243
|
+
* dispatches of a phase are reserved against the approved bound before it starts; the first failure stops.
|
|
244
|
+
*/
|
|
245
|
+
export async function runContextEnforceCanary(options) {
|
|
246
|
+
const { sourceRoot, artifactRoot, host, offlineTransport } = options;
|
|
247
|
+
const approval = structuredClone(options.approval), corpus = structuredClone(options.corpus), record = structuredClone(options.record);
|
|
248
|
+
validateContextCanaryApproval(approval, corpus, record);
|
|
249
|
+
await assertContextLiveSource(sourceRoot, approval.sourceCommit);
|
|
250
|
+
await assertContextArtifactRoot(sourceRoot, artifactRoot);
|
|
251
|
+
try {
|
|
252
|
+
return await withContextLiveSpendLedger(artifactRoot, approval.plan.budget.usd, (runsRoot, issueSpend) => canaryWithinLedger(approval, corpus, record, sourceRoot, runsRoot, issueSpend, host, offlineTransport));
|
|
253
|
+
}
|
|
254
|
+
finally {
|
|
255
|
+
host.dispose?.();
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
async function canaryWithinLedger(approval, corpus, record, sourceRoot, runsRoot, issueSpend, host, offlineTransport) {
|
|
259
|
+
const directory = resolve(runsRoot, approval.runId);
|
|
260
|
+
await mkdir(directory, { recursive: false, mode: 0o700 });
|
|
261
|
+
await writeFile(join(directory, 'canary-approval.json'), canonicalJson(approval), { flag: 'wx', mode: 0o600 });
|
|
262
|
+
await writeFile(join(directory, 'qualification-record.json'), canonicalJson(record), { flag: 'wx', mode: 0o600 });
|
|
263
|
+
const { plan } = approval;
|
|
264
|
+
const started = Date.now();
|
|
265
|
+
const budget = new ContextLiveBudget({ ...plan, budget: { ...plan.budget, usd: issueSpend.runUsdCeiling } }, started);
|
|
266
|
+
const controller = new AbortController();
|
|
267
|
+
const deadlineTimer = setTimeout(() => controller.abort(), Math.min(2_147_483_647, Math.floor(plan.budget.wallClockMs * 0.8)));
|
|
268
|
+
const results = [];
|
|
269
|
+
let stopped = null;
|
|
270
|
+
try {
|
|
271
|
+
for (const { caseId, enforce, rollback, enforceRejection, rollbackRejection } of await contextCanaryDispatches(plan.caseIds, corpus, record)) {
|
|
272
|
+
const item = corpus.cases.find(c => c.id === caseId);
|
|
273
|
+
for (const [phase, expected, expectedRejection] of [['enforce', enforce, enforceRejection], ['rollback', rollback, rollbackRejection]]) {
|
|
274
|
+
try {
|
|
275
|
+
for (let i = 0; i < expected; i++)
|
|
276
|
+
budget.reserve();
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
stopped = 'budget-exhausted';
|
|
280
|
+
break;
|
|
281
|
+
}
|
|
282
|
+
const { input, policy } = await compileContextLiveCase(item, approval.model, approval.region);
|
|
283
|
+
const aliases = item.definitions.map((_, i) => `q${i}`);
|
|
284
|
+
const definitions = Object.fromEntries(aliases.map((alias, i) => [alias, item.definitions[i]]));
|
|
285
|
+
const adapter = new JevDecisionAdapter({ region: approval.region, ...(offlineTransport ? { fetch: offlineTransport } : {}) });
|
|
286
|
+
const remaining = Math.floor(plan.budget.wallClockMs * 0.8 - (Date.now() - started));
|
|
287
|
+
if (remaining < 1 || controller.signal.aborted) {
|
|
288
|
+
stopped = 'deadline';
|
|
289
|
+
break;
|
|
290
|
+
}
|
|
291
|
+
const target = contextLiveTarget(approval, adapter.version, Math.min(30_000, remaining));
|
|
292
|
+
const ruleset = contextLiveRuleset(`canary-${item.id}`, aliases, item.definitions);
|
|
293
|
+
const binding = contextLiveBinding(ruleset, aliases, target, Math.min(remaining, 30_000 * aliases.length));
|
|
294
|
+
let result;
|
|
295
|
+
try {
|
|
296
|
+
result = await evaluateDecisionRuleset({ ruleset, binding, definitions, input: item.input, runId: approval.runId,
|
|
297
|
+
invocationId: `${approval.runId}:${item.id}:${phase}`, adapters: { jev: adapter }, signal: controller.signal,
|
|
298
|
+
resolveCredential: async (reference) => {
|
|
299
|
+
if (reference !== approval.secretServiceReference)
|
|
300
|
+
throw new Error('TV-12 credential access denied');
|
|
301
|
+
return host.resolveCredential(reference);
|
|
302
|
+
},
|
|
303
|
+
projection: { resolve: () => structuredClone(policy) },
|
|
304
|
+
batching: { enabled: true, evaluations: Object.fromEntries(aliases.map(alias => [alias, { decisionSubject: subject, independent: true,
|
|
305
|
+
egressPolicy: contextLiveDigest(policy), hostPolicy: 'tv12-canary-v1' }])) },
|
|
306
|
+
context: { input: { ...input, questions: input.questions.map(q => ({ ...q, id: decisionBatchQuestionId(q.id) })) }, profile: record.profile, estimator,
|
|
307
|
+
rollout: phase === 'enforce' ? { mode: 'enforce', qualification: record.qualification } : { mode: 'observe-only' } } });
|
|
308
|
+
}
|
|
309
|
+
catch {
|
|
310
|
+
stopped = 'evaluation-error';
|
|
311
|
+
break;
|
|
312
|
+
}
|
|
313
|
+
const row = inspect(item.id, phase, expected, expectedRejection, result, aliases, approval.model, plan.perRequestBound.totalTokens, plan.perRequestBound.outputTokens);
|
|
314
|
+
results.push(row);
|
|
315
|
+
await writeFile(join(directory, `${item.id}-${phase}.json`), canonicalJson(row), { flag: 'wx', mode: 0o600 });
|
|
316
|
+
if (!row.pass) {
|
|
317
|
+
stopped = 'canary-check-failed';
|
|
318
|
+
break;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
if (stopped)
|
|
322
|
+
break;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
finally {
|
|
326
|
+
clearTimeout(deadlineTimer);
|
|
327
|
+
}
|
|
328
|
+
const complete = !stopped && results.length === plan.caseIds.length * 2;
|
|
329
|
+
const summary = { schemaVersion: 'context-enforce-canary/v1', source: offlineTransport ? 'synthetic' : 'provider', runId: approval.runId,
|
|
330
|
+
sourceCommit: approval.sourceCommit, qualificationRecordDigest: approval.qualificationRecordDigest, canaryPlanDigest: approval.canaryPlanDigest,
|
|
331
|
+
profile: { id: record.profile.id, version: record.profile.version, digest: record.profileDigest },
|
|
332
|
+
canaryPassed: complete && results.every(r => r.pass), stopped,
|
|
333
|
+
oversizedDispatches: results.reduce((n, r) => n + r.oversizedDispatches, 0),
|
|
334
|
+
outputOversizedDispatches: results.reduce((n, r) => n + r.outputOversizedDispatches, 0),
|
|
335
|
+
enforceDispatches: results.filter(r => r.phase === 'enforce').reduce((n, r) => n + r.dispatches, 0),
|
|
336
|
+
rollbackExercised: complete && results.filter(r => r.phase === 'rollback').every(r => r.pass),
|
|
337
|
+
rollbackNativeDispatches: results.filter(r => r.phase === 'rollback').reduce((n, r) => n + r.nativeDispatches, 0),
|
|
338
|
+
cases: results.length, reserved: { requests: budget.requests, tokens: budget.tokens, usd: budget.usd }, issueSpend,
|
|
339
|
+
requestCapacityAtStop: contextLiveRequestCapacity({ ...plan, budget: { ...plan.budget, usd: issueSpend.runUsdCeiling } }), elapsedMs: Date.now() - started };
|
|
340
|
+
await writeFile(join(directory, 'summary.json'), canonicalJson(summary), { flag: 'wx', mode: 0o600 });
|
|
341
|
+
await assertContextLiveSource(sourceRoot, approval.sourceCommit);
|
|
342
|
+
return summary;
|
|
343
|
+
}
|
|
344
|
+
//# sourceMappingURL=context-live-promotion.js.map
|