@aiwg/cli 2026.9.21 → 2026.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agentic/code/providers/capability-matrix.yaml +2 -1
- package/dist/src/artifacts/cli.js +23 -18
- package/dist/src/artifacts/discovery-shadow-evaluator.js +177 -0
- package/dist/src/artifacts/discovery-shadow.js +195 -0
- package/dist/src/artifacts/fortemi-core-query-adapter.js +6 -2
- package/dist/src/artifacts/index-builder.js +47 -9
- package/dist/src/artifacts/query-engine.js +108 -5
- package/dist/src/artifacts/types.js +1 -0
- package/dist/src/cli/handlers/decision.js +88 -0
- package/dist/src/cli/handlers/gates.js +154 -0
- package/dist/src/cli/handlers/index.js +5 -1
- package/dist/src/cli/handlers/sessions.js +10 -7
- package/dist/src/cli/handlers/utilities.js +20 -0
- package/dist/src/config/aiwg-config.js +15 -2
- package/dist/src/decision/acceptance.js +25 -0
- package/dist/src/decision/adapters/jev.js +24 -2
- package/dist/src/decision/calibration/registry.js +9 -0
- package/dist/src/decision/compose.js +74 -15
- package/dist/src/decision/context-live-promotion.js +344 -0
- package/dist/src/decision/context-live-qualification.js +591 -0
- package/dist/src/decision/context-pruning.js +847 -0
- package/dist/src/decision/driver.js +430 -0
- package/dist/src/decision/egress-live-qualification.js +874 -0
- package/dist/src/decision/ensemble/contract.js +64 -1
- package/dist/src/decision/ensemble/index.js +1 -0
- package/dist/src/decision/ensemble/report.js +8 -42
- package/dist/src/decision/ensemble/runtime.js +571 -0
- package/dist/src/decision/ensemble-study/artifacts.js +85 -0
- package/dist/src/decision/ensemble-study/calibration.js +443 -0
- package/dist/src/decision/ensemble-study/corpus.js +229 -0
- package/dist/src/decision/ensemble-study/multifact-artifacts.js +23 -0
- package/dist/src/decision/ensemble-study/multifact-study.js +596 -0
- package/dist/src/decision/ensemble-study/multifact.js +241 -0
- package/dist/src/decision/ensemble-study/protocol.js +77 -0
- package/dist/src/decision/ensemble-study/score.js +462 -0
- package/dist/src/decision/ensemble-study/staged.js +87 -0
- package/dist/src/decision/ensemble-study/statistics.js +84 -0
- package/dist/src/decision/evaluate.js +227 -14
- package/dist/src/decision/export.js +4 -0
- package/dist/src/decision/feature-export.js +1074 -0
- package/dist/src/decision/graph-index.js +16 -0
- package/dist/src/decision/graph-live-qualification.js +1041 -0
- package/dist/src/decision/graph-live-workload.js +245 -0
- package/dist/src/decision/graph-skill-bridge.js +21 -3
- package/dist/src/decision/heldout/calibration.js +140 -0
- package/dist/src/decision/heldout/collector.js +430 -0
- package/dist/src/decision/heldout/contract.js +309 -0
- package/dist/src/decision/heldout/d29-generator-ids.js +12 -0
- package/dist/src/decision/heldout/d29-passage-baseline-v2.js +93 -0
- package/dist/src/decision/heldout/d29-passage-baseline-v3.js +100 -0
- package/dist/src/decision/heldout/d29-passage-baseline.js +66 -0
- package/dist/src/decision/heldout/d29-pools-v8.js +575 -0
- package/dist/src/decision/heldout/d29-pools.js +288 -0
- package/dist/src/decision/heldout/d29-v6.js +169 -0
- package/dist/src/decision/heldout/d29-v7.js +202 -0
- package/dist/src/decision/heldout/d29-v8.js +586 -0
- package/dist/src/decision/heldout/generator-registry.js +67 -0
- package/dist/src/decision/heldout/generators.js +696 -0
- package/dist/src/decision/heldout/index.js +8 -0
- package/dist/src/decision/heldout/journal.js +346 -0
- package/dist/src/decision/heldout/study-generators.js +64 -0
- package/dist/src/decision/heldout/types.js +2 -0
- package/dist/src/decision/host-policy.js +61 -0
- package/dist/src/decision/index.js +11 -0
- package/dist/src/decision/interop-profile.v1.json +112 -0
- package/dist/src/decision/interop.js +653 -0
- package/dist/src/decision/issue-triage/index.js +3 -0
- package/dist/src/decision/issue-triage/runtime.js +1069 -0
- package/dist/src/decision/issue-triage/types.js +2 -0
- package/dist/src/decision/patterns/catalog.js +28 -8
- package/dist/src/decision/patterns/playground.js +28 -11
- package/dist/src/decision/patterns/specs.js +19 -10
- package/dist/src/decision/preprocessed-evidence.js +724 -0
- package/dist/src/decision/qualification/capture.js +8 -2
- package/dist/src/decision/qualification/gate-evidence.js +19 -4
- package/dist/src/decision/qualification/load.js +8 -1
- package/dist/src/decision/qualification/quality.js +57 -49
- package/dist/src/decision/qualification/release.js +76 -3
- package/dist/src/decision/qualification/runner.js +5 -2
- package/dist/src/decision/routing/contract.js +175 -0
- package/dist/src/decision/routing/index.js +5 -0
- package/dist/src/decision/routing/report.js +216 -0
- package/dist/src/decision/routing/runtime.js +685 -0
- package/dist/src/decision/routing/types.js +2 -0
- package/dist/src/decision/sdlc-screening.js +1157 -0
- package/dist/src/decision/sensitivity/comparative.js +154 -0
- package/dist/src/decision/sensitivity/contract.js +218 -0
- package/dist/src/decision/sensitivity/index.js +5 -0
- package/dist/src/decision/sensitivity/runtime.js +912 -0
- package/dist/src/decision/sensitivity/types.js +2 -0
- package/dist/src/decision/validate.js +26 -0
- package/dist/src/extensions/commands/definitions.js +38 -0
- package/dist/src/extensions/manifest.js +44 -0
- package/dist/src/gates/discovery.js +315 -0
- package/dist/src/gates/driver.js +388 -0
- package/dist/src/gates/evaluate.js +719 -0
- package/dist/src/gates/floors.js +255 -0
- package/dist/src/gates/index.js +17 -0
- package/dist/src/gates/providers/evidence.js +38 -0
- package/dist/src/gates/providers/index.js +21 -0
- package/dist/src/gates/providers/isolated-child-source.js +446 -0
- package/dist/src/gates/providers/loader.js +1069 -0
- package/dist/src/gates/providers/paired.js +52 -0
- package/dist/src/gates/providers/proportion.js +60 -0
- package/dist/src/gates/providers/registry.js +40 -0
- package/dist/src/gates/providers/scalar.js +41 -0
- package/dist/src/gates/providers/screening.js +53 -0
- package/dist/src/gates/providers/types.js +25 -0
- package/dist/src/gates/registry.js +1003 -0
- package/dist/src/gates/report.js +102 -0
- package/dist/src/gates/schema.js +56 -0
- package/dist/src/gates/stats/binomial.js +26 -0
- package/dist/src/gates/stats/bootstrap.js +62 -0
- package/dist/src/gates/stats/clopper-pearson.js +95 -0
- package/dist/src/gates/stats/digest.js +26 -0
- package/dist/src/gates/stats/error.js +5 -0
- package/dist/src/gates/stats/index.js +9 -0
- package/dist/src/gates/stats/normal.js +35 -0
- package/dist/src/gates/stats/paired.js +120 -0
- package/dist/src/gates/stats/splits.js +30 -0
- package/dist/src/gates/types.js +8 -0
- package/dist/src/governance/redaction.js +14 -4
- package/dist/src/mcp/muse-native-config.mjs +123 -0
- package/dist/src/mcp/tools/decision.mjs +188 -0
- package/dist/src/mcp/tools/subsystems.mjs +2 -0
- package/dist/src/providers/capability-matrix.yaml +2 -1
- package/dist/src/providers/provider-definitions.js +2 -1
- package/dist/src/sessions/adapters/muse.js +336 -39
- package/dist/src/sessions/batch-import.js +7 -0
- package/dist/src/sessions/workspace-discovery.js +13 -2
- package/package.json +1 -1
- package/schemas/decision/ConformalFrozenOpenData.v2.schema.json +371 -0
- package/schemas/decision/ConformalOpenDataReport.v2.schema.json +421 -0
- package/schemas/decision/ConformalPreregistration.v2.schema.json +529 -0
- package/schemas/decision/ContextPruningCandidate.v1.schema.json +47 -0
- package/schemas/decision/ContextPruningEvaluationReport.v1.schema.json +758 -0
- package/schemas/decision/ContextPruningPreregistration.v1.schema.json +175 -0
- package/schemas/decision/ContextPruningReceipt.v1.schema.json +72 -0
- package/schemas/decision/D17Multifact.v1.schema.json +1851 -0
- package/schemas/decision/D17StudyAnalysis.v1.schema.json +42 -0
- package/schemas/decision/D17StudyAnalysis.v2.schema.json +51 -0
- package/schemas/decision/D17StudyCalibration.v1.schema.json +819 -0
- package/schemas/decision/D17StudyDryRun.v1.schema.json +169 -0
- package/schemas/decision/D17StudyDryRun.v2.schema.json +239 -0
- package/schemas/decision/D17StudyGold.v1.schema.json +108 -0
- package/schemas/decision/D17StudyGuide.v1.schema.json +63 -0
- package/schemas/decision/D17StudyNativeTemplates.v1.schema.json +619 -0
- package/schemas/decision/D17StudyProtocol.v1.schema.json +241 -0
- package/schemas/decision/D17StudyReport.v1.schema.json +902 -0
- package/schemas/decision/D17StudyReport.v2.schema.json +1319 -0
- package/schemas/decision/D17StudyReview.v1.schema.json +175 -0
- package/schemas/decision/D17StudySplits.v1.schema.json +217 -0
- package/schemas/decision/D29Study.v1.schema.json +1903 -0
- package/schemas/decision/D29Study.v2.schema.json +1344 -0
- package/schemas/decision/D29Study.v3.schema.json +609 -0
- package/schemas/decision/D29Study.v4.schema.json +1133 -0
- package/schemas/decision/D29Study.v6.schema.json +1205 -0
- package/schemas/decision/D29Study.v7.schema.json +1471 -0
- package/schemas/decision/D29Study.v8.schema.json +671 -0
- package/schemas/decision/D29Study.v9.schema.json +778 -0
- package/schemas/decision/DecisionComparativeReplayAuditSample.v1.schema.json +137 -0
- package/schemas/decision/DecisionComparativeReplayCorpus.v1.schema.json +258 -0
- package/schemas/decision/DecisionComparativeReplayFreeze.v1.schema.json +74 -0
- package/schemas/decision/DecisionComparativeReplayGold.v1.schema.json +50 -0
- package/schemas/decision/DecisionComparativeReplayGoldAsset.v1.schema.json +37 -0
- package/schemas/decision/DecisionComparativeReplayIntegrity.v1.schema.json +81 -0
- package/schemas/decision/DecisionComparativeReplayMembers.v1.schema.json +52 -0
- package/schemas/decision/DecisionComparativeReplayPreregistration.v1.schema.json +113 -0
- package/schemas/decision/DecisionComparativeReplayReport.v1.schema.json +407 -0
- package/schemas/decision/DecisionComparativeReplayReviewPackets.v1.schema.json +260 -0
- package/schemas/decision/DecisionComparativeReplaySplit.v1.schema.json +84 -0
- package/schemas/decision/DecisionFeatureObservation.v1.schema.json +564 -0
- package/schemas/decision/DecisionFeatureSet.v1.schema.json +155 -0
- package/schemas/decision/DecisionInteropMapping.v1.schema.json +85 -0
- package/schemas/decision/DecisionIssueTriageEvaluationManifest.v1.schema.json +174 -0
- package/schemas/decision/DecisionIssueTriageEvaluationReport.v1.schema.json +280 -0
- package/schemas/decision/DecisionIssueTriagePilotPack.v1.schema.json +114 -0
- package/schemas/decision/DecisionRouteReceipt.v1.schema.json +155 -0
- package/schemas/decision/DecisionRoutingPolicy.v1.schema.json +132 -0
- package/schemas/decision/DecisionRoutingShadowReport.v1.schema.json +170 -0
- package/schemas/decision/DecisionRoutingTask.v1.schema.json +49 -0
- package/schemas/decision/DecisionRuleset.schema.json +8 -2
- package/schemas/decision/DecisionRuleset.v1alpha2.schema.json +8 -2
- package/schemas/decision/DecisionSensitivityPlan.v1.schema.json +191 -0
- package/schemas/decision/DecisionSensitivityReport.v1.schema.json +163 -0
- package/schemas/decision/HeldoutApproval.v1.schema.json +312 -0
- package/schemas/decision/HeldoutAttempt.v1.schema.json +211 -0
- package/schemas/decision/HeldoutBaseline.v1.schema.json +90 -0
- package/schemas/decision/HeldoutCalibrationPhase.v1.schema.json +87 -0
- package/schemas/decision/HeldoutCorpus.v1.schema.json +167 -0
- package/schemas/decision/HeldoutEvent.v1.schema.json +41 -0
- package/schemas/decision/HeldoutFrozen.v1.schema.json +142 -0
- package/schemas/decision/HeldoutPreregistration.v1.schema.json +222 -0
- package/schemas/decision/HeldoutSpendEvent.v1.schema.json +122 -0
- package/schemas/decision/HeldoutSpendHead.v1.schema.json +25 -0
- package/schemas/decision/HeldoutSpendHead.v2.schema.json +46 -0
- package/schemas/decision/HeldoutSummary.v1.schema.json +101 -0
- package/schemas/decision/OpaDecisionLogExport.v1.schema.json +65 -0
- package/schemas/decision/PreprocessedEvidence.v1.schema.json +306 -0
- package/schemas/decision/RulesetResult.v1alpha2.schema.json +175 -0
- package/schemas/decision/SdlcEvidenceScreening.v1.schema.json +499 -0
- package/schemas/decision/SdlcScreeningPreregistration.v1.schema.json +132 -0
- package/schemas/decision/SdlcScreeningPreregistration.v2.schema.json +138 -0
- package/schemas/decision/SdlcScreeningRelease.v1.schema.json +654 -0
- package/schemas/decision/SdlcScreeningRelease.v2.schema.json +654 -0
- package/tools/agents/deploy-agents.mjs +1 -1
- package/tools/agents/providers/base.mjs +26 -7
- package/tools/agents/providers/factory.mjs +10 -2
- package/tools/agents/providers/muse-hooks.mjs +1 -29
- package/tools/agents/providers/muse.mjs +6 -3
|
@@ -34,6 +34,7 @@ export class JevDecisionAdapter {
|
|
|
34
34
|
pinnedFetch;
|
|
35
35
|
now;
|
|
36
36
|
region;
|
|
37
|
+
observeResponseBody;
|
|
37
38
|
constructor(options = {}) {
|
|
38
39
|
this.endpoint = options.endpoint ?? JEV_ENDPOINT;
|
|
39
40
|
this.fetchImpl = options.fetch ?? fetch;
|
|
@@ -42,6 +43,23 @@ export class JevDecisionAdapter {
|
|
|
42
43
|
this.pinnedFetch = options.pinnedFetch ?? pinnedHttpsFetch;
|
|
43
44
|
this.now = options.now ?? Date.now;
|
|
44
45
|
this.region = options.region?.trim() ? options.region : null;
|
|
46
|
+
this.observeResponseBody = options.observeResponseBody;
|
|
47
|
+
}
|
|
48
|
+
observeResponse(body) {
|
|
49
|
+
if (!this.observeResponseBody)
|
|
50
|
+
return;
|
|
51
|
+
let parsed = {};
|
|
52
|
+
try {
|
|
53
|
+
parsed = asRecord(parseBoundedJson(body));
|
|
54
|
+
}
|
|
55
|
+
catch { /* Invalid answers still have bounded body evidence. */ }
|
|
56
|
+
let usage = emptyUsage();
|
|
57
|
+
try {
|
|
58
|
+
usage = normalizeUsage(parsed.usage);
|
|
59
|
+
}
|
|
60
|
+
catch { /* Missing usage is unknown, including on auth errors. */ }
|
|
61
|
+
this.observeResponseBody(body, { actualModel: typeof parsed.model === 'string' && parsed.model ? parsed.model : null,
|
|
62
|
+
usage });
|
|
45
63
|
}
|
|
46
64
|
async capabilities() {
|
|
47
65
|
return {
|
|
@@ -167,6 +185,7 @@ export class JevDecisionAdapter {
|
|
|
167
185
|
if (!response.ok) {
|
|
168
186
|
try {
|
|
169
187
|
const errorBody = await readBoundedBody(response, signal);
|
|
188
|
+
this.observeResponse(errorBody);
|
|
170
189
|
if (!metadata.requestId && response.headers.get('content-type')?.includes('json')) {
|
|
171
190
|
let parsedError;
|
|
172
191
|
try {
|
|
@@ -198,6 +217,7 @@ export class JevDecisionAdapter {
|
|
|
198
217
|
let parsed;
|
|
199
218
|
try {
|
|
200
219
|
const encoded = await readBoundedBody(response, signal);
|
|
220
|
+
this.observeResponse(encoded);
|
|
201
221
|
parsed = parseBoundedJson(encoded);
|
|
202
222
|
if (request.signal.aborted)
|
|
203
223
|
return externalInterruption(request, 'terminal-response', metadata);
|
|
@@ -304,7 +324,7 @@ export class JevDecisionAdapter {
|
|
|
304
324
|
}
|
|
305
325
|
if (!response.ok) {
|
|
306
326
|
try {
|
|
307
|
-
await readBoundedBody(response, signal);
|
|
327
|
+
this.observeResponse(await readBoundedBody(response, signal));
|
|
308
328
|
}
|
|
309
329
|
catch {
|
|
310
330
|
return batchFailure(requests, failure('invalid-output', metadata));
|
|
@@ -313,7 +333,9 @@ export class JevDecisionAdapter {
|
|
|
313
333
|
return batchFailure(requests, failure(mapStatus(response.status), { ...metadata, ...(retryAfterMs === null ? {} : { retryAfterMs }) }));
|
|
314
334
|
}
|
|
315
335
|
try {
|
|
316
|
-
const
|
|
336
|
+
const encoded = await readBoundedBody(response, signal);
|
|
337
|
+
this.observeResponse(encoded);
|
|
338
|
+
const parsed = parseBoundedJson(encoded);
|
|
317
339
|
const normalized = normalizeBatchResponse(requests, parsed).map(answer => ({
|
|
318
340
|
questionId: answer.questionId,
|
|
319
341
|
observation: { ...answer.observation, ...metadata },
|
|
@@ -88,6 +88,11 @@ export class CalibrationRegistry {
|
|
|
88
88
|
const rollback = history.find(event => event.revision === eligibility.rollbackTarget.aliasRevision);
|
|
89
89
|
if (!rollback || rollback.actualIdentityDigest !== eligibility.rollbackTarget.identityDigest)
|
|
90
90
|
throw new CalibrationRegistryError('rollback target no longer matches immutable alias history');
|
|
91
|
+
const current = history.at(-1);
|
|
92
|
+
if (current !== rollback || current.kind === 'retired')
|
|
93
|
+
throw new CalibrationRegistryError('alias no longer holds the champion this promotion was approved to replace');
|
|
94
|
+
if (history.some(event => event.promotionEligibilityId === eligibility.id))
|
|
95
|
+
throw new CalibrationRegistryError(`promotion eligibility '${eligibility.id}' was already used`);
|
|
91
96
|
const event = immutable({ revision: history.length + 1, alias: eligibility.alias,
|
|
92
97
|
actualIdentityDigest: eligibility.candidateIdentityDigest, actualModel: eligibility.candidateActualModel, recordedAt: new Date(validDate(at)).toISOString(),
|
|
93
98
|
kind: 'promoted', promotionEligibilityId: eligibility.id });
|
|
@@ -172,6 +177,7 @@ export class CalibrationRegistry {
|
|
|
172
177
|
aliasHistory(alias) { return structuredClone(this.aliases.get(alias) ?? []); }
|
|
173
178
|
driftEvents(alias) { return structuredClone(alias ? this.drift.filter(event => event.alias === alias) : this.drift); }
|
|
174
179
|
promotionHistory() { return structuredClone([...this.eligibility.values()]); }
|
|
180
|
+
promotionEligibility(id) { const record = this.eligibility.get(id); return record ? structuredClone(record) : null; }
|
|
175
181
|
artifactHistory() { return structuredClone([...this.artifacts.values()]); }
|
|
176
182
|
compatibilityHistory() { return structuredClone(this.relations); }
|
|
177
183
|
}
|
|
@@ -190,6 +196,9 @@ function calibrationUsability(artifact, at) {
|
|
|
190
196
|
reasons.push('calibration-bound-exceeded');
|
|
191
197
|
if (artifact.metrics.selectiveRisk > artifact.profile.maximumSelectiveRisk)
|
|
192
198
|
reasons.push('selective-risk-bound-exceeded');
|
|
199
|
+
// An artifact is unusable before it takes effect: a back-dated request cannot borrow a later calibration.
|
|
200
|
+
if (validDate(at) < validDate(artifact.effectiveAt))
|
|
201
|
+
reasons.push('calibration-not-yet-effective');
|
|
193
202
|
if (validDate(at) >= validDate(artifact.effectiveAt) + artifact.profile.expiresAfterDays * 86_400_000)
|
|
194
203
|
reasons.push('calibration-expired');
|
|
195
204
|
return reasons;
|
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
import { evaluatePredicate } from './predicates.js';
|
|
2
2
|
import { validateAgainstSchema } from './validate.js';
|
|
3
3
|
export function composeRuleset(ruleset, input, evaluations) {
|
|
4
|
-
const
|
|
4
|
+
const referencedAliases = referencedDecisionAliases(ruleset);
|
|
5
|
+
const requiredEvaluations = new Map(ruleset.spec.evaluations
|
|
6
|
+
.filter(evaluation => referencedAliases.includes(evaluation.alias))
|
|
7
|
+
.map(evaluation => [evaluation.alias, evaluation]));
|
|
8
|
+
const suppliedFailed = Object.values(evaluations).some(result => !acceptedEvaluation(result));
|
|
9
|
+
const requiredFailed = [...requiredEvaluations].some(([alias, evaluation]) => {
|
|
10
|
+
const result = evaluations[alias];
|
|
11
|
+
return !result || result.spec.alias !== alias || !acceptedEvaluation(result) || !samePin(result.spec.decision, evaluation.decision);
|
|
12
|
+
});
|
|
13
|
+
const failed = suppliedFailed || requiredFailed;
|
|
5
14
|
if (failed) {
|
|
6
15
|
validateAgainstSchema(ruleset.spec.outputSchema, ruleset.spec.failureOutcome, 'failureOutcome');
|
|
7
16
|
return {
|
|
@@ -18,31 +27,81 @@ export function composeRuleset(ruleset, input, evaluations) {
|
|
|
18
27
|
validateAgainstSchema(ruleset.spec.outputSchema, ruleset.spec.defaultOutcome, 'defaultOutcome');
|
|
19
28
|
return { status: 'defaulted', reason: 'no-match', outcome: structuredClone(ruleset.spec.defaultOutcome), matchedRules: [] };
|
|
20
29
|
}
|
|
21
|
-
if (ruleset.spec.composition === 'collect') {
|
|
22
|
-
const
|
|
30
|
+
if (ruleset.spec.composition === 'collect' || ruleset.spec.composition.startsWith('collect-')) {
|
|
31
|
+
const collected = matches.map(rule => structuredClone(rule.outcome));
|
|
32
|
+
const outcome = ruleset.spec.composition === 'collect' ? collected : aggregateCollect(collected, ruleset.spec.composition);
|
|
23
33
|
validateAgainstSchema(ruleset.spec.outputSchema, outcome, 'collected outcome');
|
|
24
34
|
return { status: 'completed', reason: 'none', outcome, matchedRules: matches.map(rule => rule.id) };
|
|
25
35
|
}
|
|
36
|
+
if (ruleset.spec.composition === 'unique' && matches.length > 1)
|
|
37
|
+
return conflictResult(ruleset, matches);
|
|
38
|
+
if (ruleset.spec.composition === 'any') {
|
|
39
|
+
const first = sortJson(matches[0].outcome);
|
|
40
|
+
if (matches.some(rule => JSON.stringify(sortJson(rule.outcome)) !== JSON.stringify(first)))
|
|
41
|
+
return conflictResult(ruleset, matches);
|
|
42
|
+
const outcome = structuredClone(matches[0].outcome);
|
|
43
|
+
validateAgainstSchema(ruleset.spec.outputSchema, outcome, 'matched outcome');
|
|
44
|
+
return { status: 'completed', reason: 'none', outcome, matchedRules: matches.map(rule => rule.id) };
|
|
45
|
+
}
|
|
26
46
|
const top = matches.filter(rule => rule.priority === matches[0].priority);
|
|
27
47
|
const unique = new Map();
|
|
28
48
|
for (const rule of top)
|
|
29
49
|
unique.set(JSON.stringify(sortJson(rule.outcome)), rule.outcome);
|
|
30
|
-
if (unique.size > 1)
|
|
31
|
-
|
|
32
|
-
validateAgainstSchema(ruleset.spec.outputSchema, ruleset.spec.failureOutcome, 'failureOutcome');
|
|
33
|
-
return {
|
|
34
|
-
status: 'review',
|
|
35
|
-
reason: 'conflicting-outcomes',
|
|
36
|
-
outcome: structuredClone(ruleset.spec.failureOutcome),
|
|
37
|
-
matchedRules: top.map(rule => rule.id),
|
|
38
|
-
};
|
|
39
|
-
}
|
|
40
|
-
return { status: 'error', reason: 'conflicting-outcomes', matchedRules: top.map(rule => rule.id) };
|
|
41
|
-
}
|
|
50
|
+
if (unique.size > 1)
|
|
51
|
+
return conflictResult(ruleset, top);
|
|
42
52
|
const outcome = structuredClone(top[0].outcome);
|
|
43
53
|
validateAgainstSchema(ruleset.spec.outputSchema, outcome, 'matched outcome');
|
|
44
54
|
return { status: 'completed', reason: 'none', outcome, matchedRules: top.map(rule => rule.id) };
|
|
45
55
|
}
|
|
56
|
+
function acceptedEvaluation(result) {
|
|
57
|
+
return result.spec.status === 'success'
|
|
58
|
+
&& (result.spec.acceptance === undefined || result.spec.acceptance.disposition === 'act');
|
|
59
|
+
}
|
|
60
|
+
function samePin(left, right) {
|
|
61
|
+
return left.id === right.id && left.version === right.version && left.digest === right.digest;
|
|
62
|
+
}
|
|
63
|
+
function referencedDecisionAliases(ruleset) {
|
|
64
|
+
const aliases = new Set();
|
|
65
|
+
const visit = (predicate) => {
|
|
66
|
+
if ('all' in predicate)
|
|
67
|
+
predicate.all.forEach(visit);
|
|
68
|
+
else if ('any' in predicate)
|
|
69
|
+
predicate.any.forEach(visit);
|
|
70
|
+
else if ('not' in predicate)
|
|
71
|
+
visit(predicate.not);
|
|
72
|
+
else if (predicate.left.source === 'decision' && predicate.left.alias)
|
|
73
|
+
aliases.add(predicate.left.alias);
|
|
74
|
+
};
|
|
75
|
+
ruleset.spec.rules.forEach(rule => visit(rule.when));
|
|
76
|
+
return [...aliases].sort();
|
|
77
|
+
}
|
|
78
|
+
function conflictResult(ruleset, matches) {
|
|
79
|
+
if (ruleset.spec.conflict === 'review') {
|
|
80
|
+
validateAgainstSchema(ruleset.spec.outputSchema, ruleset.spec.failureOutcome, 'failureOutcome');
|
|
81
|
+
return {
|
|
82
|
+
status: 'review',
|
|
83
|
+
reason: 'conflicting-outcomes',
|
|
84
|
+
outcome: structuredClone(ruleset.spec.failureOutcome),
|
|
85
|
+
matchedRules: matches.map(rule => rule.id),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
return { status: 'error', reason: 'conflicting-outcomes', matchedRules: matches.map(rule => rule.id) };
|
|
89
|
+
}
|
|
90
|
+
function aggregateCollect(values, composition) {
|
|
91
|
+
if (composition === 'collect-count')
|
|
92
|
+
return values.length;
|
|
93
|
+
if (!values.every(value => typeof value === 'number' && Number.isFinite(value))) {
|
|
94
|
+
throw new Error(`${composition} requires finite numeric outcomes`);
|
|
95
|
+
}
|
|
96
|
+
const numbers = values;
|
|
97
|
+
if (composition === 'collect-sum')
|
|
98
|
+
return numbers.reduce((sum, value) => sum + value, 0);
|
|
99
|
+
if (composition === 'collect-min')
|
|
100
|
+
return Math.min(...numbers);
|
|
101
|
+
if (composition === 'collect-max')
|
|
102
|
+
return Math.max(...numbers);
|
|
103
|
+
return values;
|
|
104
|
+
}
|
|
46
105
|
function sortJson(value) {
|
|
47
106
|
if (Array.isArray(value))
|
|
48
107
|
return value.map(sortJson);
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TV-12 promotion steps after collection (#2681): record the reviewer-approved profile and margin chosen
|
|
3
|
+
* by the preregistered rule, then run an enforce-mode canary with an exercised rollback to single calls.
|
|
4
|
+
* Neither step changes a default: enforcement still needs a caller to pass the recorded qualification.
|
|
5
|
+
*/
|
|
6
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
7
|
+
import { join, resolve } from 'node:path';
|
|
8
|
+
import { canonicalJson } from '../security/artifact-trust.js';
|
|
9
|
+
import { admitEntry } from './entry.js';
|
|
10
|
+
import { CanonicalJsonByteEstimator, ContextPlanError, planDecisionContext } from './context-plan.js';
|
|
11
|
+
import { assertContextQualified, compareContextUsage } from './context-qualification.js';
|
|
12
|
+
import { assertContextArtifactRoot, assertContextLiveSource, compileContextLiveCase, contextLiveBinding, contextLiveDigest, contextLiveRequestCapacity, contextLiveRuleset, contextLiveTarget, ContextLiveBudget, validateContextLiveApproval, withContextLiveSpendLedger, TV12_ISSUE_USD_CAP, CREDENTIAL_REF, } from './context-live-qualification.js';
|
|
13
|
+
import { JevDecisionAdapter } from './adapters/jev.js';
|
|
14
|
+
import { decisionBatchQuestionId } from './batch.js';
|
|
15
|
+
import { contextResultEnvelopeFits, evaluateDecisionRuleset } from './evaluate.js';
|
|
16
|
+
const estimator = new CanonicalJsonByteEstimator();
|
|
17
|
+
const subject = 'synthetic-tv12';
|
|
18
|
+
const nonblank = (value) => typeof value === 'string' && value.trim().length > 0;
|
|
19
|
+
const sha256 = /^sha256:[a-f0-9]{64}$/;
|
|
20
|
+
const closed = (value, keys) => !!value && typeof value === 'object' && !Array.isArray(value)
|
|
21
|
+
&& Object.keys(value).sort().join(',') === [...keys].sort().join(',');
|
|
22
|
+
export const contextLiveRecordsDigest = (records) => contextLiveDigest([...records]
|
|
23
|
+
.sort((a, b) => `${a.caseId}\0${a.partitionId}`.localeCompare(`${b.caseId}\0${b.partitionId}`, 'en')));
|
|
24
|
+
/** Rebinds every retained comparison to the frozen corpus and returns exactly one complete provider request per partition. */
|
|
25
|
+
async function boundComparisons(approval, corpus, records) {
|
|
26
|
+
const samples = [];
|
|
27
|
+
const used = new Set();
|
|
28
|
+
for (const item of corpus.cases) {
|
|
29
|
+
const { input } = await compileContextLiveCase(item, approval.model, approval.region);
|
|
30
|
+
let plan;
|
|
31
|
+
try {
|
|
32
|
+
plan = planDecisionContext(input, corpus.profile, estimator);
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
if (error instanceof ContextPlanError && ['oversized-state', 'oversized-question'].includes(error.reason))
|
|
36
|
+
continue;
|
|
37
|
+
throw error;
|
|
38
|
+
}
|
|
39
|
+
for (const partition of plan.partitions) {
|
|
40
|
+
const partitionInput = { ...input, questions: input.questions.filter(q => partition.questionIds.includes(q.id)) };
|
|
41
|
+
const checked = planDecisionContext(partitionInput, corpus.profile, estimator);
|
|
42
|
+
const matches = records.filter(r => r.caseId === item.id && r.partitionId === partition.id);
|
|
43
|
+
const record = matches[0];
|
|
44
|
+
const usage = record && { source: record.source, servedModel: record.servedModel, requestId: record.requestId,
|
|
45
|
+
inputTokens: record.inputTokens, outputTokens: record.outputTokens, costUsd: record.costUsd };
|
|
46
|
+
if (matches.length !== 1 || !record || checked.partitions.length !== 1 || record.source !== 'provider' || record.servedModel !== approval.model
|
|
47
|
+
|| record.planDigest !== checked.planDigest || record.profileDigest !== checked.providerProfile.digest
|
|
48
|
+
|| record.estimator.id !== estimator.id || record.estimator.version !== estimator.version
|
|
49
|
+
|| record.estimatedInputTokens !== checked.partitions[0].estimate.aggregateTokens
|
|
50
|
+
|| record.actualInputTokens !== record.inputTokens || record.usageArtifactDigest !== contextLiveDigest(usage)
|
|
51
|
+
|| record.usageRef !== `usage:${record.usageArtifactDigest}`) {
|
|
52
|
+
throw new ContextPlanError('invalid-input', 'retained comparison does not bind to the frozen corpus partition');
|
|
53
|
+
}
|
|
54
|
+
used.add(record);
|
|
55
|
+
samples.push({ caseId: `${item.id}:${partition.id}`, input: partitionInput, actualInputTokens: record.actualInputTokens, source: 'provider', usageRef: record.usageRef });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if (!samples.length || used.size !== records.length)
|
|
59
|
+
throw new ContextPlanError('invalid-input', 'retained comparisons must cover exactly the planned partitions');
|
|
60
|
+
return samples;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Applies the preregistered rule (worst observed undercount plus the frozen reserve, capped by the frozen
|
|
64
|
+
* maximum) and records the reviewer's matching approval. Synthetic, partial or unbound evidence throws.
|
|
65
|
+
*/
|
|
66
|
+
export async function recordContextQualification(options) {
|
|
67
|
+
const approval = structuredClone(options.approval), corpus = structuredClone(options.corpus);
|
|
68
|
+
const records = structuredClone(options.records), review = structuredClone(options.review);
|
|
69
|
+
validateContextLiveApproval(approval, corpus);
|
|
70
|
+
admitEntry(review);
|
|
71
|
+
const { marginRule } = approval;
|
|
72
|
+
// A selected margin above the collection margin could split a request that was collected whole.
|
|
73
|
+
if (corpus.profile.safetyMarginBps < marginRule.maximumMarginBps)
|
|
74
|
+
throw new ContextPlanError('invalid-profile', 'collection margin must cover the preregistered maximum');
|
|
75
|
+
const samples = await boundComparisons(approval, corpus, records);
|
|
76
|
+
const observed = compareContextUsage(samples, corpus.profile, estimator);
|
|
77
|
+
for (const row of observed.cases) {
|
|
78
|
+
const [caseId, partitionId] = row.caseId.split(':');
|
|
79
|
+
if (records.find(r => r.caseId === caseId && r.partitionId === partitionId).undercountBps !== row.undercountBps) {
|
|
80
|
+
throw new ContextPlanError('invalid-input', 'retained undercount differs from recomputation');
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const candidateMarginBps = observed.worstUndercountBps + marginRule.extraReserveBps;
|
|
84
|
+
if (candidateMarginBps > marginRule.maximumMarginBps) {
|
|
85
|
+
throw new ContextPlanError('rollout-unqualified', 'observed undercount exceeds the preregistered maximum margin');
|
|
86
|
+
}
|
|
87
|
+
const recordsDigest = contextLiveRecordsDigest(records);
|
|
88
|
+
if (!closed(review, ['schemaVersion', 'approved', 'reviewer', 'approvalReference', 'runId', 'recordsDigest', 'selectedMarginBps', 'qualifiedProfile'])
|
|
89
|
+
|| !closed(review.qualifiedProfile, ['id', 'version']) || review.schemaVersion !== 'context-margin-review/v1' || review.approved !== true
|
|
90
|
+
|| !nonblank(review.reviewer) || !nonblank(review.approvalReference) || review.runId !== approval.runId || review.recordsDigest !== recordsDigest
|
|
91
|
+
|| review.selectedMarginBps !== candidateMarginBps || !nonblank(review.qualifiedProfile.id) || !nonblank(review.qualifiedProfile.version)
|
|
92
|
+
|| review.qualifiedProfile.id === corpus.profile.id && review.qualifiedProfile.version === corpus.profile.version) {
|
|
93
|
+
throw new ContextPlanError('rollout-unqualified', 'margin review does not approve the preregistered selection for these records');
|
|
94
|
+
}
|
|
95
|
+
const profile = { ...structuredClone(corpus.profile), id: review.qualifiedProfile.id,
|
|
96
|
+
version: review.qualifiedProfile.version, safetyMarginBps: candidateMarginBps };
|
|
97
|
+
const qualification = compareContextUsage(samples, profile, estimator);
|
|
98
|
+
assertContextQualified(qualification, profile, estimator);
|
|
99
|
+
return { schemaVersion: 'context-qualification-record/v1', runId: approval.runId, sourceCommit: approval.sourceCommit,
|
|
100
|
+
model: approval.model, region: approval.region, corpusDigest: approval.corpusDigest, preregistrationDigest: approval.preregistrationDigest,
|
|
101
|
+
approvalDigest: contextLiveDigest(approval), recordsDigest, collectionProfileDigest: contextLiveDigest(corpus.profile), marginRule,
|
|
102
|
+
selection: { worstUndercountBps: observed.worstUndercountBps, extraReserveBps: marginRule.extraReserveBps, candidateMarginBps,
|
|
103
|
+
maximumMarginBps: marginRule.maximumMarginBps }, review, profile, profileDigest: contextLiveDigest(profile), qualification };
|
|
104
|
+
}
|
|
105
|
+
/** Enforcement gate for a stored record. Any profile change, including only its version, invalidates it. */
|
|
106
|
+
export function verifyContextQualificationRecord(record, profile = record?.profile) {
|
|
107
|
+
const s = record?.selection;
|
|
108
|
+
if (record?.schemaVersion !== 'context-qualification-record/v1' || !s || s.candidateMarginBps !== s.worstUndercountBps + s.extraReserveBps
|
|
109
|
+
|| s.candidateMarginBps > s.maximumMarginBps || record.qualification?.worstUndercountBps !== s.worstUndercountBps
|
|
110
|
+
|| record.review?.approved !== true || !nonblank(record.review.reviewer) || record.review.selectedMarginBps !== s.candidateMarginBps
|
|
111
|
+
|| record.review.qualifiedProfile?.id !== profile?.id || record.review.qualifiedProfile?.version !== profile?.version
|
|
112
|
+
|| profile.safetyMarginBps !== s.candidateMarginBps || record.profileDigest !== contextLiveDigest(profile)) {
|
|
113
|
+
throw new ContextPlanError('rollout-unqualified', 'qualification record does not match the enforced profile');
|
|
114
|
+
}
|
|
115
|
+
assertContextQualified(record.qualification, profile, estimator);
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* The evaluator preflights the worst-case completion envelope before dispatch (#2797): every
|
|
119
|
+
* evaluation carries every usage entry with the binding's full retry attempts, so the 24-question
|
|
120
|
+
* `many-short` case rejects with `invalid-input` before any provider work instead of exceeding
|
|
121
|
+
* the default entry limits (`property-count`) after dispatch. The canary covers that shape as a
|
|
122
|
+
* rejected-before-dispatch case with zero dispatches in both phases.
|
|
123
|
+
*/
|
|
124
|
+
export function validateContextCanaryApproval(approval, corpus, record) {
|
|
125
|
+
admitEntry(approval);
|
|
126
|
+
const plan = approval?.plan;
|
|
127
|
+
if (!closed(approval, ['schemaVersion', 'approved', 'reviewer', 'stagingWorkspace', 'runId', 'sourceCommit', 'exactHeadCi', 'model', 'apiRevision', 'region',
|
|
128
|
+
'secretServiceReference', 'credentialResolverDigest', 'corpusDigest', 'qualificationRecordDigest', 'canaryPlanDigest', 'plan'])
|
|
129
|
+
|| !closed(plan, ['schemaVersion', 'caseIds', 'rollback', 'budget', 'perRequestBound']) || !closed(plan.budget, ['requests', 'tokens', 'usd', 'wallClockMs'])
|
|
130
|
+
|| !closed(plan.perRequestBound, ['totalTokens', 'outputTokens', 'usd', 'approvalReference'])
|
|
131
|
+
|| approval.schemaVersion !== 'context-canary-approval/v1' || approval.approved !== true || plan.schemaVersion !== 'context-canary-plan/v1' || plan.rollback !== 'observe-only'
|
|
132
|
+
|| ![approval.reviewer, approval.stagingWorkspace, approval.exactHeadCi, approval.model, approval.region, approval.secretServiceReference, plan.perRequestBound.approvalReference].every(nonblank)
|
|
133
|
+
|| !/^[a-zA-Z0-9_-]+$/.test(approval.runId) || !/^[a-f0-9]{40}$/.test(approval.sourceCommit) || approval.apiRevision !== 'v1' || /latest|unknown/i.test(approval.model)
|
|
134
|
+
|| !CREDENTIAL_REF.test(approval.secretServiceReference)
|
|
135
|
+
|| ![approval.credentialResolverDigest, approval.corpusDigest, approval.qualificationRecordDigest, approval.canaryPlanDigest].every(d => sha256.test(d))
|
|
136
|
+
|| approval.canaryPlanDigest !== contextLiveDigest(plan) || approval.corpusDigest !== contextLiveDigest(corpus) || approval.corpusDigest !== record.corpusDigest
|
|
137
|
+
|| approval.qualificationRecordDigest !== contextLiveDigest(record) || approval.model !== record.model || approval.region !== record.region
|
|
138
|
+
|| !Array.isArray(plan.caseIds) || !plan.caseIds.length || new Set(plan.caseIds).size !== plan.caseIds.length
|
|
139
|
+
|| plan.caseIds.some(id => !corpus.cases.some(c => c.id === id))
|
|
140
|
+
|| plan.budget.usd > TV12_ISSUE_USD_CAP
|
|
141
|
+
|| [plan.budget.requests, plan.budget.tokens, plan.budget.wallClockMs, plan.perRequestBound.totalTokens, plan.perRequestBound.outputTokens].some(n => !Number.isSafeInteger(n) || n < 1)
|
|
142
|
+
|| plan.perRequestBound.outputTokens > plan.perRequestBound.totalTokens
|
|
143
|
+
|| [plan.budget.usd, plan.perRequestBound.usd].some(n => !Number.isFinite(n) || n <= 0 || n > Number.MAX_SAFE_INTEGER / 1_000_000)) {
|
|
144
|
+
throw new Error('Incomplete or mismatched TV-12 canary approval');
|
|
145
|
+
}
|
|
146
|
+
verifyContextQualificationRecord(record);
|
|
147
|
+
}
|
|
148
|
+
/** Dispatches each phase would need, derived from the qualified plan; rejected phases need none. */
|
|
149
|
+
export async function contextCanaryDispatches(caseIds, corpus, record) {
|
|
150
|
+
const rows = [];
|
|
151
|
+
const adapterVersion = new JevDecisionAdapter({ region: record.region }).version;
|
|
152
|
+
for (const id of caseIds) {
|
|
153
|
+
const item = corpus.cases.find(c => c.id === id);
|
|
154
|
+
const { input } = await compileContextLiveCase(item, record.model, record.region);
|
|
155
|
+
const aliases = item.definitions.map((_, index) => `q${index}`);
|
|
156
|
+
// Mirror of the dispatch binding the canary builds per phase (single target without retries,
|
|
157
|
+
// one attempt budget per alias); the probe reads only retry ceilings and identity, never
|
|
158
|
+
// credentials, so no credential reference is carried here.
|
|
159
|
+
const envelopeBinding = { spec: { maxAttempts: aliases.length,
|
|
160
|
+
evaluations: Object.fromEntries(aliases.map(alias => [alias, { targets: [{ adapter: 'jev', adapterVersion,
|
|
161
|
+
model: record.model, retry: { maxRetries: 0, initialDelayMs: 0, maxDelayMs: 0 } }] }])) } };
|
|
162
|
+
const envelopeItems = aliases.map((alias, index) => ({ alias, definition: item.definitions[index] }));
|
|
163
|
+
let enforce = 0, rollback = 0;
|
|
164
|
+
let enforceRejection = null;
|
|
165
|
+
let rollbackRejection = null;
|
|
166
|
+
try {
|
|
167
|
+
const planned = planDecisionContext(input, record.profile, estimator);
|
|
168
|
+
// Both phases evaluate every alias in one invocation, so both check the same worst-case
|
|
169
|
+
// envelope (#2797) — each phase with its own call. The worst-case single-dispatch shape
|
|
170
|
+
// bounds the native enforce batches as well as the single rollback dispatches, so a
|
|
171
|
+
// rejected envelope needs no dispatch in that phase.
|
|
172
|
+
for (const phase of ['enforce', 'rollback']) {
|
|
173
|
+
if (!contextResultEnvelopeFits(planned, envelopeItems, envelopeBinding)) {
|
|
174
|
+
if (phase === 'enforce')
|
|
175
|
+
enforceRejection = 'invalid-input';
|
|
176
|
+
else
|
|
177
|
+
rollbackRejection = 'invalid-input';
|
|
178
|
+
}
|
|
179
|
+
else if (phase === 'enforce')
|
|
180
|
+
enforce = planned.partitions.length;
|
|
181
|
+
else
|
|
182
|
+
rollback = input.questions.length;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
catch (error) {
|
|
186
|
+
if (!(error instanceof ContextPlanError && ['oversized-state', 'oversized-question'].includes(error.reason)))
|
|
187
|
+
throw error;
|
|
188
|
+
enforceRejection = error.reason;
|
|
189
|
+
rollbackRejection = error.reason;
|
|
190
|
+
}
|
|
191
|
+
rows.push({ caseId: id, enforce, rollback, enforceRejection, rollbackRejection });
|
|
192
|
+
}
|
|
193
|
+
return rows;
|
|
194
|
+
}
|
|
195
|
+
function inspect(caseId, phase, expected, expectedRejection, result, aliases, model, bound, outputBound) {
|
|
196
|
+
const plan = result.spec.context?.plan ?? null;
|
|
197
|
+
const usage = result.spec.context?.actualUsage ?? [];
|
|
198
|
+
const evaluations = Object.values(result.spec.evaluations ?? {});
|
|
199
|
+
const attempts = evaluations.flatMap(e => e.spec.attempts ?? []);
|
|
200
|
+
const partitions = usage.map(u => {
|
|
201
|
+
const partition = plan?.partitions.find(p => p.id === u.partitionId);
|
|
202
|
+
const withinEffectiveLimits = !!plan && !!partition && u.questionIds.every(id => partition.questionIds.includes(id))
|
|
203
|
+
&& partition.estimate.aggregateTokens <= plan.limits.effectiveAggregateTokens
|
|
204
|
+
&& partition.estimate.stateAndLongestQuestionTokens <= plan.limits.effectiveStateAndLongestQuestionTokens;
|
|
205
|
+
const withinDocumentedLimits = !!plan && Number.isSafeInteger(u.actualInputTokens) && u.actualInputTokens <= plan.limits.documentedAggregateTokens && u.actualInputTokens <= bound;
|
|
206
|
+
return { partitionId: u.partitionId, questions: u.questionIds.length, estimatedInputTokens: u.estimatedInputTokens, actualInputTokens: u.actualInputTokens, withinEffectiveLimits, withinDocumentedLimits };
|
|
207
|
+
});
|
|
208
|
+
const oversizedDispatches = partitions.filter(p => !p.withinEffectiveLimits || !p.withinDocumentedLimits).length;
|
|
209
|
+
const nativeDispatches = new Set(attempts.filter(a => a.batch?.mode === 'native').map(a => a.batch.groupId)).size;
|
|
210
|
+
const requestIds = [...new Set(attempts.map(a => a.requestId).filter((id) => typeof id === 'string'))].sort();
|
|
211
|
+
const rejected = expected === 0;
|
|
212
|
+
// The provider is the only source of per-call output; null means unknown, so it fails closed.
|
|
213
|
+
// A native batch keeps one shared usage copy per request on the ruleset result while its answer
|
|
214
|
+
// attempts carry null usage by design, so requests are read there and singles from success attempts.
|
|
215
|
+
const outputReports = new Map();
|
|
216
|
+
for (const entry of result.spec.batchRequests ?? []) {
|
|
217
|
+
const key = typeof entry.requestId === 'string' && entry.requestId ? `id:${entry.requestId}` : `batch:${entry.groupId}:${entry.ordinal}`;
|
|
218
|
+
if (!outputReports.has(key))
|
|
219
|
+
outputReports.set(key, entry.usage?.outputTokens ?? null);
|
|
220
|
+
}
|
|
221
|
+
attempts.forEach((attempt, index) => {
|
|
222
|
+
if (attempt.batch?.mode === 'native' || attempt.status !== 'success')
|
|
223
|
+
return;
|
|
224
|
+
const key = typeof attempt.requestId === 'string' && attempt.requestId ? `id:${attempt.requestId}` : `single:${index}`;
|
|
225
|
+
if (!outputReports.has(key))
|
|
226
|
+
outputReports.set(key, attempt.usage?.outputTokens ?? null);
|
|
227
|
+
});
|
|
228
|
+
const outputOversizedDispatches = rejected ? 0
|
|
229
|
+
: [...outputReports.values()].filter(n => !Number.isSafeInteger(n) || n < 0 || n > outputBound).length;
|
|
230
|
+
const covered = new Set(usage.flatMap(u => u.questionIds));
|
|
231
|
+
const pass = rejected
|
|
232
|
+
? expectedRejection !== null && usage.length === 0 && attempts.length === 0 && result.spec.contextFailure?.reason === expectedRejection
|
|
233
|
+
: oversizedDispatches === 0 && outputOversizedDispatches === 0 && usage.length === expected && covered.size === aliases.length
|
|
234
|
+
&& aliases.every(alias => covered.has(decisionBatchQuestionId(alias))) && evaluations.length === aliases.length
|
|
235
|
+
&& evaluations.every(e => e.spec.status === 'success') && attempts.every(a => a.status === 'success' && a.actualModel === model)
|
|
236
|
+
&& (phase === 'rollback' ? nativeDispatches === 0 && usage.every(u => u.questionIds.length === 1) : true);
|
|
237
|
+
return { caseId, phase, expectedDispatches: expected, dispatches: usage.length, rejectedBeforeDispatch: rejected && usage.length === 0,
|
|
238
|
+
planDigest: plan?.planDigest ?? null, profileDigest: plan?.providerProfile.digest ?? null, requestIds, partitions, oversizedDispatches, nativeDispatches, pass, outputOversizedDispatches };
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Enforce-mode canary. Each case runs once with the recorded qualification (native batching split by the
|
|
242
|
+
* qualified plan), then once rolled back to `observe-only`, which evaluates every question singly. All
|
|
243
|
+
* dispatches of a phase are reserved against the approved bound before it starts; the first failure stops.
|
|
244
|
+
*/
|
|
245
|
+
export async function runContextEnforceCanary(options) {
|
|
246
|
+
const { sourceRoot, artifactRoot, host, offlineTransport } = options;
|
|
247
|
+
const approval = structuredClone(options.approval), corpus = structuredClone(options.corpus), record = structuredClone(options.record);
|
|
248
|
+
validateContextCanaryApproval(approval, corpus, record);
|
|
249
|
+
await assertContextLiveSource(sourceRoot, approval.sourceCommit);
|
|
250
|
+
await assertContextArtifactRoot(sourceRoot, artifactRoot);
|
|
251
|
+
try {
|
|
252
|
+
return await withContextLiveSpendLedger(artifactRoot, approval.plan.budget.usd, (runsRoot, issueSpend) => canaryWithinLedger(approval, corpus, record, sourceRoot, runsRoot, issueSpend, host, offlineTransport));
|
|
253
|
+
}
|
|
254
|
+
finally {
|
|
255
|
+
host.dispose?.();
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
async function canaryWithinLedger(approval, corpus, record, sourceRoot, runsRoot, issueSpend, host, offlineTransport) {
|
|
259
|
+
const directory = resolve(runsRoot, approval.runId);
|
|
260
|
+
await mkdir(directory, { recursive: false, mode: 0o700 });
|
|
261
|
+
await writeFile(join(directory, 'canary-approval.json'), canonicalJson(approval), { flag: 'wx', mode: 0o600 });
|
|
262
|
+
await writeFile(join(directory, 'qualification-record.json'), canonicalJson(record), { flag: 'wx', mode: 0o600 });
|
|
263
|
+
const { plan } = approval;
|
|
264
|
+
const started = Date.now();
|
|
265
|
+
const budget = new ContextLiveBudget({ ...plan, budget: { ...plan.budget, usd: issueSpend.runUsdCeiling } }, started);
|
|
266
|
+
const controller = new AbortController();
|
|
267
|
+
const deadlineTimer = setTimeout(() => controller.abort(), Math.min(2_147_483_647, Math.floor(plan.budget.wallClockMs * 0.8)));
|
|
268
|
+
const results = [];
|
|
269
|
+
let stopped = null;
|
|
270
|
+
try {
|
|
271
|
+
for (const { caseId, enforce, rollback, enforceRejection, rollbackRejection } of await contextCanaryDispatches(plan.caseIds, corpus, record)) {
|
|
272
|
+
const item = corpus.cases.find(c => c.id === caseId);
|
|
273
|
+
for (const [phase, expected, expectedRejection] of [['enforce', enforce, enforceRejection], ['rollback', rollback, rollbackRejection]]) {
|
|
274
|
+
try {
|
|
275
|
+
for (let i = 0; i < expected; i++)
|
|
276
|
+
budget.reserve();
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
stopped = 'budget-exhausted';
|
|
280
|
+
break;
|
|
281
|
+
}
|
|
282
|
+
const { input, policy } = await compileContextLiveCase(item, approval.model, approval.region);
|
|
283
|
+
const aliases = item.definitions.map((_, i) => `q${i}`);
|
|
284
|
+
const definitions = Object.fromEntries(aliases.map((alias, i) => [alias, item.definitions[i]]));
|
|
285
|
+
const adapter = new JevDecisionAdapter({ region: approval.region, ...(offlineTransport ? { fetch: offlineTransport } : {}) });
|
|
286
|
+
const remaining = Math.floor(plan.budget.wallClockMs * 0.8 - (Date.now() - started));
|
|
287
|
+
if (remaining < 1 || controller.signal.aborted) {
|
|
288
|
+
stopped = 'deadline';
|
|
289
|
+
break;
|
|
290
|
+
}
|
|
291
|
+
const target = contextLiveTarget(approval, adapter.version, Math.min(30_000, remaining));
|
|
292
|
+
const ruleset = contextLiveRuleset(`canary-${item.id}`, aliases, item.definitions);
|
|
293
|
+
const binding = contextLiveBinding(ruleset, aliases, target, Math.min(remaining, 30_000 * aliases.length));
|
|
294
|
+
let result;
|
|
295
|
+
try {
|
|
296
|
+
result = await evaluateDecisionRuleset({ ruleset, binding, definitions, input: item.input, runId: approval.runId,
|
|
297
|
+
invocationId: `${approval.runId}:${item.id}:${phase}`, adapters: { jev: adapter }, signal: controller.signal,
|
|
298
|
+
resolveCredential: async (reference) => {
|
|
299
|
+
if (reference !== approval.secretServiceReference)
|
|
300
|
+
throw new Error('TV-12 credential access denied');
|
|
301
|
+
return host.resolveCredential(reference);
|
|
302
|
+
},
|
|
303
|
+
projection: { resolve: () => structuredClone(policy) },
|
|
304
|
+
batching: { enabled: true, evaluations: Object.fromEntries(aliases.map(alias => [alias, { decisionSubject: subject, independent: true,
|
|
305
|
+
egressPolicy: contextLiveDigest(policy), hostPolicy: 'tv12-canary-v1' }])) },
|
|
306
|
+
context: { input: { ...input, questions: input.questions.map(q => ({ ...q, id: decisionBatchQuestionId(q.id) })) }, profile: record.profile, estimator,
|
|
307
|
+
rollout: phase === 'enforce' ? { mode: 'enforce', qualification: record.qualification } : { mode: 'observe-only' } } });
|
|
308
|
+
}
|
|
309
|
+
catch {
|
|
310
|
+
stopped = 'evaluation-error';
|
|
311
|
+
break;
|
|
312
|
+
}
|
|
313
|
+
const row = inspect(item.id, phase, expected, expectedRejection, result, aliases, approval.model, plan.perRequestBound.totalTokens, plan.perRequestBound.outputTokens);
|
|
314
|
+
results.push(row);
|
|
315
|
+
await writeFile(join(directory, `${item.id}-${phase}.json`), canonicalJson(row), { flag: 'wx', mode: 0o600 });
|
|
316
|
+
if (!row.pass) {
|
|
317
|
+
stopped = 'canary-check-failed';
|
|
318
|
+
break;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
if (stopped)
|
|
322
|
+
break;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
finally {
|
|
326
|
+
clearTimeout(deadlineTimer);
|
|
327
|
+
}
|
|
328
|
+
const complete = !stopped && results.length === plan.caseIds.length * 2;
|
|
329
|
+
const summary = { schemaVersion: 'context-enforce-canary/v1', source: offlineTransport ? 'synthetic' : 'provider', runId: approval.runId,
|
|
330
|
+
sourceCommit: approval.sourceCommit, qualificationRecordDigest: approval.qualificationRecordDigest, canaryPlanDigest: approval.canaryPlanDigest,
|
|
331
|
+
profile: { id: record.profile.id, version: record.profile.version, digest: record.profileDigest },
|
|
332
|
+
canaryPassed: complete && results.every(r => r.pass), stopped,
|
|
333
|
+
oversizedDispatches: results.reduce((n, r) => n + r.oversizedDispatches, 0),
|
|
334
|
+
outputOversizedDispatches: results.reduce((n, r) => n + r.outputOversizedDispatches, 0),
|
|
335
|
+
enforceDispatches: results.filter(r => r.phase === 'enforce').reduce((n, r) => n + r.dispatches, 0),
|
|
336
|
+
rollbackExercised: complete && results.filter(r => r.phase === 'rollback').every(r => r.pass),
|
|
337
|
+
rollbackNativeDispatches: results.filter(r => r.phase === 'rollback').reduce((n, r) => n + r.nativeDispatches, 0),
|
|
338
|
+
cases: results.length, reserved: { requests: budget.requests, tokens: budget.tokens, usd: budget.usd }, issueSpend,
|
|
339
|
+
requestCapacityAtStop: contextLiveRequestCapacity({ ...plan, budget: { ...plan.budget, usd: issueSpend.runUsdCeiling } }), elapsedMs: Date.now() - started };
|
|
340
|
+
await writeFile(join(directory, 'summary.json'), canonicalJson(summary), { flag: 'wx', mode: 0o600 });
|
|
341
|
+
await assertContextLiveSource(sourceRoot, approval.sourceCommit);
|
|
342
|
+
return summary;
|
|
343
|
+
}
|
|
344
|
+
//# sourceMappingURL=context-live-promotion.js.map
|