@kontextmind/kxm 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +19 -0
- package/.kxm/README.md +14 -0
- package/.kxm/assets/README.md +5 -0
- package/.kxm/assets/retrospectives/README.md +5 -0
- package/.kxm/config/README.md +5 -0
- package/.kxm/config/agents.json +43 -0
- package/.kxm/config/env.example +56 -0
- package/.kxm/config/update.example.yaml +9 -0
- package/.kxm/config/workflows/fix.json +160 -0
- package/.kxm/config/workflows/jira-development.json +116 -0
- package/.kxm/config/workflows/provenance-quorum.json +150 -0
- package/.kxm/config/workflows/v04-dogfood.json +72 -0
- package/CHANGELOG.md +465 -0
- package/LICENSE +21 -0
- package/README.md +306 -0
- package/SECURITY.md +72 -0
- package/docs/README.md +48 -0
- package/docs/agent-communication-envelopes-and-gates.md +553 -0
- package/docs/architecture.md +242 -0
- package/docs/assignment-runner.md +241 -0
- package/docs/configuration.md +361 -0
- package/docs/continuous-improvement.md +114 -0
- package/docs/getting-started.md +253 -0
- package/docs/kxm-handbook.md +1090 -0
- package/docs/operations.md +205 -0
- package/docs/provenance-gates.md +291 -0
- package/docs/skills.md +45 -0
- package/docs/templates/README.md +95 -0
- package/docs/templates/adr.md +88 -0
- package/docs/templates/architecture.md +120 -0
- package/docs/templates/bug-fix.md +109 -0
- package/docs/templates/feature.md +108 -0
- package/docs/templates/handoff.md +72 -0
- package/docs/templates/postmortem.md +77 -0
- package/docs/templates/research.md +100 -0
- package/docs/templates/review.md +85 -0
- package/docs/templates/runbook.md +73 -0
- package/docs/templates/test-plan.md +87 -0
- package/docs/templates/test-report.md +72 -0
- package/docs/test-matrix.md +121 -0
- package/docs/troubleshooting.md +249 -0
- package/docs/vnext/README.md +62 -0
- package/docs/vnext/architecture.md +185 -0
- package/docs/vnext/effects-and-recovery.md +172 -0
- package/docs/vnext/lifecycles.md +235 -0
- package/docs/vnext/migration.md +220 -0
- package/docs/vnext/routing.md +184 -0
- package/docs/vnext/synchronization.md +172 -0
- package/docs/vnext/terminology.md +240 -0
- package/docs/vnext/validation.md +335 -0
- package/docs/webhook-workflows.md +240 -0
- package/docs/workflow-guide.md +1150 -0
- package/examples/README.md +102 -0
- package/examples/provenance-workflow.json +40 -0
- package/examples/requester.ts +30 -0
- package/examples/reviewer-agent.ts +29 -0
- package/examples/roundtrip.ts +46 -0
- package/examples/vnext/.kxm/agents/coordinator.yaml +16 -0
- package/examples/vnext/.kxm/agents/critic-1.yaml +16 -0
- package/examples/vnext/.kxm/agents/critic-2.yaml +15 -0
- package/examples/vnext/.kxm/agents/critic-3.yaml +15 -0
- package/examples/vnext/.kxm/agents/implementer.yaml +15 -0
- package/examples/vnext/.kxm/agents/planner.yaml +13 -0
- package/examples/vnext/.kxm/agents/reproducer.yaml +15 -0
- package/examples/vnext/.kxm/agents/reviewer.yaml +15 -0
- package/examples/vnext/.kxm/gates.yaml +8 -0
- package/examples/vnext/.kxm/models/critic-claude.yaml +11 -0
- package/examples/vnext/.kxm/models/critic-gemini.yaml +11 -0
- package/examples/vnext/.kxm/models/critic-grok.yaml +12 -0
- package/examples/vnext/.kxm/models/implementation.yaml +14 -0
- package/examples/vnext/.kxm/models/primary.yaml +17 -0
- package/examples/vnext/.kxm/prices.yaml +111 -0
- package/examples/vnext/.kxm/project/env.yaml +7 -0
- package/examples/vnext/.kxm/project.yaml +32 -0
- package/examples/vnext/.kxm/repo/repo.yaml +8 -0
- package/examples/vnext/.kxm/workflows/default.yaml +92 -0
- package/examples/vnext/.kxm/workflows/fix.yaml +376 -0
- package/examples/vnext/.kxm/workflows/improve.yaml +57 -0
- package/examples/vnext/README.md +53 -0
- package/examples/vnext/records/assignment-result-recorded.json +63 -0
- package/examples/vnext/records/assignment-result.json +46 -0
- package/examples/vnext/records/context-candidate.json +42 -0
- package/examples/vnext/records/delivery-manifest.json +66 -0
- package/examples/vnext/records/effect-uncertainty-resolved-sync.json +67 -0
- package/examples/vnext/records/effect-uncertainty-resolved.json +62 -0
- package/examples/vnext/records/run-created.json +54 -0
- package/examples/vnext/records/sync-event.json +65 -0
- package/examples/vnext/repositories/api/.kxm/repo/env.yaml +7 -0
- package/examples/vnext/repositories/api/.kxm/repo/repo.yaml +8 -0
- package/examples/vnext/repositories/web/.kxm/repo/repo.yaml +8 -0
- package/examples/workflow-signal.ts +63 -0
- package/package.json +129 -0
- package/plugins/kxm/.claude-plugin/plugin.json +73 -0
- package/plugins/kxm/.mcp.json +19 -0
- package/plugins/kxm/README.md +93 -0
- package/plugins/kxm/dist/cli.js +42853 -0
- package/plugins/kxm/dist/client.js +416 -0
- package/plugins/kxm/dist/core.js +1823 -0
- package/plugins/kxm/dist/extension.js +3797 -0
- package/plugins/kxm/dist/mcp-server.js +17104 -0
- package/plugins/kxm/dist/runtime.js +23361 -0
- package/plugins/kxm/dist/server.js +13640 -0
- package/plugins/kxm/dist/vnext-runtime-supervisor.js +21109 -0
- package/plugins/kxm/package.json +12 -0
- package/plugins/kxm/skills/kxm/SKILL.md +97 -0
- package/plugins/kxm/skills/kxm/references/protocol.md +103 -0
- package/plugins/kxm/skills/kxm-session/SKILL.md +53 -0
- package/plugins/kxm/src/arbiter.ts +355 -0
- package/plugins/kxm/src/artifacts-exist.ts +62 -0
- package/plugins/kxm/src/autocomplete.ts +236 -0
- package/plugins/kxm/src/cli.ts +3707 -0
- package/plugins/kxm/src/client.ts +614 -0
- package/plugins/kxm/src/commands.ts +1063 -0
- package/plugins/kxm/src/config.ts +290 -0
- package/plugins/kxm/src/context/providers.ts +101 -0
- package/plugins/kxm/src/context-packet.ts +332 -0
- package/plugins/kxm/src/context.ts +499 -0
- package/plugins/kxm/src/core.ts +6 -0
- package/plugins/kxm/src/database.ts +563 -0
- package/plugins/kxm/src/diagnostics.ts +184 -0
- package/plugins/kxm/src/envelope.ts +118 -0
- package/plugins/kxm/src/extension.ts +895 -0
- package/plugins/kxm/src/external-effects.ts +299 -0
- package/plugins/kxm/src/github-watch.ts +255 -0
- package/plugins/kxm/src/hub-binding.ts +160 -0
- package/plugins/kxm/src/hub.ts +2502 -0
- package/plugins/kxm/src/improve.ts +383 -0
- package/plugins/kxm/src/inbox.ts +10 -0
- package/plugins/kxm/src/kxm-install-kind.ts +113 -0
- package/plugins/kxm/src/kxm-update-config.ts +39 -0
- package/plugins/kxm/src/kxm-update.ts +238 -0
- package/plugins/kxm/src/local-snapshot.ts +406 -0
- package/plugins/kxm/src/logger.ts +198 -0
- package/plugins/kxm/src/mcp-server.ts +143 -0
- package/plugins/kxm/src/memory.ts +385 -0
- package/plugins/kxm/src/nous-pi.ts +287 -0
- package/plugins/kxm/src/nous-provider.ts +729 -0
- package/plugins/kxm/src/price-calc.ts +87 -0
- package/plugins/kxm/src/prices.ts +121 -0
- package/plugins/kxm/src/protocol.ts +172 -0
- package/plugins/kxm/src/recovery.ts +211 -0
- package/plugins/kxm/src/redact.ts +26 -0
- package/plugins/kxm/src/retrospective.ts +400 -0
- package/plugins/kxm/src/routing.ts +830 -0
- package/plugins/kxm/src/runtime.ts +9 -0
- package/plugins/kxm/src/server.ts +117 -0
- package/plugins/kxm/src/session-work.ts +571 -0
- package/plugins/kxm/src/session.ts +184 -0
- package/plugins/kxm/src/skills.ts +535 -0
- package/plugins/kxm/src/state.ts +326 -0
- package/plugins/kxm/src/store.ts +637 -0
- package/plugins/kxm/src/studio-layout.ts +268 -0
- package/plugins/kxm/src/suggest.ts +162 -0
- package/plugins/kxm/src/task-manager.ts +244 -0
- package/plugins/kxm/src/telemetry.ts +116 -0
- package/plugins/kxm/src/tui.ts +1046 -0
- package/plugins/kxm/src/vnext-bindings.ts +403 -0
- package/plugins/kxm/src/vnext-config.ts +1646 -0
- package/plugins/kxm/src/vnext-engine-artifacts.ts +86 -0
- package/plugins/kxm/src/vnext-engine-command.ts +533 -0
- package/plugins/kxm/src/vnext-engine-compile.ts +722 -0
- package/plugins/kxm/src/vnext-engine-evidence.ts +273 -0
- package/plugins/kxm/src/vnext-engine-fold.ts +1400 -0
- package/plugins/kxm/src/vnext-engine-gate-records.ts +583 -0
- package/plugins/kxm/src/vnext-engine-plan.ts +717 -0
- package/plugins/kxm/src/vnext-engine.ts +2458 -0
- package/plugins/kxm/src/vnext-gate-hash.ts +10 -0
- package/plugins/kxm/src/vnext-harness.ts +1142 -0
- package/plugins/kxm/src/vnext-init.ts +430 -0
- package/plugins/kxm/src/vnext-migrate.ts +1848 -0
- package/plugins/kxm/src/vnext-oneshot-producer.ts +424 -0
- package/plugins/kxm/src/vnext-permission.ts +936 -0
- package/plugins/kxm/src/vnext-pi-producer.ts +628 -0
- package/plugins/kxm/src/vnext-repair.ts +1094 -0
- package/plugins/kxm/src/vnext-runtime-owner.ts +320 -0
- package/plugins/kxm/src/vnext-runtime-store.ts +1560 -0
- package/plugins/kxm/src/vnext-runtime-supervisor.ts +586 -0
- package/plugins/kxm/src/vnext-runtime.ts +663 -0
- package/plugins/kxm/src/vnext-template.ts +247 -0
- package/plugins/kxm/src/wiki.ts +313 -0
- package/plugins/kxm/src/workflow.ts +1548 -0
- package/schemas/vnext/README.md +46 -0
- package/schemas/vnext/agent.schema.json +40 -0
- package/schemas/vnext/assignment-result.schema.json +66 -0
- package/schemas/vnext/backup-manifest.schema.json +89 -0
- package/schemas/vnext/candidate.schema.json +109 -0
- package/schemas/vnext/common.schema.json +422 -0
- package/schemas/vnext/context-candidate.schema.json +76 -0
- package/schemas/vnext/context-packet.schema.json +192 -0
- package/schemas/vnext/delivery-manifest.schema.json +159 -0
- package/schemas/vnext/environment.schema.json +66 -0
- package/schemas/vnext/gate-registry.schema.json +109 -0
- package/schemas/vnext/handoff-manifest.schema.json +146 -0
- package/schemas/vnext/init-operation.schema.json +61 -0
- package/schemas/vnext/local-repository-bindings.schema.json +30 -0
- package/schemas/vnext/memory-record.schema.json +45 -0
- package/schemas/vnext/migration-decision.schema.json +26 -0
- package/schemas/vnext/migration-plan.schema.json +123 -0
- package/schemas/vnext/migration-receipt.schema.json +52 -0
- package/schemas/vnext/model.schema.json +42 -0
- package/schemas/vnext/permission-diff.schema.json +57 -0
- package/schemas/vnext/prices.schema.json +115 -0
- package/schemas/vnext/project.schema.json +85 -0
- package/schemas/vnext/repository.schema.json +24 -0
- package/schemas/vnext/run-event.schema.json +460 -0
- package/schemas/vnext/session-brief.schema.json +153 -0
- package/schemas/vnext/sync-event.schema.json +234 -0
- package/schemas/vnext/template-provenance.schema.json +38 -0
- package/schemas/vnext/workflow.schema.json +248 -0
- package/scripts/assignment-run.d.mts +354 -0
- package/scripts/assignment-run.mjs +4451 -0
- package/scripts/build-runtime.mjs +56 -0
- package/scripts/check-generated.mjs +77 -0
- package/scripts/check-versions.mjs +34 -0
- package/scripts/emit-codex-artifacts.d.mts +9 -0
- package/scripts/emit-codex-artifacts.mjs +91 -0
- package/scripts/harness-run.d.mts +83 -0
- package/scripts/harness-run.mjs +2095 -0
- package/scripts/kxm-hub.mjs +105 -0
- package/scripts/kxm-publish-npm.mjs +327 -0
- package/scripts/kxm-release-github.mjs +472 -0
- package/scripts/kxm-runtime-supervisor.mjs +7 -0
- package/scripts/kxm-worker.mjs +1127 -0
- package/scripts/kxm.mjs +27 -0
- package/scripts/roster-policy.d.mts +20 -0
- package/scripts/roster-policy.mjs +161 -0
- package/scripts/smoke-multi-pi.mjs +479 -0
|
@@ -0,0 +1,2458 @@
|
|
|
1
|
+
import { createHash, randomBytes, timingSafeEqual } from "node:crypto";
|
|
2
|
+
import {
|
|
3
|
+
buildFormalContextPacket,
|
|
4
|
+
formatContextPacketForPrompt,
|
|
5
|
+
type FormalContextPacketV2,
|
|
6
|
+
type HandoffManifestV1,
|
|
7
|
+
} from "./context-packet.ts";
|
|
8
|
+
import { compileVnextWorkflow, type VnextCompiledPlan, type VnextCompiledStep } from "./vnext-engine-compile.ts";
|
|
9
|
+
import {
|
|
10
|
+
isTerminalRunStatus,
|
|
11
|
+
vnextFoldPanelAttempt,
|
|
12
|
+
vnextFoldPanelAttemptIds,
|
|
13
|
+
vnextJoinAll,
|
|
14
|
+
type VnextRunState,
|
|
15
|
+
} from "./vnext-engine-fold.ts";
|
|
16
|
+
import {
|
|
17
|
+
VNEXT_RUN_PLAN_SCHEMA,
|
|
18
|
+
freezeVnextCompiledPlan,
|
|
19
|
+
hashVnextRunPlanEnvelope,
|
|
20
|
+
loadVnextRunPlanEnvelope,
|
|
21
|
+
parseGateDefinition,
|
|
22
|
+
rehydrateVnextCompiledPlanFromStore,
|
|
23
|
+
type VnextPinnedGates,
|
|
24
|
+
type VnextRunPlanEnvelope,
|
|
25
|
+
} from "./vnext-engine-plan.ts";
|
|
26
|
+
import { gateRegistryHash } from "./vnext-gate-hash.ts";
|
|
27
|
+
import {
|
|
28
|
+
admitVnextRun,
|
|
29
|
+
armVnextGateHold,
|
|
30
|
+
bindVnextSchedulerPolicy,
|
|
31
|
+
dropVnextGateStopHook,
|
|
32
|
+
enqueueVnextScheduledRun,
|
|
33
|
+
finishVnextOwnedGate,
|
|
34
|
+
markVnextGateHoldUnsettled,
|
|
35
|
+
registerVnextAttemptController,
|
|
36
|
+
releaseVnextRun,
|
|
37
|
+
resolveVnextGateHold,
|
|
38
|
+
unregisterVnextAttemptController,
|
|
39
|
+
vnextAdmittedToken,
|
|
40
|
+
vnextAttemptController,
|
|
41
|
+
vnextAttemptControllers,
|
|
42
|
+
vnextGateHold,
|
|
43
|
+
vnextSchedulerPolicy,
|
|
44
|
+
} from "./vnext-runtime-owner.ts";
|
|
45
|
+
import {
|
|
46
|
+
cancelVnextRun,
|
|
47
|
+
foldStoredVnextRun,
|
|
48
|
+
isVnextRuntimeContextClosed,
|
|
49
|
+
persistVnextRunState,
|
|
50
|
+
registerVnextRuntimeCloseHook,
|
|
51
|
+
vnextEventBase,
|
|
52
|
+
vnextIncrementMonotonicNs,
|
|
53
|
+
vnextMonotonicNs,
|
|
54
|
+
vnextPolicyRevisions,
|
|
55
|
+
vnextProjectAdmissionLimits,
|
|
56
|
+
type VnextMemoryRevisionOptions,
|
|
57
|
+
type VnextRuntimeContext,
|
|
58
|
+
} from "./vnext-runtime.ts";
|
|
59
|
+
import {
|
|
60
|
+
newVnextAssignmentId,
|
|
61
|
+
newVnextAttemptId,
|
|
62
|
+
newVnextEventId,
|
|
63
|
+
projectRuntimeKey,
|
|
64
|
+
runtimeError,
|
|
65
|
+
type VnextAttemptCapabilityRow,
|
|
66
|
+
type VnextGateAttemptRow,
|
|
67
|
+
type VnextGateEvidenceRow,
|
|
68
|
+
type VnextGateObservationRow,
|
|
69
|
+
type VnextRunEvent,
|
|
70
|
+
type VnextRunRecord,
|
|
71
|
+
type VnextRunStatus,
|
|
72
|
+
} from "./vnext-runtime-store.ts";
|
|
73
|
+
import { vnextCanonicalJson, type JsonValue, type VnextProjectBundle } from "./vnext-config.ts";
|
|
74
|
+
import { evaluateArtifactsGate } from "./vnext-engine-artifacts.ts";
|
|
75
|
+
import { createCommandObserver } from "./vnext-engine-command.ts";
|
|
76
|
+
import {
|
|
77
|
+
recordGateCancelObservedInTransaction,
|
|
78
|
+
recordGateIntentInTransaction,
|
|
79
|
+
recordGateNoStartInTransaction,
|
|
80
|
+
recordGateSettlementInTransaction,
|
|
81
|
+
recordGateSpawnedInTransaction,
|
|
82
|
+
recordGateUncertainInTransaction,
|
|
83
|
+
type VnextGateObservationInput,
|
|
84
|
+
} from "./vnext-engine-gate-records.ts";
|
|
85
|
+
import {
|
|
86
|
+
ROUTING_RECORD_V2_SCHEMA,
|
|
87
|
+
type RoutingRecordV2,
|
|
88
|
+
parseRoutingRecordV2,
|
|
89
|
+
behavioralConfigHash,
|
|
90
|
+
} from "./routing.ts";
|
|
91
|
+
|
|
92
|
+
const trustedProducers = new WeakSet<object>();
|
|
93
|
+
const CAPABILITY_PREFIX = "kxm-attempt-capability\0";
|
|
94
|
+
|
|
95
|
+
export interface VnextProducerRequest {
|
|
96
|
+
readonly runId: string;
|
|
97
|
+
readonly stepId: string;
|
|
98
|
+
readonly stepAttempt: number;
|
|
99
|
+
readonly assignmentId: string;
|
|
100
|
+
readonly attemptId: string;
|
|
101
|
+
readonly agentId: string;
|
|
102
|
+
readonly capability: string;
|
|
103
|
+
readonly allowedOutcomes: readonly string[];
|
|
104
|
+
readonly signal: AbortSignal;
|
|
105
|
+
readonly instanceNo?: number | undefined;
|
|
106
|
+
readonly scopeEpoch?: number | undefined;
|
|
107
|
+
readonly prompt?: string | undefined;
|
|
108
|
+
readonly model?: string | undefined;
|
|
109
|
+
readonly provider?: string | undefined;
|
|
110
|
+
readonly thinking?: string | undefined;
|
|
111
|
+
readonly agentRole?: string | undefined;
|
|
112
|
+
readonly harness?: string | undefined;
|
|
113
|
+
readonly contextPacket?: FormalContextPacketV2 | undefined;
|
|
114
|
+
readonly handoffManifest?: HandoffManifestV1 | undefined;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export interface VnextProducerResult {
|
|
118
|
+
readonly outcome: string;
|
|
119
|
+
readonly costBasis?: "metered" | "unmetered" | "unknown";
|
|
120
|
+
readonly costUsd?: number | null;
|
|
121
|
+
readonly tokensIn?: number | null;
|
|
122
|
+
readonly tokensOut?: number | null;
|
|
123
|
+
readonly cacheReadTokens?: number | null;
|
|
124
|
+
readonly cacheWriteTokens?: number | null;
|
|
125
|
+
readonly contextTokens?: number | null;
|
|
126
|
+
readonly latencyMs?: number;
|
|
127
|
+
readonly harness?: string;
|
|
128
|
+
readonly provider?: string;
|
|
129
|
+
readonly requestedModel?: string;
|
|
130
|
+
readonly effectiveModel?: string;
|
|
131
|
+
readonly thinking?: string;
|
|
132
|
+
readonly agentRole?: string;
|
|
133
|
+
readonly behavioralSha256?: string;
|
|
134
|
+
readonly priceRef?: string;
|
|
135
|
+
readonly providerMetadata?: Record<string, string | number | boolean>;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export interface VnextProducer {
|
|
139
|
+
readonly id: "driver-simulated" | "pi" | "oneshot" | string;
|
|
140
|
+
produce(request: VnextProducerRequest): Promise<VnextProducerResult>;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export interface VnextRunHandoff {
|
|
144
|
+
readonly reason: "limit_unsupported" | "step_unsupported" | "attempt_unreconciled" | "cancel_pending_foreign" | "gate_unsupported" | "gate_outcome_undeclared" | "gate_recovery_pending" | "attempt_unsettled" | "gate_uncertain_blocked";
|
|
145
|
+
readonly field?: string;
|
|
146
|
+
readonly stepId?: string;
|
|
147
|
+
readonly attemptId?: string;
|
|
148
|
+
readonly detail: string;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export interface VnextRunDriveResult {
|
|
152
|
+
readonly state: VnextRunState;
|
|
153
|
+
readonly handoff?: VnextRunHandoff;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export interface VnextAttemptCapabilityRecord extends VnextAttemptCapabilityRow {
|
|
157
|
+
readonly runPlanHash: string;
|
|
158
|
+
readonly toolPolicyRevision: string;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export function createVnextSimulatedProducer(
|
|
162
|
+
script: (request: VnextProducerRequest) => Promise<VnextProducerResult> | VnextProducerResult,
|
|
163
|
+
): VnextProducer {
|
|
164
|
+
const producer = Object.freeze({
|
|
165
|
+
id: "driver-simulated" as const,
|
|
166
|
+
async produce(request: VnextProducerRequest): Promise<VnextProducerResult> {
|
|
167
|
+
const start = Date.now();
|
|
168
|
+
try {
|
|
169
|
+
const raw = await script(request);
|
|
170
|
+
const latencyMs = typeof raw.latencyMs === "number" ? raw.latencyMs : Math.max(0, Date.now() - start);
|
|
171
|
+
return {
|
|
172
|
+
costBasis: "unmetered",
|
|
173
|
+
tokensIn: null,
|
|
174
|
+
tokensOut: null,
|
|
175
|
+
cacheReadTokens: null,
|
|
176
|
+
cacheWriteTokens: null,
|
|
177
|
+
...raw,
|
|
178
|
+
latencyMs,
|
|
179
|
+
};
|
|
180
|
+
} catch (error) {
|
|
181
|
+
return Promise.reject(error);
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
});
|
|
185
|
+
trustedProducers.add(producer);
|
|
186
|
+
return producer;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
export function registerTrustedProducer(producer: VnextProducer): void {
|
|
190
|
+
trustedProducers.add(producer);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function requireTrustedProducer(producer: VnextProducer): void {
|
|
194
|
+
if (!trustedProducers.has(producer)) {
|
|
195
|
+
throw runtimeError("engine_producer_untrusted", "producer", "producer is not a trusted factory instance");
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function hashCapabilitySecret(secret: string): string {
|
|
200
|
+
return `sha256:${createHash("sha256").update(`${CAPABILITY_PREFIX}${secret}`, "utf8").digest("hex")}`;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
export function mintVnextCapabilitySecret(): { secret: string; hash: string } {
|
|
204
|
+
const secret = `kxmcap_${randomBytes(32).toString("base64url")}`;
|
|
205
|
+
return { secret, hash: hashCapabilitySecret(secret) };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function mintCapabilitySecret(): { secret: string; hash: string } {
|
|
209
|
+
return mintVnextCapabilitySecret();
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
export function pinVnextCompiledPlan(
|
|
213
|
+
context: VnextRuntimeContext,
|
|
214
|
+
bundle: VnextProjectBundle,
|
|
215
|
+
runId: string,
|
|
216
|
+
options?: VnextMemoryRevisionOptions,
|
|
217
|
+
): { plan: VnextCompiledPlan; runPlanHash: string; idempotent: boolean; event?: VnextRunEvent } {
|
|
218
|
+
return context.eventStore.transaction(() => {
|
|
219
|
+
const run = requireRun(context, runId);
|
|
220
|
+
if (run.projectId !== context.projectId || run.homeRuntimeId !== context.homeRuntimeId) {
|
|
221
|
+
throw runtimeError("run_owner_mismatch", runId, "run is not owned by this runtime context");
|
|
222
|
+
}
|
|
223
|
+
if (String(bundle.project.value.id) !== run.projectId) {
|
|
224
|
+
throw runtimeError("run_owner_mismatch", runId, "bundle project does not match the run");
|
|
225
|
+
}
|
|
226
|
+
const workflow = bundle.workflows.get(run.workflowId);
|
|
227
|
+
if (!workflow) throw runtimeError("run_workflow_unknown", run.workflowId, `workflow ${run.workflowId} does not exist in this project`);
|
|
228
|
+
const revisions = vnextPolicyRevisions(bundle, options);
|
|
229
|
+
if (
|
|
230
|
+
revisions.configRevision !== run.configRevision
|
|
231
|
+
|| revisions.executorPolicyRevision !== run.executorPolicyRevision
|
|
232
|
+
|| revisions.toolPolicyRevision !== run.toolPolicyRevision
|
|
233
|
+
|| revisions.memoryRevision !== run.memoryRevision
|
|
234
|
+
) {
|
|
235
|
+
throw runtimeError("run_revision_drift", runId, "bundle revisions do not match the accepted run");
|
|
236
|
+
}
|
|
237
|
+
const compiled = freezeVnextCompiledPlan(compileVnextWorkflow({
|
|
238
|
+
id: run.workflowId,
|
|
239
|
+
value: workflow.value,
|
|
240
|
+
...(workflow.logicalPath !== undefined ? { logicalPath: workflow.logicalPath } : {}),
|
|
241
|
+
}));
|
|
242
|
+
const envelope: VnextRunPlanEnvelope = {
|
|
243
|
+
schema: VNEXT_RUN_PLAN_SCHEMA,
|
|
244
|
+
runId: run.runId,
|
|
245
|
+
projectId: run.projectId,
|
|
246
|
+
homeRuntimeId: run.homeRuntimeId,
|
|
247
|
+
workflowId: run.workflowId,
|
|
248
|
+
revisions: {
|
|
249
|
+
config: run.configRevision,
|
|
250
|
+
executorPolicy: run.executorPolicyRevision,
|
|
251
|
+
toolPolicy: run.toolPolicyRevision,
|
|
252
|
+
memory: run.memoryRevision,
|
|
253
|
+
},
|
|
254
|
+
projectLimits: vnextProjectAdmissionLimits(bundle),
|
|
255
|
+
plan: compiled,
|
|
256
|
+
gates: pinnedGatesForCompiledPlan(compiled, bundle, context.projectRoot),
|
|
257
|
+
};
|
|
258
|
+
const runPlanHash = hashVnextRunPlanEnvelope(envelope);
|
|
259
|
+
const existing = context.eventStore.runPlan(runId);
|
|
260
|
+
if (existing) {
|
|
261
|
+
if (existing.runPlanHash !== runPlanHash) {
|
|
262
|
+
throw runtimeError("run_plan_pinned", runId, "run already has a different pinned plan");
|
|
263
|
+
}
|
|
264
|
+
return { plan: compiled, runPlanHash, idempotent: true };
|
|
265
|
+
}
|
|
266
|
+
if (run.status !== "created") {
|
|
267
|
+
throw runtimeError("run_plan_pinned", runId, "a plan can only be pinned from created");
|
|
268
|
+
}
|
|
269
|
+
const now = new Date().toISOString();
|
|
270
|
+
const sequence = context.eventStore.nextSequence(runId);
|
|
271
|
+
const event: VnextRunEvent = {
|
|
272
|
+
...vnextEventBase(context, run, now, vnextMonotonicNs()),
|
|
273
|
+
eventId: newVnextEventId(),
|
|
274
|
+
eventType: "run.status_changed",
|
|
275
|
+
sequence,
|
|
276
|
+
payload: { status: "preparing", runPlanHash },
|
|
277
|
+
};
|
|
278
|
+
context.eventStore.insertRunPlan({
|
|
279
|
+
runId,
|
|
280
|
+
runPlanHash,
|
|
281
|
+
envelope: vnextCanonicalJson(envelope as unknown as JsonValue),
|
|
282
|
+
pinnedSequence: sequence,
|
|
283
|
+
});
|
|
284
|
+
context.eventStore.appendEvent(event);
|
|
285
|
+
const state = foldStoredVnextRun(context, run);
|
|
286
|
+
persistVnextRunState(context, runId, state, sequence);
|
|
287
|
+
context.eventStore.updateRunStatus(runId, "preparing", now);
|
|
288
|
+
return { plan: compiled, runPlanHash, idempotent: false, event };
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
export function rehydrateVnextCompiledPlan(context: VnextRuntimeContext, runId: string): VnextCompiledPlan {
|
|
293
|
+
const run = requireRun(context, runId);
|
|
294
|
+
return rehydrateVnextCompiledPlanFromStore(context.eventStore, run);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
export function startVnextRun(context: VnextRuntimeContext, runId: string, options: { allowLimits?: boolean } = {}): VnextRunDriveResult {
|
|
298
|
+
return context.eventStore.transaction(() => {
|
|
299
|
+
const run = requireRun(context, runId);
|
|
300
|
+
const envelope = loadVnextRunPlanEnvelope(context.eventStore, run);
|
|
301
|
+
const unsupported = options.allowLimits ? undefined : unsupportedLimit(envelope);
|
|
302
|
+
const state = foldStoredVnextRun(context, run);
|
|
303
|
+
if (unsupported) {
|
|
304
|
+
return { state, handoff: unsupported };
|
|
305
|
+
}
|
|
306
|
+
if (state.status !== "preparing") {
|
|
307
|
+
return { state };
|
|
308
|
+
}
|
|
309
|
+
const now = new Date().toISOString();
|
|
310
|
+
const sequence = context.eventStore.nextSequence(runId);
|
|
311
|
+
const event: VnextRunEvent = {
|
|
312
|
+
...vnextEventBase(context, run, now, vnextMonotonicNs()),
|
|
313
|
+
eventId: newVnextEventId(),
|
|
314
|
+
eventType: "run.status_changed",
|
|
315
|
+
sequence,
|
|
316
|
+
payload: { status: "running" },
|
|
317
|
+
};
|
|
318
|
+
context.eventStore.appendEvent(event);
|
|
319
|
+
const next = foldStoredVnextRun(context, run);
|
|
320
|
+
persistVnextRunState(context, runId, next, sequence);
|
|
321
|
+
context.eventStore.updateRunStatus(runId, "running", now);
|
|
322
|
+
return { state: next };
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
export async function stepVnextRun(context: VnextRuntimeContext, runId: string, producer: VnextProducer): Promise<VnextRunDriveResult> {
|
|
327
|
+
requireTrustedProducer(producer);
|
|
328
|
+
return withAdmission(context, runId, (token) => stepLocked(context, runId, producer, token));
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
export async function driveVnextRun(
|
|
332
|
+
context: VnextRuntimeContext,
|
|
333
|
+
runId: string,
|
|
334
|
+
producer: VnextProducer,
|
|
335
|
+
options: { allowLimits?: boolean } = {},
|
|
336
|
+
): Promise<VnextRunDriveResult> {
|
|
337
|
+
requireTrustedProducer(producer);
|
|
338
|
+
const run = requireRun(context, runId);
|
|
339
|
+
if (run.status === "created") {
|
|
340
|
+
throw runtimeError("run_plan_missing", runId, "drive requires a pinned plan");
|
|
341
|
+
}
|
|
342
|
+
return withAdmission(context, runId, (token) => driveAdmitted(context, runId, producer, token, options));
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
async function driveAdmitted(
|
|
346
|
+
context: VnextRuntimeContext,
|
|
347
|
+
runId: string,
|
|
348
|
+
producer: VnextProducer,
|
|
349
|
+
token: string,
|
|
350
|
+
options: { allowLimits?: boolean } = {},
|
|
351
|
+
): Promise<VnextRunDriveResult> {
|
|
352
|
+
const run = requireRun(context, runId);
|
|
353
|
+
if (run.status === "preparing") {
|
|
354
|
+
const started = startVnextRun(context, runId, options);
|
|
355
|
+
if (started.handoff || isTerminalRunStatus(started.state.status)) return started;
|
|
356
|
+
}
|
|
357
|
+
let latest: VnextRunDriveResult = { state: foldStoredVnextRun(context, requireRun(context, runId)) };
|
|
358
|
+
while (latest.state.status === "running") {
|
|
359
|
+
const before = context.eventStore.runState(runId)?.lastSequence;
|
|
360
|
+
latest = await stepLocked(context, runId, producer, token);
|
|
361
|
+
if (latest.handoff) return latest;
|
|
362
|
+
if (latest.state.status === "running") {
|
|
363
|
+
const after = context.eventStore.runState(runId)?.lastSequence;
|
|
364
|
+
if (after === before) {
|
|
365
|
+
throw runtimeError("run_events_illegal", runId, "drive made no progress");
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
return latest;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
async function withAdmission<T>(context: VnextRuntimeContext, runId: string, fn: (token: string) => Promise<T>): Promise<T> {
|
|
373
|
+
const token = admitPinnedRun(context, runId);
|
|
374
|
+
try {
|
|
375
|
+
return await fn(token);
|
|
376
|
+
} finally {
|
|
377
|
+
releaseVnextRun(context.eventStore.path, runId, token);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function admitPinnedRun(context: VnextRuntimeContext, runId: string): string {
|
|
382
|
+
const run = requireRun(context, runId);
|
|
383
|
+
const envelope = loadVnextRunPlanEnvelope(context.eventStore, run);
|
|
384
|
+
return admitVnextRun(
|
|
385
|
+
context.eventStore.path,
|
|
386
|
+
runId,
|
|
387
|
+
envelope.revisions.config,
|
|
388
|
+
envelope.projectLimits.maxConcurrentRuns,
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
export class VnextRunScheduler {
|
|
393
|
+
private readonly context: VnextRuntimeContext;
|
|
394
|
+
private readonly configRevision: string;
|
|
395
|
+
|
|
396
|
+
private constructor(context: VnextRuntimeContext, configRevision: string) {
|
|
397
|
+
this.context = context;
|
|
398
|
+
this.configRevision = configRevision;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
static for(context: VnextRuntimeContext, bundle: VnextProjectBundle): VnextRunScheduler {
|
|
402
|
+
if (String(bundle.project.value.id) !== context.projectId) {
|
|
403
|
+
throw runtimeError("run_owner_mismatch", context.projectId, "scheduler bundle project does not match the runtime context");
|
|
404
|
+
}
|
|
405
|
+
const limits = vnextProjectAdmissionLimits(bundle);
|
|
406
|
+
bindVnextSchedulerPolicy(context.eventStore.path, limits.maxConcurrentRuns, bundle.configRevision);
|
|
407
|
+
return new VnextRunScheduler(context, bundle.configRevision);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
enqueue(runId: string, producer: VnextProducer): Promise<VnextRunDriveResult> {
|
|
411
|
+
requireTrustedProducer(producer);
|
|
412
|
+
const policy = vnextSchedulerPolicy(this.context.eventStore.path);
|
|
413
|
+
if (!policy || policy.configRevision !== this.configRevision) {
|
|
414
|
+
return Promise.reject(runtimeError("scheduler_policy_conflict", runId, "scheduler handle does not match the active policy"));
|
|
415
|
+
}
|
|
416
|
+
try {
|
|
417
|
+
const run = requireRun(this.context, runId);
|
|
418
|
+
const envelope = loadVnextRunPlanEnvelope(this.context.eventStore, run);
|
|
419
|
+
return enqueueVnextScheduledRun(
|
|
420
|
+
this.context.eventStore.path,
|
|
421
|
+
runId,
|
|
422
|
+
envelope.revisions.config,
|
|
423
|
+
envelope.projectLimits.maxConcurrentRuns,
|
|
424
|
+
(token) => driveAdmitted(this.context, runId, producer, token),
|
|
425
|
+
) as Promise<VnextRunDriveResult>;
|
|
426
|
+
} catch (error) {
|
|
427
|
+
return Promise.reject(error);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
export function verifyVnextAttemptCapability(
|
|
433
|
+
context: VnextRuntimeContext,
|
|
434
|
+
presented: string,
|
|
435
|
+
expected: { runId: string; attemptId: string },
|
|
436
|
+
): VnextAttemptCapabilityRecord {
|
|
437
|
+
const presentedHash = hashCapabilitySecret(presented);
|
|
438
|
+
const row = context.eventStore.capabilityByHash(presentedHash);
|
|
439
|
+
const run = context.eventStore.run(expected.runId);
|
|
440
|
+
const folded = run ? foldStoredVnextRun(context, run) : undefined;
|
|
441
|
+
const hashBuf = Buffer.from(presentedHash);
|
|
442
|
+
const storedBuf = Buffer.from(row?.capabilityHash ?? presentedHash);
|
|
443
|
+
const hashMatch = hashBuf.length === storedBuf.length && timingSafeEqual(hashBuf, storedBuf);
|
|
444
|
+
const located = vnextFoldPanelAttempt(folded?.currentStep, expected.attemptId);
|
|
445
|
+
const ok = Boolean(
|
|
446
|
+
hashMatch
|
|
447
|
+
&& row
|
|
448
|
+
&& row.state === "issued"
|
|
449
|
+
&& row.runId === expected.runId
|
|
450
|
+
&& row.attemptId === expected.attemptId
|
|
451
|
+
&& (row.producerId === "driver-simulated" || row.producerId === "pi" || row.producerId === "oneshot")
|
|
452
|
+
&& folded
|
|
453
|
+
&& folded.status === "running"
|
|
454
|
+
&& folded.currentStep
|
|
455
|
+
&& folded.currentStep.stepId === row.stepId
|
|
456
|
+
&& folded.currentStep.stepAttempt === row.stepAttempt
|
|
457
|
+
&& located
|
|
458
|
+
&& located.assignmentId === row.assignmentId
|
|
459
|
+
&& located.attempt.status === "executing",
|
|
460
|
+
);
|
|
461
|
+
if (!ok) {
|
|
462
|
+
throw runtimeError("capability_rejected", expected.attemptId, "attempt capability was rejected");
|
|
463
|
+
}
|
|
464
|
+
return {
|
|
465
|
+
...row!,
|
|
466
|
+
runPlanHash: folded!.runPlanHash ?? "",
|
|
467
|
+
toolPolicyRevision: run!.toolPolicyRevision,
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
function invokeProducer(
|
|
472
|
+
producer: VnextProducer,
|
|
473
|
+
request: VnextProducerRequest,
|
|
474
|
+
): Promise<{ result?: VnextProducerResult; error: boolean }> {
|
|
475
|
+
let pending: Promise<VnextProducerResult>;
|
|
476
|
+
try {
|
|
477
|
+
pending = Promise.resolve(producer.produce(request));
|
|
478
|
+
} catch {
|
|
479
|
+
return Promise.resolve({ error: true });
|
|
480
|
+
}
|
|
481
|
+
return pending.then(
|
|
482
|
+
(result) => ({ result, error: false }),
|
|
483
|
+
() => ({ error: true }),
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
export interface VnextPreparedGateDispatch {
|
|
488
|
+
readonly run: VnextRunRecord;
|
|
489
|
+
readonly envelope: VnextRunPlanEnvelope;
|
|
490
|
+
readonly stepId: string;
|
|
491
|
+
readonly stepAttempt: number;
|
|
492
|
+
readonly assignmentId: string;
|
|
493
|
+
readonly attemptId: string;
|
|
494
|
+
readonly effectId: string;
|
|
495
|
+
readonly controller: AbortController;
|
|
496
|
+
readonly state: VnextRunState;
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
export interface VnextGateDispatchSeams {
|
|
500
|
+
afterEvaluate?: ((prepared: VnextPreparedGateDispatch) => void) | undefined;
|
|
501
|
+
afterObservationWrite?: ((prepared: VnextPreparedGateDispatch) => void) | undefined;
|
|
502
|
+
afterUncertaintyWrite?: ((prepared: VnextPreparedGateDispatch) => void) | undefined;
|
|
503
|
+
afterSettlementCommit?: ((prepared: VnextPreparedGateDispatch) => void) | undefined;
|
|
504
|
+
afterSpawnRecord?: ((prepared: VnextPreparedGateDispatch) => void) | undefined;
|
|
505
|
+
beforeCommandSpawn?: ((prepared: VnextPreparedGateDispatch) => void) | undefined;
|
|
506
|
+
beforeCommandSettle?: ((prepared: VnextPreparedGateDispatch) => void) | undefined;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
export const vnextGateDispatchSeams: VnextGateDispatchSeams = {};
|
|
510
|
+
|
|
511
|
+
async function stepLocked(context: VnextRuntimeContext, runId: string, producer: VnextProducer, token: string): Promise<VnextRunDriveResult> {
|
|
512
|
+
const prepared = context.eventStore.transaction(() => prepareDispatch(context, runId, producer.id));
|
|
513
|
+
if (prepared.kind === "return") {
|
|
514
|
+
return prepared.handoff ? { state: prepared.state, handoff: prepared.handoff } : { state: prepared.state };
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
if (prepared.kind === "gate") {
|
|
518
|
+
const step = prepared.envelope.plan.steps[prepared.stepId];
|
|
519
|
+
if (!step || step.kind !== "gate") {
|
|
520
|
+
throw runtimeError("run_events_illegal", prepared.run.runId, "prepared gate step is not a gate");
|
|
521
|
+
}
|
|
522
|
+
const definition = prepared.envelope.gates.definitions[step.gate];
|
|
523
|
+
if (definition?.kind === "command") {
|
|
524
|
+
return runPreparedCommandGate(context, prepared, definition, token);
|
|
525
|
+
}
|
|
526
|
+
if (!definition || definition.kind !== "artifacts-exist") {
|
|
527
|
+
throw runtimeError("run_events_illegal", prepared.run.runId, "prepared gate is not artifacts-exist");
|
|
528
|
+
}
|
|
529
|
+
registerVnextAttemptController(context.eventStore.path, runId, { attemptId: prepared.attemptId, controller: prepared.controller });
|
|
530
|
+
try {
|
|
531
|
+
const observation = evaluateArtifactsGate(context, definition);
|
|
532
|
+
vnextGateDispatchSeams.afterEvaluate?.(prepared);
|
|
533
|
+
return context.eventStore.transaction(() => settlePreparedGate(context, prepared, observation));
|
|
534
|
+
} finally {
|
|
535
|
+
unregisterVnextAttemptController(context.eventStore.path, runId, prepared.attemptId);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
return drivePanel(context, prepared.panel, producer);
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
async function runPreparedCommandGate(
|
|
543
|
+
context: VnextRuntimeContext,
|
|
544
|
+
prepared: VnextPreparedGateDispatch,
|
|
545
|
+
definition: { readonly kind: "command"; readonly argv: readonly string[]; readonly timeoutMs: number; readonly cwd?: "control" },
|
|
546
|
+
token: string,
|
|
547
|
+
): Promise<VnextRunDriveResult> {
|
|
548
|
+
const storePath = context.eventStore.path;
|
|
549
|
+
const observer = createCommandObserver({
|
|
550
|
+
definition,
|
|
551
|
+
cwd: context.projectRoot,
|
|
552
|
+
signal: prepared.controller.signal,
|
|
553
|
+
onSpawned: () => {
|
|
554
|
+
if (isVnextRuntimeContextClosed(context)) return;
|
|
555
|
+
try {
|
|
556
|
+
context.eventStore.transaction(() => {
|
|
557
|
+
recordGateSpawnedInTransaction(context, prepared.run, prepared.envelope, prepared.stepId);
|
|
558
|
+
vnextGateDispatchSeams.afterSpawnRecord?.(prepared);
|
|
559
|
+
});
|
|
560
|
+
} catch (error) {
|
|
561
|
+
// Mark the hold unsettled synchronously BEFORE the observer stop path
|
|
562
|
+
// can take any fallible action (group signalling) on this attempt.
|
|
563
|
+
try {
|
|
564
|
+
markVnextGateHoldUnsettled(storePath, prepared.run.runId, token, prepared.attemptId);
|
|
565
|
+
} catch {
|
|
566
|
+
// The hold is already unsettled or released; the original failure governs.
|
|
567
|
+
}
|
|
568
|
+
throw error;
|
|
569
|
+
}
|
|
570
|
+
},
|
|
571
|
+
});
|
|
572
|
+
armVnextGateHold(storePath, prepared.run.runId, token, prepared.attemptId, () => observer.requestStop("cancel"));
|
|
573
|
+
registerVnextAttemptController(storePath, prepared.run.runId, { attemptId: prepared.attemptId, controller: prepared.controller });
|
|
574
|
+
const unhookClose = registerVnextRuntimeCloseHook(context, () => {
|
|
575
|
+
try {
|
|
576
|
+
markVnextGateHoldUnsettled(storePath, prepared.run.runId, token, prepared.attemptId);
|
|
577
|
+
} catch {
|
|
578
|
+
// Hold may already be unsettled.
|
|
579
|
+
}
|
|
580
|
+
dropVnextGateStopHook(storePath, prepared.run.runId, prepared.attemptId);
|
|
581
|
+
observer.beginBoundedCleanup();
|
|
582
|
+
});
|
|
583
|
+
try {
|
|
584
|
+
vnextGateDispatchSeams.beforeCommandSpawn?.(prepared);
|
|
585
|
+
const outcome = await observer.run();
|
|
586
|
+
dropVnextGateStopHook(storePath, prepared.run.runId, prepared.attemptId);
|
|
587
|
+
if (isVnextRuntimeContextClosed(context)) {
|
|
588
|
+
try {
|
|
589
|
+
markVnextGateHoldUnsettled(storePath, prepared.run.runId, token, prepared.attemptId);
|
|
590
|
+
} catch {
|
|
591
|
+
// already unsettled
|
|
592
|
+
}
|
|
593
|
+
return closedCommandHandoff(context, prepared, outcome.kind === "uncertain" ? outcome.reason : "lost-close");
|
|
594
|
+
}
|
|
595
|
+
if (outcome.kind === "no-start") {
|
|
596
|
+
return commitOwnedCommandSettlement(context, prepared, token, () => (
|
|
597
|
+
recordGateNoStartInTransaction(context, prepared.run, prepared.envelope, prepared.stepId, outcome.observation)
|
|
598
|
+
), outcome.observation);
|
|
599
|
+
}
|
|
600
|
+
if (outcome.kind === "complete") {
|
|
601
|
+
try {
|
|
602
|
+
vnextGateDispatchSeams.beforeCommandSettle?.(prepared);
|
|
603
|
+
} catch (error) {
|
|
604
|
+
// Retain the unsettled hold and attempt a truthful recording-error
|
|
605
|
+
// uncertainty only if no observation was committed.
|
|
606
|
+
return recoverCommandRecordingFailure(context, prepared, token, outcome.observation, error);
|
|
607
|
+
}
|
|
608
|
+
if (isVnextRuntimeContextClosed(context)) {
|
|
609
|
+
try {
|
|
610
|
+
markVnextGateHoldUnsettled(storePath, prepared.run.runId, token, prepared.attemptId);
|
|
611
|
+
} catch {
|
|
612
|
+
// already unsettled
|
|
613
|
+
}
|
|
614
|
+
return closedCommandHandoff(context, prepared, "lost-close");
|
|
615
|
+
}
|
|
616
|
+
return commitOwnedCommandSettlement(context, prepared, token, () => settlePreparedGate(context, prepared, outcome.observation), outcome.observation);
|
|
617
|
+
}
|
|
618
|
+
return commitCommandUncertainty(context, prepared, token, outcome.reason, outcome.observation);
|
|
619
|
+
} finally {
|
|
620
|
+
unhookClose();
|
|
621
|
+
unregisterVnextAttemptController(storePath, prepared.run.runId, prepared.attemptId);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
function closedCommandHandoff(
|
|
626
|
+
context: VnextRuntimeContext,
|
|
627
|
+
prepared: VnextPreparedGateDispatch,
|
|
628
|
+
reason: string,
|
|
629
|
+
): VnextRunDriveResult {
|
|
630
|
+
let state: VnextRunDriveResult["state"];
|
|
631
|
+
try {
|
|
632
|
+
state = foldStoredVnextRun(context, requireRun(context, prepared.run.runId));
|
|
633
|
+
} catch {
|
|
634
|
+
state = prepared.state;
|
|
635
|
+
}
|
|
636
|
+
return {
|
|
637
|
+
state,
|
|
638
|
+
handoff: {
|
|
639
|
+
reason: "attempt_unsettled",
|
|
640
|
+
stepId: prepared.stepId,
|
|
641
|
+
attemptId: prepared.attemptId,
|
|
642
|
+
detail: reason,
|
|
643
|
+
},
|
|
644
|
+
};
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
function commitOwnedCommandSettlement(
|
|
648
|
+
context: VnextRuntimeContext,
|
|
649
|
+
prepared: VnextPreparedGateDispatch,
|
|
650
|
+
token: string,
|
|
651
|
+
write: () => { state: VnextRunDriveResult["state"] },
|
|
652
|
+
observation: VnextGateObservationInput,
|
|
653
|
+
): VnextRunDriveResult {
|
|
654
|
+
let committed: { state: VnextRunDriveResult["state"] };
|
|
655
|
+
try {
|
|
656
|
+
committed = context.eventStore.transaction(() => write());
|
|
657
|
+
} catch (error) {
|
|
658
|
+
return recoverCommandRecordingFailure(context, prepared, token, observation, error);
|
|
659
|
+
}
|
|
660
|
+
try {
|
|
661
|
+
vnextGateDispatchSeams.afterSettlementCommit?.(prepared);
|
|
662
|
+
} catch (error) {
|
|
663
|
+
return retainCommittedCommandProof(context, prepared, token, error);
|
|
664
|
+
}
|
|
665
|
+
try {
|
|
666
|
+
finishVnextOwnedGate(context.eventStore.path, prepared.run.runId, token, prepared.attemptId);
|
|
667
|
+
} catch (error) {
|
|
668
|
+
return retainCommittedCommandProof(context, prepared, token, error);
|
|
669
|
+
}
|
|
670
|
+
return committed;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
function retainCommittedCommandProof(
|
|
674
|
+
context: VnextRuntimeContext,
|
|
675
|
+
prepared: VnextPreparedGateDispatch,
|
|
676
|
+
token: string,
|
|
677
|
+
error: unknown,
|
|
678
|
+
): never {
|
|
679
|
+
try {
|
|
680
|
+
finishOwnedGateAfterCommittedTerminalProof(context, prepared, token);
|
|
681
|
+
} catch {
|
|
682
|
+
// Invalid identity or hold state inside finish stays fail-closed; do not force or retry a release.
|
|
683
|
+
}
|
|
684
|
+
throw error;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
function finishOwnedGateAfterCommittedTerminalProof(
|
|
688
|
+
context: VnextRuntimeContext,
|
|
689
|
+
prepared: VnextPreparedGateDispatch,
|
|
690
|
+
token: string,
|
|
691
|
+
): void {
|
|
692
|
+
if (isVnextRuntimeContextClosed(context)) return;
|
|
693
|
+
const run = context.eventStore.run(prepared.run.runId);
|
|
694
|
+
if (!run || run.projectId !== context.projectId || run.homeRuntimeId !== context.homeRuntimeId) return;
|
|
695
|
+
const row = context.eventStore.gateAttempt(prepared.attemptId);
|
|
696
|
+
const preparedIdentity = {
|
|
697
|
+
runId: prepared.run.runId,
|
|
698
|
+
attemptId: prepared.attemptId,
|
|
699
|
+
assignmentId: prepared.assignmentId,
|
|
700
|
+
effectId: prepared.effectId,
|
|
701
|
+
stepId: prepared.stepId,
|
|
702
|
+
};
|
|
703
|
+
if (!row || !gateRowIdentityMatches(row, preparedIdentity) || row.projectId !== context.projectId || row.homeRuntimeId !== context.homeRuntimeId) {
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
const observation = context.eventStore.gateObservationForAttempt(prepared.attemptId);
|
|
707
|
+
const evidence = context.eventStore.gateEvidenceForAttempt(prepared.attemptId);
|
|
708
|
+
const events = context.eventStore.events(prepared.run.runId, 0, 1_000_000);
|
|
709
|
+
let state;
|
|
710
|
+
try {
|
|
711
|
+
state = foldStoredVnextRun(context, run);
|
|
712
|
+
} catch {
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
const capability = context.eventStore.capabilityByAttempt(prepared.attemptId);
|
|
716
|
+
if (
|
|
717
|
+
!capability
|
|
718
|
+
|| capability.state !== "settled"
|
|
719
|
+
|| capability.runId !== prepared.run.runId
|
|
720
|
+
|| capability.attemptId !== prepared.attemptId
|
|
721
|
+
|| capability.assignmentId !== prepared.assignmentId
|
|
722
|
+
|| capability.stepId !== prepared.stepId
|
|
723
|
+
|| capability.stepAttempt !== prepared.stepAttempt
|
|
724
|
+
) {
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
const terminalProof = isEvaluatedSettledProof(row, observation, evidence)
|
|
728
|
+
|| isExcludedTerminalProof(state, row, observation, evidence, events);
|
|
729
|
+
if (!terminalProof) return;
|
|
730
|
+
const hold = vnextGateHold(context.eventStore.path, prepared.run.runId);
|
|
731
|
+
const admittedToken = vnextAdmittedToken(context.eventStore.path, prepared.run.runId);
|
|
732
|
+
if (
|
|
733
|
+
!hold
|
|
734
|
+
|| hold.state !== "active"
|
|
735
|
+
|| hold.attemptId !== prepared.attemptId
|
|
736
|
+
|| hold.token !== token
|
|
737
|
+
|| admittedToken !== token
|
|
738
|
+
) {
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
finishVnextOwnedGate(context.eventStore.path, prepared.run.runId, token, prepared.attemptId);
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
function recoverCommandRecordingFailure(
|
|
745
|
+
context: VnextRuntimeContext,
|
|
746
|
+
prepared: VnextPreparedGateDispatch,
|
|
747
|
+
token: string,
|
|
748
|
+
observation: VnextGateObservationInput,
|
|
749
|
+
error: unknown,
|
|
750
|
+
): VnextRunDriveResult {
|
|
751
|
+
markVnextGateHoldUnsettled(context.eventStore.path, prepared.run.runId, token, prepared.attemptId);
|
|
752
|
+
if (isVnextRuntimeContextClosed(context)) {
|
|
753
|
+
throw error;
|
|
754
|
+
}
|
|
755
|
+
const existing = context.eventStore.gateObservationForAttempt(prepared.attemptId);
|
|
756
|
+
if (existing) {
|
|
757
|
+
throw runtimeError(
|
|
758
|
+
"run_events_illegal",
|
|
759
|
+
prepared.run.runId,
|
|
760
|
+
`command observation recording failed after committed proof: ${error instanceof Error ? error.message : String(error)}`,
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
const folded = foldStoredVnextRun(context, requireRun(context, prepared.run.runId));
|
|
764
|
+
if (folded.currentStep?.attemptId === prepared.attemptId && folded.currentStep.observationId) {
|
|
765
|
+
throw runtimeError(
|
|
766
|
+
"run_events_illegal",
|
|
767
|
+
prepared.run.runId,
|
|
768
|
+
"command observation recording failed after committed proof",
|
|
769
|
+
);
|
|
770
|
+
}
|
|
771
|
+
const incomplete: VnextGateObservationInput = {
|
|
772
|
+
...observation,
|
|
773
|
+
completeness: "incomplete",
|
|
774
|
+
errorClass: observation.errorClass ?? "recording-error",
|
|
775
|
+
stopCause: observation.stopCause === "none" ? "error" : observation.stopCause,
|
|
776
|
+
};
|
|
777
|
+
return commitCommandUncertainty(context, prepared, token, "recording-error", incomplete, error);
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
function commitCommandUncertainty(
|
|
781
|
+
context: VnextRuntimeContext,
|
|
782
|
+
prepared: VnextPreparedGateDispatch,
|
|
783
|
+
token: string,
|
|
784
|
+
reason: string,
|
|
785
|
+
observation: VnextGateObservationInput,
|
|
786
|
+
priorError?: unknown,
|
|
787
|
+
): VnextRunDriveResult {
|
|
788
|
+
try {
|
|
789
|
+
markVnextGateHoldUnsettled(context.eventStore.path, prepared.run.runId, token, prepared.attemptId);
|
|
790
|
+
} catch {
|
|
791
|
+
// already unsettled
|
|
792
|
+
}
|
|
793
|
+
if (isVnextRuntimeContextClosed(context)) {
|
|
794
|
+
if (priorError) throw priorError;
|
|
795
|
+
return closedCommandHandoff(context, prepared, reason);
|
|
796
|
+
}
|
|
797
|
+
const existing = context.eventStore.gateObservationForAttempt(prepared.attemptId);
|
|
798
|
+
if (existing) {
|
|
799
|
+
throw runtimeError(
|
|
800
|
+
"run_events_illegal",
|
|
801
|
+
prepared.run.runId,
|
|
802
|
+
"command observation already committed; uncertainty cannot replace it",
|
|
803
|
+
);
|
|
804
|
+
}
|
|
805
|
+
const incomplete: VnextGateObservationInput = { ...observation, completeness: "incomplete" };
|
|
806
|
+
try {
|
|
807
|
+
const written = context.eventStore.transaction(() => {
|
|
808
|
+
const result = recordGateUncertainInTransaction(context, prepared.run, prepared.envelope, prepared.stepId, reason, incomplete);
|
|
809
|
+
vnextGateDispatchSeams.afterUncertaintyWrite?.(prepared);
|
|
810
|
+
return result;
|
|
811
|
+
});
|
|
812
|
+
return {
|
|
813
|
+
state: written.state,
|
|
814
|
+
handoff: {
|
|
815
|
+
reason: "attempt_unsettled",
|
|
816
|
+
stepId: prepared.stepId,
|
|
817
|
+
attemptId: prepared.attemptId,
|
|
818
|
+
detail: reason,
|
|
819
|
+
},
|
|
820
|
+
};
|
|
821
|
+
} catch (error) {
|
|
822
|
+
if (priorError) throw priorError;
|
|
823
|
+
throw error;
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
interface PreparedDispatch {
|
|
828
|
+
run: VnextRunRecord;
|
|
829
|
+
plan: VnextCompiledPlan;
|
|
830
|
+
step: VnextCompiledStep;
|
|
831
|
+
stepId: string;
|
|
832
|
+
stepAttempt: number;
|
|
833
|
+
assignmentId: string;
|
|
834
|
+
attemptId: string;
|
|
835
|
+
agentId: string;
|
|
836
|
+
capabilityHash: string;
|
|
837
|
+
producerId: string;
|
|
838
|
+
request: VnextProducerRequest;
|
|
839
|
+
controller: AbortController;
|
|
840
|
+
state: VnextRunState;
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
interface PreparedPanel {
|
|
844
|
+
run: VnextRunRecord;
|
|
845
|
+
plan: VnextCompiledPlan;
|
|
846
|
+
step: VnextCompiledStep;
|
|
847
|
+
stepId: string;
|
|
848
|
+
stepAttempt: number;
|
|
849
|
+
target: number;
|
|
850
|
+
maxParallel: number;
|
|
851
|
+
first: PreparedDispatch;
|
|
852
|
+
state: VnextRunState;
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
export interface VnextPanelMemberHook {
|
|
856
|
+
readonly attemptId: string;
|
|
857
|
+
readonly assignmentId: string;
|
|
858
|
+
readonly stepId: string;
|
|
859
|
+
readonly stepAttempt: number;
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
export interface VnextPanelDispatchSeams {
|
|
863
|
+
afterBirth?: ((member: VnextPanelMemberHook) => void) | undefined;
|
|
864
|
+
beforeAppendExecuting?: ((member: VnextPanelMemberHook) => void) | undefined;
|
|
865
|
+
failAppendExecuting?: ((member: VnextPanelMemberHook) => boolean) | undefined;
|
|
866
|
+
beforeInvoke?: ((member: VnextPanelMemberHook) => void) | undefined;
|
|
867
|
+
failSettleMember?: ((member: VnextPanelMemberHook) => boolean) | undefined;
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
export const vnextPanelDispatchSeams: VnextPanelDispatchSeams = {};
|
|
871
|
+
|
|
872
|
+
function unreconciledPanelAttemptId(state: VnextRunState): string | undefined {
|
|
873
|
+
const current = state.currentStep;
|
|
874
|
+
if (!current) return undefined;
|
|
875
|
+
for (const assignmentId of current.panel.order) {
|
|
876
|
+
const assignment = current.panel.assignments[assignmentId];
|
|
877
|
+
if (!assignment) continue;
|
|
878
|
+
for (const [attemptId, attempt] of Object.entries(assignment.attempts)) {
|
|
879
|
+
if (attempt.status === "starting" || attempt.status === "executing" || attempt.status === "settling") {
|
|
880
|
+
return attemptId;
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
return undefined;
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
function prepareDispatch(
|
|
888
|
+
context: VnextRuntimeContext,
|
|
889
|
+
runId: string,
|
|
890
|
+
producerId?: "driver-simulated" | "pi" | string,
|
|
891
|
+
): { kind: "panel"; panel: PreparedPanel } | { kind: "return"; state: VnextRunState; handoff?: VnextRunHandoff } | ({ kind: "gate" } & VnextPreparedGateDispatch) {
|
|
892
|
+
const run = requireRun(context, runId);
|
|
893
|
+
const plan = rehydrateVnextCompiledPlanFromStore(context.eventStore, run);
|
|
894
|
+
const state = foldStoredVnextRun(context, run);
|
|
895
|
+
if (state.status === "cancelling" && vnextAttemptControllers(context.eventStore.path, runId).length === 0) {
|
|
896
|
+
return {
|
|
897
|
+
kind: "return",
|
|
898
|
+
state,
|
|
899
|
+
handoff: {
|
|
900
|
+
reason: "cancel_pending_foreign",
|
|
901
|
+
detail: "cancellation is pending in another process",
|
|
902
|
+
...(state.currentStep?.attemptId ? { attemptId: state.currentStep.attemptId } : {}),
|
|
903
|
+
},
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
const unreconciledAttemptId = unreconciledPanelAttemptId(state);
|
|
907
|
+
if (unreconciledAttemptId) {
|
|
908
|
+
const currentStepId = state.currentStep?.stepId;
|
|
909
|
+
if (currentStepId === undefined) {
|
|
910
|
+
throw runtimeError("run_events_illegal", runId, "unreconciled attempt is missing step identity");
|
|
911
|
+
}
|
|
912
|
+
return {
|
|
913
|
+
kind: "return",
|
|
914
|
+
state,
|
|
915
|
+
handoff: {
|
|
916
|
+
reason: "attempt_unreconciled",
|
|
917
|
+
attemptId: unreconciledAttemptId,
|
|
918
|
+
stepId: currentStepId,
|
|
919
|
+
detail: "issued attempt is not held by this process",
|
|
920
|
+
},
|
|
921
|
+
};
|
|
922
|
+
}
|
|
923
|
+
if (state.status === "blocked_uncertain") {
|
|
924
|
+
return {
|
|
925
|
+
kind: "return",
|
|
926
|
+
state,
|
|
927
|
+
handoff: {
|
|
928
|
+
reason: "gate_uncertain_blocked",
|
|
929
|
+
...(state.currentStep?.stepId ? { stepId: state.currentStep.stepId } : {}),
|
|
930
|
+
...(state.currentStep?.attemptId ? { attemptId: state.currentStep.attemptId } : {}),
|
|
931
|
+
detail: "run is blocked_uncertain awaiting external signal or operator intervention",
|
|
932
|
+
},
|
|
933
|
+
};
|
|
934
|
+
}
|
|
935
|
+
if (state.status !== "running" || state.currentStep) {
|
|
936
|
+
if (state.currentStep?.effectState === "blocked_uncertain") {
|
|
937
|
+
return {
|
|
938
|
+
kind: "return",
|
|
939
|
+
state,
|
|
940
|
+
handoff: {
|
|
941
|
+
reason: "attempt_unsettled",
|
|
942
|
+
stepId: state.currentStep.stepId,
|
|
943
|
+
...(state.currentStep.attemptId ? { attemptId: state.currentStep.attemptId } : {}),
|
|
944
|
+
detail: "step effect is blocked_uncertain",
|
|
945
|
+
},
|
|
946
|
+
};
|
|
947
|
+
}
|
|
948
|
+
return { kind: "return", state };
|
|
949
|
+
}
|
|
950
|
+
const stepId = state.pendingStepId ?? plan.entryStepId;
|
|
951
|
+
const step = plan.steps[stepId];
|
|
952
|
+
if (!step) throw runtimeError("run_events_illegal", runId, `unknown pending step ${stepId}`);
|
|
953
|
+
|
|
954
|
+
const allEvents = context.eventStore.events(runId, 0, 100000);
|
|
955
|
+
let accumulatedMeteredCost = 0;
|
|
956
|
+
let unmeteredOrUnknownAttempts = 0;
|
|
957
|
+
for (const ev of allEvents) {
|
|
958
|
+
if (ev.eventType === "routing.attempt.recorded") {
|
|
959
|
+
const routing = (ev.payload as { routing?: RoutingRecordV2 })?.routing;
|
|
960
|
+
if (routing?.costBasis === "metered" && typeof routing.costUsd === "number") {
|
|
961
|
+
accumulatedMeteredCost += routing.costUsd;
|
|
962
|
+
} else if (routing?.costBasis === "unmetered" || routing?.costBasis === "unknown") {
|
|
963
|
+
unmeteredOrUnknownAttempts += 1;
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
if (plan.limits.maxModelCost !== undefined && accumulatedMeteredCost >= plan.limits.maxModelCost) {
|
|
969
|
+
return { kind: "return", ...failBudget(context, run, plan, state, "budget_model_cost", stepId) };
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
const maxUnmeteredAttempts = 100;
|
|
973
|
+
if (unmeteredOrUnknownAttempts >= maxUnmeteredAttempts) {
|
|
974
|
+
return { kind: "return", ...failBudget(context, run, plan, state, "budget_unmetered_attempts", stepId) };
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
if (step.kind === "gate") {
|
|
978
|
+
const envelope = loadVnextRunPlanEnvelope(context.eventStore, run);
|
|
979
|
+
const unsupported = unsupportedGateStep(plan, step, envelope, context);
|
|
980
|
+
if (unsupported) return { kind: "return", state, handoff: { ...unsupported, stepId } };
|
|
981
|
+
const definition = envelope.gates.definitions[step.gate];
|
|
982
|
+
if (definition?.kind === "command") {
|
|
983
|
+
const preflightResult = gateRecoveryPreflight(context);
|
|
984
|
+
const firstBlocking = preflightResult.blocking[0];
|
|
985
|
+
if (firstBlocking) {
|
|
986
|
+
return {
|
|
987
|
+
kind: "return",
|
|
988
|
+
state,
|
|
989
|
+
handoff: {
|
|
990
|
+
reason: "gate_recovery_pending",
|
|
991
|
+
attemptId: firstBlocking.attemptId,
|
|
992
|
+
detail: `${preflightResult.blocking.length} unfinished gate attempt(s) in project`,
|
|
993
|
+
},
|
|
994
|
+
};
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
const result = recordGateIntentInTransaction(context, run, envelope, stepId);
|
|
998
|
+
const current = result.state.currentStep;
|
|
999
|
+
const attemptId = current?.attemptId;
|
|
1000
|
+
const assignmentId = current?.assignmentId;
|
|
1001
|
+
const effectId = current?.effectId;
|
|
1002
|
+
if (
|
|
1003
|
+
!current
|
|
1004
|
+
|| current.stepId !== stepId
|
|
1005
|
+
|| attemptId === undefined
|
|
1006
|
+
|| assignmentId === undefined
|
|
1007
|
+
|| effectId === undefined
|
|
1008
|
+
) {
|
|
1009
|
+
throw runtimeError("run_events_illegal", runId, "gate intent did not bind exact attempt identity");
|
|
1010
|
+
}
|
|
1011
|
+
return {
|
|
1012
|
+
kind: "gate",
|
|
1013
|
+
run,
|
|
1014
|
+
envelope,
|
|
1015
|
+
stepId,
|
|
1016
|
+
stepAttempt: current.stepAttempt,
|
|
1017
|
+
assignmentId,
|
|
1018
|
+
attemptId,
|
|
1019
|
+
effectId,
|
|
1020
|
+
controller: new AbortController(),
|
|
1021
|
+
state: result.state,
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
const unsupported = unsupportedStep(plan, step);
|
|
1026
|
+
if (unsupported) return { kind: "return", state, handoff: { ...unsupported, stepId } };
|
|
1027
|
+
const used = state.stepAttempts[stepId] ?? 0;
|
|
1028
|
+
if (used >= step.maxAttempts) {
|
|
1029
|
+
return { kind: "return", ...failBudget(context, run, plan, state, "budget_step_attempts", stepId) };
|
|
1030
|
+
}
|
|
1031
|
+
const stepAttempt = used + 1;
|
|
1032
|
+
const now = new Date().toISOString();
|
|
1033
|
+
let sequence = context.eventStore.nextSequence(runId);
|
|
1034
|
+
let mono = vnextMonotonicNs();
|
|
1035
|
+
const events: VnextRunEvent[] = [];
|
|
1036
|
+
const push = (eventType: string, payload: Record<string, unknown>): void => {
|
|
1037
|
+
events.push({
|
|
1038
|
+
...vnextEventBase(context, run, now, mono),
|
|
1039
|
+
eventId: newVnextEventId(),
|
|
1040
|
+
eventType,
|
|
1041
|
+
sequence: sequence++,
|
|
1042
|
+
payload,
|
|
1043
|
+
});
|
|
1044
|
+
mono = vnextIncrementMonotonicNs(mono);
|
|
1045
|
+
};
|
|
1046
|
+
push("step.entered", { stepId, stepAttempt, status: "pending" });
|
|
1047
|
+
push("step.status_changed", { stepId, status: "preparing", previousStatus: "pending" });
|
|
1048
|
+
for (const event of events) context.eventStore.appendEvent(event);
|
|
1049
|
+
const entered = foldStoredVnextRun(context, run);
|
|
1050
|
+
persistVnextRunState(context, runId, entered, events[events.length - 1]!.sequence);
|
|
1051
|
+
const first = birthMember(context, {
|
|
1052
|
+
run,
|
|
1053
|
+
plan,
|
|
1054
|
+
step,
|
|
1055
|
+
stepId,
|
|
1056
|
+
stepAttempt,
|
|
1057
|
+
enterRunning: true,
|
|
1058
|
+
producerId,
|
|
1059
|
+
});
|
|
1060
|
+
return {
|
|
1061
|
+
kind: "panel",
|
|
1062
|
+
panel: {
|
|
1063
|
+
run,
|
|
1064
|
+
plan,
|
|
1065
|
+
step,
|
|
1066
|
+
stepId,
|
|
1067
|
+
stepAttempt,
|
|
1068
|
+
target: step.assignments.target,
|
|
1069
|
+
maxParallel: step.assignments.maxParallel,
|
|
1070
|
+
first,
|
|
1071
|
+
state: first.state,
|
|
1072
|
+
},
|
|
1073
|
+
};
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
function panelInFlight(state: VnextRunState): number {
|
|
1077
|
+
const current = state.currentStep;
|
|
1078
|
+
if (!current) return 0;
|
|
1079
|
+
let count = 0;
|
|
1080
|
+
for (const assignmentId of current.panel.order) {
|
|
1081
|
+
const assignment = current.panel.assignments[assignmentId];
|
|
1082
|
+
if (!assignment) continue;
|
|
1083
|
+
for (const attempt of Object.values(assignment.attempts)) {
|
|
1084
|
+
if (attempt.status === "starting" || attempt.status === "executing" || attempt.status === "settling") count += 1;
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
return count;
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
function panelFrozenState(state: VnextRunState): boolean {
|
|
1091
|
+
const current = state.currentStep;
|
|
1092
|
+
if (!current) return true;
|
|
1093
|
+
return current.outcome !== undefined
|
|
1094
|
+
|| current.status === "passed"
|
|
1095
|
+
|| current.status === "failed"
|
|
1096
|
+
|| current.status === "cancelled";
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
function birthAllowed(state: VnextRunState, step: VnextCompiledStep): boolean {
|
|
1100
|
+
if (state.status !== "running" || state.cancelRequested || panelFrozenState(state)) return false;
|
|
1101
|
+
const born = state.currentStep?.panel.order.length ?? 0;
|
|
1102
|
+
if (born >= step.assignments.target) return false;
|
|
1103
|
+
if (panelInFlight(state) >= step.assignments.maxParallel) return false;
|
|
1104
|
+
return true;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
function birthMember(
|
|
1108
|
+
context: VnextRuntimeContext,
|
|
1109
|
+
input: {
|
|
1110
|
+
run: VnextRunRecord;
|
|
1111
|
+
plan: VnextCompiledPlan;
|
|
1112
|
+
step: VnextCompiledStep;
|
|
1113
|
+
stepId: string;
|
|
1114
|
+
stepAttempt: number;
|
|
1115
|
+
enterRunning?: boolean | undefined;
|
|
1116
|
+
producerId?: ("driver-simulated" | "pi" | string) | undefined;
|
|
1117
|
+
},
|
|
1118
|
+
): PreparedDispatch {
|
|
1119
|
+
const run = requireRun(context, input.run.runId);
|
|
1120
|
+
const folded = foldStoredVnextRun(context, run);
|
|
1121
|
+
if (!birthAllowed(folded, input.step)) {
|
|
1122
|
+
throw runtimeError("run_events_illegal", run.runId, "member birth is not legal");
|
|
1123
|
+
}
|
|
1124
|
+
const born = folded.currentStep?.panel.order.length ?? 0;
|
|
1125
|
+
const allowed = input.step.assignments.allowedAgents;
|
|
1126
|
+
const agentId = (allowed && allowed.length > born && allowed[born])
|
|
1127
|
+
? allowed[born]!
|
|
1128
|
+
: (input.step.kind === "agent" || input.step.kind === "moa" ? input.step.agent : "coordinator");
|
|
1129
|
+
const assignmentId = newVnextAssignmentId();
|
|
1130
|
+
const attemptId = newVnextAttemptId();
|
|
1131
|
+
const minted = mintCapabilitySecret();
|
|
1132
|
+
const controller = new AbortController();
|
|
1133
|
+
const now = new Date().toISOString();
|
|
1134
|
+
let sequence = context.eventStore.nextSequence(run.runId);
|
|
1135
|
+
let mono = vnextMonotonicNs();
|
|
1136
|
+
const events: VnextRunEvent[] = [];
|
|
1137
|
+
const push = (eventType: string, payload: Record<string, unknown>): void => {
|
|
1138
|
+
events.push({
|
|
1139
|
+
...vnextEventBase(context, run, now, mono),
|
|
1140
|
+
eventId: newVnextEventId(),
|
|
1141
|
+
eventType,
|
|
1142
|
+
sequence: sequence++,
|
|
1143
|
+
payload,
|
|
1144
|
+
});
|
|
1145
|
+
mono = vnextIncrementMonotonicNs(mono);
|
|
1146
|
+
};
|
|
1147
|
+
push("assignment.created", { assignmentId, stepId: input.stepId, stepAttempt: input.stepAttempt, agentId, status: "created" });
|
|
1148
|
+
push("assignment.accepted", { assignmentId, status: "accepted" });
|
|
1149
|
+
push("attempt.created", { attemptId, assignmentId, status: "created" });
|
|
1150
|
+
push("assignment.dispatched", { assignmentId, capabilityHash: minted.hash, status: "dispatched" });
|
|
1151
|
+
push("attempt.status_changed", { attemptId, status: "starting" });
|
|
1152
|
+
if (input.enterRunning) {
|
|
1153
|
+
push("step.status_changed", { stepId: input.stepId, status: "running", previousStatus: "preparing" });
|
|
1154
|
+
}
|
|
1155
|
+
for (const event of events) context.eventStore.appendEvent(event);
|
|
1156
|
+
context.eventStore.insertCapability({
|
|
1157
|
+
attemptId,
|
|
1158
|
+
runId: run.runId,
|
|
1159
|
+
assignmentId,
|
|
1160
|
+
stepId: input.stepId,
|
|
1161
|
+
stepAttempt: input.stepAttempt,
|
|
1162
|
+
producerId: input.producerId ?? "driver-simulated",
|
|
1163
|
+
capabilityHash: minted.hash,
|
|
1164
|
+
state: "issued",
|
|
1165
|
+
});
|
|
1166
|
+
const next = foldStoredVnextRun(context, run);
|
|
1167
|
+
persistVnextRunState(context, run.runId, next, events[events.length - 1]!.sequence);
|
|
1168
|
+
const contextPacket = buildFormalContextPacket({
|
|
1169
|
+
project: run.projectId,
|
|
1170
|
+
targetRole: agentId,
|
|
1171
|
+
task: {
|
|
1172
|
+
taskId: run.runId,
|
|
1173
|
+
stepId: input.stepId,
|
|
1174
|
+
stepAttempt: input.stepAttempt,
|
|
1175
|
+
objective: input.step.description ?? input.step.instructions ?? input.stepId,
|
|
1176
|
+
allowedOutcomes: [...input.step.outcomes],
|
|
1177
|
+
permissionCeiling: "edit",
|
|
1178
|
+
},
|
|
1179
|
+
acceptanceCriteria: input.step.requiredEvidence.map((ev) => ({
|
|
1180
|
+
id: ev.key,
|
|
1181
|
+
description: `Provide ${ev.kind} evidence for ${ev.key}`,
|
|
1182
|
+
verificationKind: "witness",
|
|
1183
|
+
required: true,
|
|
1184
|
+
})),
|
|
1185
|
+
plan: {
|
|
1186
|
+
planHash: input.plan.planHash?.stageId ?? "default",
|
|
1187
|
+
activeStepIndex: input.step.index,
|
|
1188
|
+
totalSteps: input.plan.order.length,
|
|
1189
|
+
settledDecisions: [],
|
|
1190
|
+
},
|
|
1191
|
+
});
|
|
1192
|
+
const generatedPrompt = formatContextPacketForPrompt(contextPacket);
|
|
1193
|
+
|
|
1194
|
+
const member: PreparedDispatch = {
|
|
1195
|
+
run,
|
|
1196
|
+
plan: input.plan,
|
|
1197
|
+
step: input.step,
|
|
1198
|
+
stepId: input.stepId,
|
|
1199
|
+
stepAttempt: input.stepAttempt,
|
|
1200
|
+
assignmentId,
|
|
1201
|
+
attemptId,
|
|
1202
|
+
agentId,
|
|
1203
|
+
capabilityHash: minted.hash,
|
|
1204
|
+
producerId: input.producerId ?? "driver-simulated",
|
|
1205
|
+
request: {
|
|
1206
|
+
runId: run.runId,
|
|
1207
|
+
stepId: input.stepId,
|
|
1208
|
+
stepAttempt: input.stepAttempt,
|
|
1209
|
+
assignmentId,
|
|
1210
|
+
attemptId,
|
|
1211
|
+
agentId,
|
|
1212
|
+
instanceNo: born + 1,
|
|
1213
|
+
scopeEpoch: 1,
|
|
1214
|
+
capability: minted.secret,
|
|
1215
|
+
allowedOutcomes: input.step.outcomes,
|
|
1216
|
+
signal: controller.signal,
|
|
1217
|
+
prompt: input.step.instructions ? `${input.step.instructions}\n\n${generatedPrompt}` : generatedPrompt,
|
|
1218
|
+
contextPacket,
|
|
1219
|
+
},
|
|
1220
|
+
controller,
|
|
1221
|
+
state: next,
|
|
1222
|
+
};
|
|
1223
|
+
vnextPanelDispatchSeams.afterBirth?.(member);
|
|
1224
|
+
return member;
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
function appendExecuting(context: VnextRuntimeContext, dispatch: PreparedDispatch): void {
|
|
1228
|
+
const run = requireRun(context, dispatch.run.runId);
|
|
1229
|
+
const now = new Date().toISOString();
|
|
1230
|
+
let sequence = context.eventStore.nextSequence(dispatch.run.runId);
|
|
1231
|
+
let mono = vnextMonotonicNs();
|
|
1232
|
+
const events: VnextRunEvent[] = [
|
|
1233
|
+
{
|
|
1234
|
+
...vnextEventBase(context, run, now, mono),
|
|
1235
|
+
eventId: newVnextEventId(),
|
|
1236
|
+
eventType: "assignment.executing",
|
|
1237
|
+
sequence: sequence++,
|
|
1238
|
+
payload: { assignmentId: dispatch.assignmentId, status: "executing" },
|
|
1239
|
+
},
|
|
1240
|
+
{
|
|
1241
|
+
...vnextEventBase(context, run, now, vnextIncrementMonotonicNs(mono)),
|
|
1242
|
+
eventId: newVnextEventId(),
|
|
1243
|
+
eventType: "attempt.status_changed",
|
|
1244
|
+
sequence: sequence++,
|
|
1245
|
+
payload: { attemptId: dispatch.attemptId, status: "executing" },
|
|
1246
|
+
},
|
|
1247
|
+
];
|
|
1248
|
+
for (const event of events) context.eventStore.appendEvent(event);
|
|
1249
|
+
const next = foldStoredVnextRun(context, run);
|
|
1250
|
+
persistVnextRunState(context, run.runId, next, events[events.length - 1]!.sequence);
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
type MemberWork =
|
|
1254
|
+
| { attemptId: string; invoked: true; produced: { result?: VnextProducerResult; error: boolean } }
|
|
1255
|
+
| { attemptId: string; invoked: false; skipped: true }
|
|
1256
|
+
| { attemptId: string; invoked: false; appendFailed: true; error: unknown };
|
|
1257
|
+
|
|
1258
|
+
function singletonPanel(step: VnextCompiledStep): boolean {
|
|
1259
|
+
return step.assignments.maximum === 1 && step.assignments.target === 1;
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
function unreconciledHandoff(state: VnextRunState, stepId: string, attemptId?: string): VnextRunDriveResult {
|
|
1263
|
+
return {
|
|
1264
|
+
state,
|
|
1265
|
+
handoff: {
|
|
1266
|
+
reason: "attempt_unreconciled",
|
|
1267
|
+
stepId,
|
|
1268
|
+
...(attemptId !== undefined ? { attemptId } : {}),
|
|
1269
|
+
detail: "issued attempt is not held by this process",
|
|
1270
|
+
},
|
|
1271
|
+
};
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
function capabilityHashBound(stored: string, minted: string): boolean {
|
|
1275
|
+
const left = Buffer.from(stored);
|
|
1276
|
+
const right = Buffer.from(minted);
|
|
1277
|
+
return left.length === right.length && timingSafeEqual(left, right);
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
function capabilityMatchesDispatch(
|
|
1281
|
+
capability: VnextAttemptCapabilityRow | undefined,
|
|
1282
|
+
member: PreparedDispatch,
|
|
1283
|
+
allowedStates: ReadonlyArray<VnextAttemptCapabilityRow["state"]>,
|
|
1284
|
+
expectedProducerId?: string,
|
|
1285
|
+
): boolean {
|
|
1286
|
+
return Boolean(
|
|
1287
|
+
capability
|
|
1288
|
+
&& capabilityHashBound(capability.capabilityHash, member.capabilityHash)
|
|
1289
|
+
&& capability.runId === member.run.runId
|
|
1290
|
+
&& capability.stepId === member.stepId
|
|
1291
|
+
&& capability.stepAttempt === member.stepAttempt
|
|
1292
|
+
&& capability.assignmentId === member.assignmentId
|
|
1293
|
+
&& capability.attemptId === member.attemptId
|
|
1294
|
+
&& capability.producerId === (expectedProducerId ?? member.producerId ?? "driver-simulated")
|
|
1295
|
+
&& allowedStates.includes(capability.state),
|
|
1296
|
+
);
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
async function drivePanel(
|
|
1300
|
+
context: VnextRuntimeContext,
|
|
1301
|
+
panel: PreparedPanel,
|
|
1302
|
+
producer: VnextProducer,
|
|
1303
|
+
): Promise<VnextRunDriveResult> {
|
|
1304
|
+
const runId = panel.run.runId;
|
|
1305
|
+
const owned = new Map<string, PreparedDispatch>();
|
|
1306
|
+
const pending = new Map<string, Promise<MemberWork>>();
|
|
1307
|
+
let stopBirths = false;
|
|
1308
|
+
let settlementFailed = false;
|
|
1309
|
+
|
|
1310
|
+
const register = (member: PreparedDispatch): void => {
|
|
1311
|
+
registerVnextAttemptController(context.eventStore.path, runId, { attemptId: member.attemptId, controller: member.controller });
|
|
1312
|
+
owned.set(member.attemptId, member);
|
|
1313
|
+
};
|
|
1314
|
+
|
|
1315
|
+
const abortOwned = (): void => {
|
|
1316
|
+
for (const member of owned.values()) member.controller.abort();
|
|
1317
|
+
};
|
|
1318
|
+
|
|
1319
|
+
const launch = (member: PreparedDispatch): void => {
|
|
1320
|
+
register(member);
|
|
1321
|
+
const work = (async (): Promise<MemberWork> => {
|
|
1322
|
+
try {
|
|
1323
|
+
vnextPanelDispatchSeams.beforeAppendExecuting?.(member);
|
|
1324
|
+
const started = context.eventStore.transaction(() => {
|
|
1325
|
+
if (vnextPanelDispatchSeams.failAppendExecuting?.(member)) {
|
|
1326
|
+
throw runtimeError("run_events_illegal", runId, "appendExecuting failed");
|
|
1327
|
+
}
|
|
1328
|
+
const run = requireRun(context, runId);
|
|
1329
|
+
const state = foldStoredVnextRun(context, run);
|
|
1330
|
+
const capability = context.eventStore.capabilityByAttempt(member.attemptId);
|
|
1331
|
+
const located = vnextFoldPanelAttempt(state.currentStep, member.attemptId);
|
|
1332
|
+
if (
|
|
1333
|
+
state.cancelRequested
|
|
1334
|
+
|| state.status === "cancelling"
|
|
1335
|
+
|| !capabilityMatchesDispatch(capability, member, ["issued"], producer.id)
|
|
1336
|
+
|| !located
|
|
1337
|
+
|| located.assignmentId !== member.assignmentId
|
|
1338
|
+
|| located.attempt.status !== "starting"
|
|
1339
|
+
) {
|
|
1340
|
+
return false;
|
|
1341
|
+
}
|
|
1342
|
+
appendExecuting(context, member);
|
|
1343
|
+
return true;
|
|
1344
|
+
});
|
|
1345
|
+
if (!started) return { attemptId: member.attemptId, invoked: false, skipped: true };
|
|
1346
|
+
const executingBound = (): boolean => {
|
|
1347
|
+
const run = requireRun(context, runId);
|
|
1348
|
+
const state = foldStoredVnextRun(context, run);
|
|
1349
|
+
const capability = context.eventStore.capabilityByAttempt(member.attemptId);
|
|
1350
|
+
const located = vnextFoldPanelAttempt(state.currentStep, member.attemptId);
|
|
1351
|
+
if (
|
|
1352
|
+
state.cancelRequested
|
|
1353
|
+
|| state.status === "cancelling"
|
|
1354
|
+
|| !capabilityMatchesDispatch(capability, member, ["issued"], producer.id)
|
|
1355
|
+
|| !located
|
|
1356
|
+
|| located.assignmentId !== member.assignmentId
|
|
1357
|
+
|| located.attempt.status !== "executing"
|
|
1358
|
+
) {
|
|
1359
|
+
return false;
|
|
1360
|
+
}
|
|
1361
|
+
return true;
|
|
1362
|
+
};
|
|
1363
|
+
if (!executingBound()) return { attemptId: member.attemptId, invoked: false, skipped: true };
|
|
1364
|
+
vnextPanelDispatchSeams.beforeInvoke?.(member);
|
|
1365
|
+
if (!executingBound()) return { attemptId: member.attemptId, invoked: false, skipped: true };
|
|
1366
|
+
const produced = await invokeProducer(producer, member.request);
|
|
1367
|
+
return { attemptId: member.attemptId, invoked: true, produced };
|
|
1368
|
+
} catch (error) {
|
|
1369
|
+
member.controller.abort();
|
|
1370
|
+
return { attemptId: member.attemptId, invoked: false, appendFailed: true, error };
|
|
1371
|
+
}
|
|
1372
|
+
})();
|
|
1373
|
+
pending.set(member.attemptId, work);
|
|
1374
|
+
};
|
|
1375
|
+
|
|
1376
|
+
const tryBirth = (): PreparedDispatch | undefined => {
|
|
1377
|
+
if (stopBirths) return undefined;
|
|
1378
|
+
return context.eventStore.transaction(() => {
|
|
1379
|
+
const run = requireRun(context, runId);
|
|
1380
|
+
const state = foldStoredVnextRun(context, run);
|
|
1381
|
+
if (!birthAllowed(state, panel.step)) return undefined;
|
|
1382
|
+
return birthMember(context, {
|
|
1383
|
+
run,
|
|
1384
|
+
plan: panel.plan,
|
|
1385
|
+
step: panel.step,
|
|
1386
|
+
stepId: panel.stepId,
|
|
1387
|
+
stepAttempt: panel.stepAttempt,
|
|
1388
|
+
producerId: producer.id,
|
|
1389
|
+
});
|
|
1390
|
+
});
|
|
1391
|
+
};
|
|
1392
|
+
|
|
1393
|
+
const settleInvoked = (member: PreparedDispatch, produced: { result?: VnextProducerResult; error: boolean }): boolean => {
|
|
1394
|
+
try {
|
|
1395
|
+
context.eventStore.transaction(() => {
|
|
1396
|
+
if (vnextPanelDispatchSeams.failSettleMember?.(member)) {
|
|
1397
|
+
throw runtimeError("run_events_illegal", runId, "member settlement write failed");
|
|
1398
|
+
}
|
|
1399
|
+
settleMember(context, member, produced.result, produced.error);
|
|
1400
|
+
});
|
|
1401
|
+
return true;
|
|
1402
|
+
} catch (err) {
|
|
1403
|
+
if (err instanceof Error && /costBasis/.test(err.message)) {
|
|
1404
|
+
throw err;
|
|
1405
|
+
}
|
|
1406
|
+
settlementFailed = true;
|
|
1407
|
+
stopBirths = true;
|
|
1408
|
+
return false;
|
|
1409
|
+
}
|
|
1410
|
+
};
|
|
1411
|
+
|
|
1412
|
+
const drainPendingInvoked = async (): Promise<void> => {
|
|
1413
|
+
const rest = await Promise.allSettled([...pending.values()]);
|
|
1414
|
+
pending.clear();
|
|
1415
|
+
for (const result of rest) {
|
|
1416
|
+
if (result.status !== "fulfilled") continue;
|
|
1417
|
+
const item = result.value;
|
|
1418
|
+
const sibling = owned.get(item.attemptId);
|
|
1419
|
+
if (sibling && item.invoked) {
|
|
1420
|
+
settleInvoked(sibling, item.produced);
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
};
|
|
1424
|
+
|
|
1425
|
+
try {
|
|
1426
|
+
launch(panel.first);
|
|
1427
|
+
while (true) {
|
|
1428
|
+
while (!stopBirths && !settlementFailed) {
|
|
1429
|
+
try {
|
|
1430
|
+
const next = tryBirth();
|
|
1431
|
+
if (!next) break;
|
|
1432
|
+
launch(next);
|
|
1433
|
+
} catch {
|
|
1434
|
+
stopBirths = true;
|
|
1435
|
+
abortOwned();
|
|
1436
|
+
await drainPendingInvoked();
|
|
1437
|
+
const state = foldStoredVnextRun(context, requireRun(context, runId));
|
|
1438
|
+
return unreconciledHandoff(state, panel.stepId);
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1441
|
+
if (pending.size === 0) break;
|
|
1442
|
+
const finished = await Promise.race(pending.values());
|
|
1443
|
+
pending.delete(finished.attemptId);
|
|
1444
|
+
const member = owned.get(finished.attemptId);
|
|
1445
|
+
if (!member) continue;
|
|
1446
|
+
if ("appendFailed" in finished && finished.appendFailed) {
|
|
1447
|
+
stopBirths = true;
|
|
1448
|
+
abortOwned();
|
|
1449
|
+
const rest = await Promise.all([...pending.values()]);
|
|
1450
|
+
pending.clear();
|
|
1451
|
+
if (singletonPanel(panel.step)) {
|
|
1452
|
+
context.eventStore.transaction(() => hardStopUnrecorded(context, member, "executing_unrecorded"));
|
|
1453
|
+
throw finished.error;
|
|
1454
|
+
}
|
|
1455
|
+
for (const item of rest) {
|
|
1456
|
+
const sibling = owned.get(item.attemptId);
|
|
1457
|
+
if (sibling && item.invoked) {
|
|
1458
|
+
settleInvoked(sibling, item.produced);
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
const state = foldStoredVnextRun(context, requireRun(context, runId));
|
|
1462
|
+
return unreconciledHandoff(state, panel.stepId, member.attemptId);
|
|
1463
|
+
}
|
|
1464
|
+
if (!finished.invoked) {
|
|
1465
|
+
stopBirths = true;
|
|
1466
|
+
continue;
|
|
1467
|
+
}
|
|
1468
|
+
if (!settleInvoked(member, finished.produced)) {
|
|
1469
|
+
abortOwned();
|
|
1470
|
+
const rest = await Promise.all([...pending.values()]);
|
|
1471
|
+
pending.clear();
|
|
1472
|
+
for (const item of rest) {
|
|
1473
|
+
const sibling = owned.get(item.attemptId);
|
|
1474
|
+
if (sibling && item.invoked) {
|
|
1475
|
+
settleInvoked(sibling, item.produced);
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
const state = foldStoredVnextRun(context, requireRun(context, runId));
|
|
1479
|
+
return unreconciledHandoff(state, panel.stepId, member.attemptId);
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
const folded = foldStoredVnextRun(context, requireRun(context, runId));
|
|
1484
|
+
const leftover = unreconciledPanelAttemptId(folded);
|
|
1485
|
+
if (leftover || settlementFailed) {
|
|
1486
|
+
return unreconciledHandoff(folded, panel.stepId, leftover);
|
|
1487
|
+
}
|
|
1488
|
+
const born = folded.currentStep?.panel.order.length ?? 0;
|
|
1489
|
+
const allTerminal = Boolean(
|
|
1490
|
+
folded.currentStep
|
|
1491
|
+
&& born > 0
|
|
1492
|
+
&& folded.currentStep.panel.order.every((assignmentId) => folded.currentStep?.panel.assignments[assignmentId]?.status === "terminal")
|
|
1493
|
+
&& vnextFoldPanelAttemptIds(folded.currentStep).every((attemptId) => vnextFoldPanelAttempt(folded.currentStep, attemptId)?.attempt.status === "terminal"),
|
|
1494
|
+
);
|
|
1495
|
+
if (!allTerminal) {
|
|
1496
|
+
return unreconciledHandoff(folded, panel.stepId, leftover);
|
|
1497
|
+
}
|
|
1498
|
+
if (born < panel.target && folded.status !== "cancelling" && !folded.cancelRequested) {
|
|
1499
|
+
return unreconciledHandoff(folded, panel.stepId);
|
|
1500
|
+
}
|
|
1501
|
+
try {
|
|
1502
|
+
return context.eventStore.transaction(() => joinPanel(context, panel));
|
|
1503
|
+
} catch {
|
|
1504
|
+
const state = foldStoredVnextRun(context, requireRun(context, runId));
|
|
1505
|
+
return unreconciledHandoff(state, panel.stepId);
|
|
1506
|
+
}
|
|
1507
|
+
} catch (error) {
|
|
1508
|
+
abortOwned();
|
|
1509
|
+
await drainPendingInvoked();
|
|
1510
|
+
throw error;
|
|
1511
|
+
} finally {
|
|
1512
|
+
for (const attemptId of owned.keys()) {
|
|
1513
|
+
unregisterVnextAttemptController(context.eventStore.path, runId, attemptId);
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
function settleMember(
|
|
1519
|
+
context: VnextRuntimeContext,
|
|
1520
|
+
dispatch: PreparedDispatch,
|
|
1521
|
+
result: VnextProducerResult | undefined,
|
|
1522
|
+
produceError: boolean,
|
|
1523
|
+
): void {
|
|
1524
|
+
const run = requireRun(context, dispatch.run.runId);
|
|
1525
|
+
const state = foldStoredVnextRun(context, run);
|
|
1526
|
+
const capability = context.eventStore.capabilityByAttempt(dispatch.attemptId);
|
|
1527
|
+
const located = vnextFoldPanelAttempt(state.currentStep, dispatch.attemptId);
|
|
1528
|
+
if (
|
|
1529
|
+
!located
|
|
1530
|
+
|| located.assignmentId !== dispatch.assignmentId
|
|
1531
|
+
|| located.attempt.status !== "executing"
|
|
1532
|
+
|| !capability
|
|
1533
|
+
|| !capabilityMatchesDispatch(capability, dispatch, ["issued", "revoked"])
|
|
1534
|
+
) {
|
|
1535
|
+
throw runtimeError("run_events_illegal", run.runId, "settle attempted for a non-current capability");
|
|
1536
|
+
}
|
|
1537
|
+
const cancelling = state.status === "cancelling" || capability.state === "revoked";
|
|
1538
|
+
const now = new Date().toISOString();
|
|
1539
|
+
let sequence = context.eventStore.nextSequence(run.runId);
|
|
1540
|
+
let mono = vnextMonotonicNs();
|
|
1541
|
+
const events: VnextRunEvent[] = [];
|
|
1542
|
+
const push = (eventType: string, payload: Record<string, unknown>): void => {
|
|
1543
|
+
events.push({
|
|
1544
|
+
...vnextEventBase(context, run, now, mono),
|
|
1545
|
+
eventId: newVnextEventId(),
|
|
1546
|
+
eventType,
|
|
1547
|
+
sequence: sequence++,
|
|
1548
|
+
payload,
|
|
1549
|
+
});
|
|
1550
|
+
mono = vnextIncrementMonotonicNs(mono);
|
|
1551
|
+
};
|
|
1552
|
+
push("attempt.status_changed", { attemptId: dispatch.attemptId, status: "settling" });
|
|
1553
|
+
if (cancelling) {
|
|
1554
|
+
push("assignment.result_recorded", { assignmentId: dispatch.assignmentId, resultClass: "cancelled", status: "result_recorded" });
|
|
1555
|
+
push("attempt.status_changed", { attemptId: dispatch.attemptId, status: "terminal" });
|
|
1556
|
+
push("assignment.terminal", { assignmentId: dispatch.assignmentId, outcome: "cancelled", status: "terminal" });
|
|
1557
|
+
} else if (produceError) {
|
|
1558
|
+
const envelope = loadVnextRunPlanEnvelope(context.eventStore, run);
|
|
1559
|
+
const rawHash = result?.behavioralSha256 ?? behavioralConfigHash({
|
|
1560
|
+
requestedModel: result?.requestedModel ?? "simulated",
|
|
1561
|
+
effectiveModel: result?.effectiveModel ?? result?.requestedModel ?? "simulated",
|
|
1562
|
+
agentRole: dispatch.step.kind === "agent" || dispatch.step.kind === "moa" ? dispatch.step.agent : undefined,
|
|
1563
|
+
toolPolicyVersion: envelope.revisions.toolPolicy,
|
|
1564
|
+
});
|
|
1565
|
+
const behavioralSha256 = rawHash.startsWith("sha256:") ? rawHash : `sha256:${rawHash}`;
|
|
1566
|
+
|
|
1567
|
+
const routingRecord: RoutingRecordV2 = {
|
|
1568
|
+
schema: ROUTING_RECORD_V2_SCHEMA,
|
|
1569
|
+
recordedAt: now,
|
|
1570
|
+
project: run.projectId,
|
|
1571
|
+
runId: run.runId,
|
|
1572
|
+
stepId: dispatch.stepId,
|
|
1573
|
+
assignmentId: dispatch.assignmentId,
|
|
1574
|
+
attemptId: dispatch.attemptId,
|
|
1575
|
+
harness: result?.harness ?? (dispatch.producerId === "pi" ? "pi" : "driver-simulated"),
|
|
1576
|
+
provider: result?.provider ?? "simulated",
|
|
1577
|
+
requestedModel: result?.requestedModel ?? "simulated",
|
|
1578
|
+
effectiveModel: result?.effectiveModel ?? result?.requestedModel ?? "simulated",
|
|
1579
|
+
behavioralSha256,
|
|
1580
|
+
latencyMs: typeof result?.latencyMs === "number" ? result.latencyMs : 0,
|
|
1581
|
+
costBasis: "unknown",
|
|
1582
|
+
costUsd: null,
|
|
1583
|
+
retries: Math.max(0, dispatch.stepAttempt - 1),
|
|
1584
|
+
};
|
|
1585
|
+
push("routing.attempt.recorded", { routing: parseRoutingRecordV2(routingRecord) });
|
|
1586
|
+
|
|
1587
|
+
push("assignment.result_recorded", {
|
|
1588
|
+
assignmentId: dispatch.assignmentId,
|
|
1589
|
+
resultClass: "producer_rejected",
|
|
1590
|
+
status: "result_recorded",
|
|
1591
|
+
});
|
|
1592
|
+
push("attempt.status_changed", { attemptId: dispatch.attemptId, status: "terminal" });
|
|
1593
|
+
push("assignment.terminal", { assignmentId: dispatch.assignmentId, outcome: "failed", status: "terminal" });
|
|
1594
|
+
} else {
|
|
1595
|
+
if (!result || result.costBasis === undefined || result.costBasis === null) {
|
|
1596
|
+
throw runtimeError("settle_missing_cost_basis", run.runId, `attempt settlement rejected: missing required costBasis for attempt ${dispatch.attemptId}`);
|
|
1597
|
+
}
|
|
1598
|
+
if (result.costBasis !== "metered" && result.costBasis !== "unmetered" && result.costBasis !== "unknown") {
|
|
1599
|
+
throw runtimeError("settle_invalid_cost_basis", run.runId, `attempt settlement rejected: invalid costBasis ${String(result.costBasis)}`);
|
|
1600
|
+
}
|
|
1601
|
+
if (result.costBasis === "metered") {
|
|
1602
|
+
if (typeof result.costUsd !== "number" || !Number.isFinite(result.costUsd) || result.costUsd < 0) {
|
|
1603
|
+
throw runtimeError("settle_invalid_cost", run.runId, "attempt settlement rejected: metered costBasis requires non-negative finite costUsd");
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1607
|
+
const envelope = loadVnextRunPlanEnvelope(context.eventStore, run);
|
|
1608
|
+
const rawHash = result.behavioralSha256 ?? behavioralConfigHash({
|
|
1609
|
+
requestedModel: result.requestedModel ?? "simulated",
|
|
1610
|
+
effectiveModel: result.effectiveModel ?? result.requestedModel ?? "simulated",
|
|
1611
|
+
agentRole: dispatch.step.kind === "agent" || dispatch.step.kind === "moa" ? dispatch.step.agent : undefined,
|
|
1612
|
+
toolPolicyVersion: envelope.revisions.toolPolicy,
|
|
1613
|
+
});
|
|
1614
|
+
const behavioralSha256 = rawHash.startsWith("sha256:") ? rawHash : `sha256:${rawHash}`;
|
|
1615
|
+
|
|
1616
|
+
const routingRecord: RoutingRecordV2 = {
|
|
1617
|
+
schema: ROUTING_RECORD_V2_SCHEMA,
|
|
1618
|
+
recordedAt: now,
|
|
1619
|
+
project: run.projectId,
|
|
1620
|
+
runId: run.runId,
|
|
1621
|
+
stepId: dispatch.stepId,
|
|
1622
|
+
assignmentId: dispatch.assignmentId,
|
|
1623
|
+
attemptId: dispatch.attemptId,
|
|
1624
|
+
harness: result.harness ?? (dispatch.producerId === "pi" ? "pi" : "driver-simulated"),
|
|
1625
|
+
provider: result.provider ?? "simulated",
|
|
1626
|
+
requestedModel: result.requestedModel ?? "simulated",
|
|
1627
|
+
effectiveModel: result.effectiveModel ?? result.requestedModel ?? "simulated",
|
|
1628
|
+
behavioralSha256,
|
|
1629
|
+
latencyMs: typeof result.latencyMs === "number" ? result.latencyMs : 0,
|
|
1630
|
+
costBasis: result.costBasis,
|
|
1631
|
+
costUsd: result.costBasis === "metered" ? (result.costUsd ?? 0) : (result.costUsd ?? null),
|
|
1632
|
+
retries: Math.max(0, dispatch.stepAttempt - 1),
|
|
1633
|
+
};
|
|
1634
|
+
|
|
1635
|
+
if (result.thinking !== undefined) routingRecord.thinking = result.thinking;
|
|
1636
|
+
if (result.agentRole !== undefined) routingRecord.agentRole = result.agentRole;
|
|
1637
|
+
if (result.contextTokens !== undefined) routingRecord.contextTokens = result.contextTokens;
|
|
1638
|
+
if (result.tokensIn !== undefined) routingRecord.tokensIn = result.tokensIn;
|
|
1639
|
+
if (result.tokensOut !== undefined) routingRecord.tokensOut = result.tokensOut;
|
|
1640
|
+
if (result.cacheReadTokens !== undefined) routingRecord.cacheReadTokens = result.cacheReadTokens;
|
|
1641
|
+
if (result.cacheWriteTokens !== undefined) routingRecord.cacheWriteTokens = result.cacheWriteTokens;
|
|
1642
|
+
if (result.priceRef !== undefined) routingRecord.priceRef = result.priceRef;
|
|
1643
|
+
if (result.providerMetadata !== undefined) routingRecord.providerMetadata = result.providerMetadata;
|
|
1644
|
+
|
|
1645
|
+
push("routing.attempt.recorded", { routing: parseRoutingRecordV2(routingRecord) });
|
|
1646
|
+
|
|
1647
|
+
const outcome = typeof result.outcome === "string" ? result.outcome : undefined;
|
|
1648
|
+
const known = outcome !== undefined && dispatch.step.outcomes.includes(outcome);
|
|
1649
|
+
if (!known) {
|
|
1650
|
+
push("assignment.result_recorded", {
|
|
1651
|
+
assignmentId: dispatch.assignmentId,
|
|
1652
|
+
resultClass: "outcome_unknown",
|
|
1653
|
+
status: "result_recorded",
|
|
1654
|
+
});
|
|
1655
|
+
push("attempt.status_changed", { attemptId: dispatch.attemptId, status: "terminal" });
|
|
1656
|
+
push("assignment.terminal", { assignmentId: dispatch.assignmentId, outcome: "failed", status: "terminal" });
|
|
1657
|
+
} else {
|
|
1658
|
+
push("assignment.result_recorded", {
|
|
1659
|
+
assignmentId: dispatch.assignmentId,
|
|
1660
|
+
outcome,
|
|
1661
|
+
resultClass: "outcome",
|
|
1662
|
+
status: "result_recorded",
|
|
1663
|
+
});
|
|
1664
|
+
push("attempt.status_changed", { attemptId: dispatch.attemptId, status: "terminal" });
|
|
1665
|
+
push("assignment.terminal", { assignmentId: dispatch.assignmentId, outcome, status: "terminal" });
|
|
1666
|
+
}
|
|
1667
|
+
}
|
|
1668
|
+
context.eventStore.settleCapability(dispatch.attemptId, "settled");
|
|
1669
|
+
for (const event of events) context.eventStore.appendEvent(event);
|
|
1670
|
+
const next = foldStoredVnextRun(context, run);
|
|
1671
|
+
persistVnextRunState(context, run.runId, next, events[events.length - 1]!.sequence);
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
function joinPanel(context: VnextRuntimeContext, panel: PreparedPanel): VnextRunDriveResult {
|
|
1675
|
+
const run = requireRun(context, panel.run.runId);
|
|
1676
|
+
const state = foldStoredVnextRun(context, run);
|
|
1677
|
+
const current = state.currentStep;
|
|
1678
|
+
if (!current || current.stepId !== panel.stepId) {
|
|
1679
|
+
throw runtimeError("run_events_illegal", run.runId, "join attempted without the prepared step");
|
|
1680
|
+
}
|
|
1681
|
+
const joined = vnextJoinAll(panel.step, current.panel);
|
|
1682
|
+
const now = new Date().toISOString();
|
|
1683
|
+
let sequence = context.eventStore.nextSequence(run.runId);
|
|
1684
|
+
let mono = vnextMonotonicNs();
|
|
1685
|
+
const events: VnextRunEvent[] = [];
|
|
1686
|
+
const push = (eventType: string, payload: Record<string, unknown>): void => {
|
|
1687
|
+
events.push({
|
|
1688
|
+
...vnextEventBase(context, run, now, mono),
|
|
1689
|
+
eventId: newVnextEventId(),
|
|
1690
|
+
eventType,
|
|
1691
|
+
sequence: sequence++,
|
|
1692
|
+
payload,
|
|
1693
|
+
});
|
|
1694
|
+
mono = vnextIncrementMonotonicNs(mono);
|
|
1695
|
+
};
|
|
1696
|
+
|
|
1697
|
+
const failRun = (reason: string): VnextRunDriveResult => {
|
|
1698
|
+
push("step.status_changed", { stepId: panel.stepId, status: "failed", previousStatus: "running" });
|
|
1699
|
+
push("run.status_changed", { status: "failed", reason, stepId: panel.stepId });
|
|
1700
|
+
for (const event of events) context.eventStore.appendEvent(event);
|
|
1701
|
+
const next = foldStoredVnextRun(context, run);
|
|
1702
|
+
persistVnextRunState(context, run.runId, next, events[events.length - 1]!.sequence);
|
|
1703
|
+
context.eventStore.updateRunStatus(run.runId, "failed", now);
|
|
1704
|
+
return { state: next };
|
|
1705
|
+
};
|
|
1706
|
+
|
|
1707
|
+
const cancelRun = (): VnextRunDriveResult => {
|
|
1708
|
+
push("step.status_changed", { stepId: panel.stepId, status: "cancelled", previousStatus: "running" });
|
|
1709
|
+
push("run.status_changed", { status: "cancelled", reason: "operator_cancel" });
|
|
1710
|
+
for (const event of events) context.eventStore.appendEvent(event);
|
|
1711
|
+
const next = foldStoredVnextRun(context, run);
|
|
1712
|
+
persistVnextRunState(context, run.runId, next, events[events.length - 1]!.sequence);
|
|
1713
|
+
context.eventStore.updateRunStatus(run.runId, "cancelled", now);
|
|
1714
|
+
return { state: next };
|
|
1715
|
+
};
|
|
1716
|
+
|
|
1717
|
+
const members: Array<{ resultClass: string | undefined; outcome: string | undefined }> = [];
|
|
1718
|
+
for (const assignmentId of current.panel.order) {
|
|
1719
|
+
const assignment = current.panel.assignments[assignmentId];
|
|
1720
|
+
const attemptId = assignment?.currentAttemptId;
|
|
1721
|
+
const attempt = attemptId ? assignment?.attempts[attemptId] : undefined;
|
|
1722
|
+
members.push({ resultClass: attempt?.resultClass, outcome: attempt?.outcome });
|
|
1723
|
+
}
|
|
1724
|
+
const rejected = members.some((member) => member.resultClass === "outcome_unknown" || member.resultClass === "producer_rejected");
|
|
1725
|
+
const declared = members.filter((member) => member.resultClass === "outcome");
|
|
1726
|
+
const declaredConflict = declared.length > 0 && declared.some((member) => member.outcome !== declared[0]?.outcome);
|
|
1727
|
+
|
|
1728
|
+
if (state.cancelRequested || state.status === "cancelling") {
|
|
1729
|
+
if (rejected || declaredConflict) return failRun(rejected ? "outcome_unknown" : "join_conflict");
|
|
1730
|
+
return cancelRun();
|
|
1731
|
+
}
|
|
1732
|
+
if (joined.tag === "rejected") return failRun("outcome_unknown");
|
|
1733
|
+
if (joined.tag === "conflict") return failRun("join_conflict");
|
|
1734
|
+
if (joined.tag === "cancelled") {
|
|
1735
|
+
throw runtimeError("run_events_illegal", run.runId, "cancelled join requires cancellation authority");
|
|
1736
|
+
}
|
|
1737
|
+
if (joined.tag !== "outcome") {
|
|
1738
|
+
throw runtimeError("run_events_illegal", run.runId, "join attempted before the panel is joinable");
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
const outcome = joined.outcome;
|
|
1742
|
+
const selected = panel.step.transitions[outcome];
|
|
1743
|
+
if (!selected) return failRun("outcome_unknown");
|
|
1744
|
+
push("step.outcome_recorded", { stepId: panel.stepId, stepAttempt: panel.stepAttempt, outcome });
|
|
1745
|
+
const stepStatus = outcome === "passed" ? "passed" : outcome === "cancelled" ? "cancelled" : "failed";
|
|
1746
|
+
push("step.status_changed", { stepId: panel.stepId, status: stepStatus, previousStatus: "running" });
|
|
1747
|
+
const budget = transitionBudgetFailure(panel.plan, state, panel.stepId, outcome);
|
|
1748
|
+
if (budget) {
|
|
1749
|
+
push("run.status_changed", { status: "failed", reason: budget, stepId: panel.stepId, outcome });
|
|
1750
|
+
for (const event of events) context.eventStore.appendEvent(event);
|
|
1751
|
+
const next = foldStoredVnextRun(context, run);
|
|
1752
|
+
persistVnextRunState(context, run.runId, next, events[events.length - 1]!.sequence);
|
|
1753
|
+
context.eventStore.updateRunStatus(run.runId, "failed", now);
|
|
1754
|
+
return { state: next };
|
|
1755
|
+
}
|
|
1756
|
+
if (selected.to === "step") {
|
|
1757
|
+
push("step.transitioned", { fromStepId: panel.stepId, toStepId: selected.target, outcome });
|
|
1758
|
+
} else {
|
|
1759
|
+
push("step.transitioned", { fromStepId: panel.stepId, status: selected.terminalStatus, outcome });
|
|
1760
|
+
push("run.status_changed", { status: selected.terminalStatus });
|
|
1761
|
+
}
|
|
1762
|
+
for (const event of events) context.eventStore.appendEvent(event);
|
|
1763
|
+
const next = foldStoredVnextRun(context, run);
|
|
1764
|
+
persistVnextRunState(context, run.runId, next, events[events.length - 1]!.sequence);
|
|
1765
|
+
context.eventStore.updateRunStatus(run.runId, next.status as VnextRunStatus, now);
|
|
1766
|
+
return { state: next };
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
function failBudget(
|
|
1770
|
+
context: VnextRuntimeContext,
|
|
1771
|
+
run: VnextRunRecord,
|
|
1772
|
+
plan: VnextCompiledPlan,
|
|
1773
|
+
state: VnextRunState,
|
|
1774
|
+
reason: string,
|
|
1775
|
+
stepId: string,
|
|
1776
|
+
): { state: VnextRunState } {
|
|
1777
|
+
const now = new Date().toISOString();
|
|
1778
|
+
const sequence = context.eventStore.nextSequence(run.runId);
|
|
1779
|
+
const event: VnextRunEvent = {
|
|
1780
|
+
...vnextEventBase(context, run, now, vnextMonotonicNs()),
|
|
1781
|
+
eventId: newVnextEventId(),
|
|
1782
|
+
eventType: "run.status_changed",
|
|
1783
|
+
sequence,
|
|
1784
|
+
payload: { status: "failed", reason, stepId },
|
|
1785
|
+
};
|
|
1786
|
+
context.eventStore.appendEvent(event);
|
|
1787
|
+
const next = foldStoredVnextRun(context, run);
|
|
1788
|
+
persistVnextRunState(context, run.runId, next, sequence);
|
|
1789
|
+
context.eventStore.updateRunStatus(run.runId, "failed", now);
|
|
1790
|
+
void state;
|
|
1791
|
+
return { state: next };
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1794
|
+
function hardStopUnrecorded(context: VnextRuntimeContext, dispatch: PreparedDispatch, reason: string): void {
|
|
1795
|
+
const run = requireRun(context, dispatch.run.runId);
|
|
1796
|
+
const now = new Date().toISOString();
|
|
1797
|
+
const sequence = context.eventStore.nextSequence(run.runId);
|
|
1798
|
+
const event: VnextRunEvent = {
|
|
1799
|
+
...vnextEventBase(context, run, now, vnextMonotonicNs()),
|
|
1800
|
+
eventId: newVnextEventId(),
|
|
1801
|
+
eventType: "run.status_changed",
|
|
1802
|
+
sequence,
|
|
1803
|
+
payload: { status: "failed", reason, stepId: dispatch.stepId },
|
|
1804
|
+
};
|
|
1805
|
+
context.eventStore.appendEvent(event);
|
|
1806
|
+
context.eventStore.settleCapability(dispatch.attemptId, "revoked");
|
|
1807
|
+
const next = foldStoredVnextRun(context, run);
|
|
1808
|
+
persistVnextRunState(context, run.runId, next, sequence);
|
|
1809
|
+
context.eventStore.updateRunStatus(run.runId, "failed", now);
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
export function vnextTransitionBudgetFailure(plan: VnextCompiledPlan, state: VnextRunState, fromStepId: string, outcome: string): string | undefined {
|
|
1813
|
+
return transitionBudgetFailure(plan, state, fromStepId, outcome);
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
function transitionBudgetFailure(plan: VnextCompiledPlan, state: VnextRunState, fromStepId: string, outcome: string): string | undefined {
|
|
1817
|
+
const selected = plan.steps[fromStepId]?.transitions[outcome];
|
|
1818
|
+
if (!selected) return "outcome_unknown";
|
|
1819
|
+
const edgeKey = `${fromStepId}:${outcome}`;
|
|
1820
|
+
const edgeUsed = (state.edgeTransitions[edgeKey] ?? 0) + 1;
|
|
1821
|
+
if (selected.maxTransitions !== undefined && edgeUsed > selected.maxTransitions) return "budget_edge";
|
|
1822
|
+
if (state.transitionsUsed + 1 > plan.transitionBudget) return "budget_transitions";
|
|
1823
|
+
if (selected.to === "step") {
|
|
1824
|
+
const target = plan.steps[selected.target];
|
|
1825
|
+
const used = state.stepAttempts[selected.target] ?? 0;
|
|
1826
|
+
if (target && used >= target.maxAttempts) return "budget_step_attempts";
|
|
1827
|
+
}
|
|
1828
|
+
return undefined;
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1831
|
+
function unsupportedLimit(envelope: VnextRunPlanEnvelope): VnextRunHandoff | undefined {
|
|
1832
|
+
if (envelope.plan.limits.maxRunDurationMs !== undefined) {
|
|
1833
|
+
return { reason: "limit_unsupported", field: "limits.maxRunDurationMs", detail: "duration budget enforcement is not available in this slice" };
|
|
1834
|
+
}
|
|
1835
|
+
if (envelope.plan.limits.maxAgentTimeMs !== undefined) {
|
|
1836
|
+
return { reason: "limit_unsupported", field: "limits.maxAgentTimeMs", detail: "agent-time budget enforcement is not available in this slice" };
|
|
1837
|
+
}
|
|
1838
|
+
if (envelope.projectLimits.maxRunDurationMs !== undefined) {
|
|
1839
|
+
return { reason: "limit_unsupported", field: "project.limits.maxRunDurationMs", detail: "duration budget enforcement is not available in this slice" };
|
|
1840
|
+
}
|
|
1841
|
+
if (envelope.projectLimits.maxAgentTimeMs !== undefined) {
|
|
1842
|
+
return { reason: "limit_unsupported", field: "project.limits.maxAgentTimeMs", detail: "agent-time budget enforcement is not available in this slice" };
|
|
1843
|
+
}
|
|
1844
|
+
return undefined;
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
function unsupportedStep(plan: VnextCompiledPlan, step: VnextCompiledStep): Omit<VnextRunHandoff, "stepId"> | undefined {
|
|
1848
|
+
if (step.kind === "gate") throw runtimeError("run_plan_corrupt", plan.workflowId, `unsupportedStep called on gate without envelope; use unsupportedGateStep instead`);
|
|
1849
|
+
if (step.kind !== "agent" && step.kind !== "moa" && step.kind !== "approval" && step.kind !== "wait") {
|
|
1850
|
+
return { reason: "step_unsupported", field: "kind", detail: `step kind ${step.kind} is not executed in this slice` };
|
|
1851
|
+
}
|
|
1852
|
+
if (step.assignments.maxAttemptsPerAssignment > 2) {
|
|
1853
|
+
return { reason: "step_unsupported", field: "assignments.maxAttemptsPerAssignment", detail: "only up to two physical attempts are supported in this slice" };
|
|
1854
|
+
}
|
|
1855
|
+
if (step.join.strategy !== "all" && step.join.strategy !== "all-settled") {
|
|
1856
|
+
return { reason: "step_unsupported", field: "join.strategy", detail: "only join all or all-settled is supported" };
|
|
1857
|
+
}
|
|
1858
|
+
if (step.join.strategy === "all" && step.join.minimumPassed !== undefined) {
|
|
1859
|
+
return { reason: "step_unsupported", field: "join.minimumPassed", detail: "minimumPassed requires join strategy all-settled" };
|
|
1860
|
+
}
|
|
1861
|
+
if (step.join.cancelRemaining) {
|
|
1862
|
+
return { reason: "step_unsupported", field: "join.cancelRemaining", detail: "cancelRemaining is not executed in this slice" };
|
|
1863
|
+
}
|
|
1864
|
+
if (step.assignments.distinctBy.some((d) => d !== "provider")) {
|
|
1865
|
+
return { reason: "step_unsupported", field: "assignments.distinctBy", detail: "only distinctBy provider is supported in this slice" };
|
|
1866
|
+
}
|
|
1867
|
+
if (step.assignments.maxWriteRepositories !== undefined && step.assignments.maxWriteRepositories > 1) {
|
|
1868
|
+
return { reason: "step_unsupported", field: "assignments.maxWriteRepositories", detail: "at most one write repository is supported in this slice" };
|
|
1869
|
+
}
|
|
1870
|
+
for (const evidence of step.requiredEvidence) {
|
|
1871
|
+
if (!evidence || !["artifact", "assignment-result", "gate", "approval", "receipt"].includes(evidence.kind)) {
|
|
1872
|
+
return { reason: "step_unsupported", field: "requiredEvidence", detail: `evidence kind '${evidence?.kind}' is not supported in this slice` };
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
if (step.timeoutMs !== undefined && step.timeoutMs <= 0) {
|
|
1876
|
+
return { reason: "step_unsupported", field: "timeoutMs", detail: "timeoutMs must be positive" };
|
|
1877
|
+
}
|
|
1878
|
+
if (step.safeSpeculation) return { reason: "step_unsupported", field: "safeSpeculation", detail: "speculation is not executed in this slice" };
|
|
1879
|
+
if (step.tools) return { reason: "step_unsupported", field: "tools", detail: "tools are not executed in this slice" };
|
|
1880
|
+
if (step.secrets.length > 0) return { reason: "step_unsupported", field: "secrets", detail: "secrets are not executed in this slice" };
|
|
1881
|
+
for (const [repoId, access] of Object.entries(step.repositories)) {
|
|
1882
|
+
if (access !== "read" && access !== "write" && access !== "none") {
|
|
1883
|
+
return { reason: "step_unsupported", field: "repositories", detail: `invalid repository access '${access}' on ${repoId}` };
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
return undefined;
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
function unsupportedGateStep(plan: VnextCompiledPlan, step: VnextCompiledStep & { kind: "gate" }, envelope: VnextRunPlanEnvelope, context?: { projectRoot: string }): Omit<VnextRunHandoff, "stepId"> | undefined {
|
|
1890
|
+
if (!step.gate) return { reason: "step_unsupported", field: "gate", detail: "gate step is missing gate id" };
|
|
1891
|
+
if (envelope.gates.registry === null) {
|
|
1892
|
+
return { reason: "step_unsupported", field: "gate", detail: `step ${step.id} refers to a gate but registry is null` };
|
|
1893
|
+
}
|
|
1894
|
+
const definition = envelope.gates.definitions[step.gate];
|
|
1895
|
+
if (!definition) {
|
|
1896
|
+
return { reason: "step_unsupported", field: "gate", detail: `gate definition ${step.gate} not found in registry` };
|
|
1897
|
+
}
|
|
1898
|
+
if (definition.kind === "reserved") {
|
|
1899
|
+
return { reason: "gate_unsupported", field: "gate", detail: `gate ${step.gate} is reserved and cannot be executed` };
|
|
1900
|
+
}
|
|
1901
|
+
|
|
1902
|
+
// Check assignments constraints
|
|
1903
|
+
if (step.assignments.allowedAgents && step.assignments.allowedAgents.length > 0) {
|
|
1904
|
+
return { reason: "step_unsupported", field: "assignments.allowedAgents", detail: "gate steps do not support allowedAgents" };
|
|
1905
|
+
}
|
|
1906
|
+
if (step.assignments.minimum !== 1 || step.assignments.target !== 1 || step.assignments.maximum !== 1) {
|
|
1907
|
+
return { reason: "step_unsupported", field: "assignments", detail: "gate steps require exactly 1 agent assignment" };
|
|
1908
|
+
}
|
|
1909
|
+
if (step.assignments.maxParallel !== 1) {
|
|
1910
|
+
return { reason: "step_unsupported", field: "assignments.maxParallel", detail: "gate steps require maxParallel of 1" };
|
|
1911
|
+
}
|
|
1912
|
+
if (step.assignments.maxAttemptsPerAssignment !== 1) {
|
|
1913
|
+
return { reason: "step_unsupported", field: "assignments.maxAttemptsPerAssignment", detail: "gate steps require maxAttemptsPerAssignment of 1" };
|
|
1914
|
+
}
|
|
1915
|
+
if (step.assignments.distinctBy && step.assignments.distinctBy.length > 0) {
|
|
1916
|
+
return { reason: "step_unsupported", field: "assignments.distinctBy", detail: "gate steps do not support distinctBy" };
|
|
1917
|
+
}
|
|
1918
|
+
if (step.assignments.maxWriteRepositories) {
|
|
1919
|
+
return { reason: "step_unsupported", field: "assignments.maxWriteRepositories", detail: "gate steps do not support maxWriteRepositories" };
|
|
1920
|
+
}
|
|
1921
|
+
|
|
1922
|
+
// Check join constraints
|
|
1923
|
+
if (step.join.strategy !== "all") {
|
|
1924
|
+
return { reason: "step_unsupported", field: "join", detail: `gate steps require join strategy 'all', not '${step.join.strategy}'` };
|
|
1925
|
+
}
|
|
1926
|
+
if (step.join.minimumPassed !== undefined) {
|
|
1927
|
+
return { reason: "step_unsupported", field: "join", detail: "gate steps do not support minimumPassed" };
|
|
1928
|
+
}
|
|
1929
|
+
if (step.join.cancelRemaining) {
|
|
1930
|
+
return { reason: "step_unsupported", field: "join", detail: "gate steps do not support cancelRemaining" };
|
|
1931
|
+
}
|
|
1932
|
+
|
|
1933
|
+
// Check required evidence constraints
|
|
1934
|
+
for (const evidence of step.requiredEvidence) {
|
|
1935
|
+
if (!evidence || !["gate", "artifact", "receipt", "assignment-result", "approval"].includes(evidence.kind)) {
|
|
1936
|
+
return { reason: "step_unsupported", field: "requiredEvidence", detail: `gate steps do not support evidence kind '${evidence?.kind}'` };
|
|
1937
|
+
}
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
// Check other constraints
|
|
1941
|
+
if (step.safeSpeculation) {
|
|
1942
|
+
return { reason: "step_unsupported", field: "safeSpeculation", detail: "gate steps do not support safeSpeculation" };
|
|
1943
|
+
}
|
|
1944
|
+
if (step.tools) {
|
|
1945
|
+
return { reason: "step_unsupported", field: "tools", detail: "gate steps do not support tools" };
|
|
1946
|
+
}
|
|
1947
|
+
if (step.secrets && step.secrets.length > 0) {
|
|
1948
|
+
return { reason: "step_unsupported", field: "secrets", detail: "gate steps do not support secrets" };
|
|
1949
|
+
}
|
|
1950
|
+
if (step.model) {
|
|
1951
|
+
return { reason: "step_unsupported", field: "model", detail: "gate steps do not support model" };
|
|
1952
|
+
}
|
|
1953
|
+
if (plan.reproOracle?.stageId === step.id) {
|
|
1954
|
+
return { reason: "step_unsupported", field: "reproOracle", detail: "gate steps cannot be repro oracle" };
|
|
1955
|
+
}
|
|
1956
|
+
if (plan.planHash?.stageId === step.id) {
|
|
1957
|
+
return { reason: "step_unsupported", field: "planHash", detail: "gate steps cannot be plan hash oracle" };
|
|
1958
|
+
}
|
|
1959
|
+
|
|
1960
|
+
// Check timeout constraints
|
|
1961
|
+
if (step.timeoutMs !== undefined) {
|
|
1962
|
+
if (definition.kind === "artifacts-exist") {
|
|
1963
|
+
return { reason: "step_unsupported", field: "timeoutMs", detail: "artifacts-exist gates do not support timeoutMs" };
|
|
1964
|
+
}
|
|
1965
|
+
if (definition.kind === "command" && definition.timeoutMs !== undefined && step.timeoutMs < definition.timeoutMs) {
|
|
1966
|
+
return { reason: "step_unsupported", field: "timeoutMs", detail: "command step timeoutMs narrower than the registry timeout is refused" };
|
|
1967
|
+
}
|
|
1968
|
+
}
|
|
1969
|
+
|
|
1970
|
+
// Check repositories constraints
|
|
1971
|
+
const repositories = step.repositories;
|
|
1972
|
+
if (!repositories || Object.keys(repositories).length === 0) {
|
|
1973
|
+
return { reason: "step_unsupported", field: "repositories", detail: "gate steps require repository declarations" };
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
if (definition.kind === "artifacts-exist") {
|
|
1977
|
+
const controlAccess = repositories.control;
|
|
1978
|
+
if (controlAccess !== "read" && controlAccess !== "write") {
|
|
1979
|
+
return { reason: "step_unsupported", field: "repositories", detail: "artifacts-exist gates require control repository access as read or write" };
|
|
1980
|
+
}
|
|
1981
|
+
for (const [repoId, access] of Object.entries(repositories)) {
|
|
1982
|
+
if (repoId !== "control" && access !== "none") {
|
|
1983
|
+
return { reason: "step_unsupported", field: "repositories", detail: "artifacts-exist gates require other repositories to have access 'none'" };
|
|
1984
|
+
}
|
|
1985
|
+
}
|
|
1986
|
+
} else if (definition.kind === "command") {
|
|
1987
|
+
for (const [repoId, access] of Object.entries(repositories)) {
|
|
1988
|
+
if (access !== "read" && access !== "write" && access !== "none") {
|
|
1989
|
+
return { reason: "step_unsupported", field: "repositories", detail: `invalid repository access '${access}' on ${repoId}` };
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
const hasWrite = Object.values(repositories).some((access) => access === "write");
|
|
1993
|
+
if (!hasWrite) {
|
|
1994
|
+
return { reason: "step_unsupported", field: "repositories", detail: "command gates require at least one repository with write access" };
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
// Check outcomes constraints
|
|
1999
|
+
const expectPass = step.expect === "pass";
|
|
2000
|
+
if (expectPass) {
|
|
2001
|
+
if (!step.outcomes.includes("passed")) {
|
|
2002
|
+
return { reason: "gate_outcome_undeclared", field: "outcomes", detail: "step expects pass but missing required outcome passed; spell it as 'passed'" };
|
|
2003
|
+
}
|
|
2004
|
+
if (!step.outcomes.includes("implementation-failure") && !step.outcomes.includes("failed")) {
|
|
2005
|
+
return { reason: "gate_outcome_undeclared", field: "outcomes", detail: "step expects pass but missing required failure outcome; declare implementation-failure or failed" };
|
|
2006
|
+
}
|
|
2007
|
+
} else {
|
|
2008
|
+
if (!step.outcomes.includes("repro-missing")) {
|
|
2009
|
+
return { reason: "gate_outcome_undeclared", field: "outcomes", detail: "step expects fail but missing required outcome repro-missing; spell it as 'repro-missing'" };
|
|
2010
|
+
}
|
|
2011
|
+
if (!step.outcomes.includes("implementation-failure") && !step.outcomes.includes("failed")) {
|
|
2012
|
+
return { reason: "gate_outcome_undeclared", field: "outcomes", detail: "step expects fail but missing required failure outcome; declare implementation-failure or failed" };
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
|
|
2016
|
+
// Check control root match
|
|
2017
|
+
if (context && projectRuntimeKey(context.projectRoot) !== envelope.gates.controlRoot.projectKey) {
|
|
2018
|
+
throw runtimeError("run_owner_mismatch", plan.workflowId, `gate control root does not match project`);
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
// Check expect constraint for artifacts-exist
|
|
2022
|
+
if (definition.kind === "artifacts-exist" && step.expect === "fail") {
|
|
2023
|
+
return { reason: "step_unsupported", field: "expect", detail: "artifacts-exist gates cannot expect fail" };
|
|
2024
|
+
}
|
|
2025
|
+
|
|
2026
|
+
return undefined;
|
|
2027
|
+
}
|
|
2028
|
+
|
|
2029
|
+
function pinnedGatesForCompiledPlan(
|
|
2030
|
+
plan: VnextCompiledPlan,
|
|
2031
|
+
bundle: VnextProjectBundle,
|
|
2032
|
+
projectRoot: string,
|
|
2033
|
+
): VnextPinnedGates {
|
|
2034
|
+
const referenced = new Set<string>();
|
|
2035
|
+
for (const step of Object.values(plan.steps)) {
|
|
2036
|
+
if (step.kind === "gate") referenced.add(step.gate);
|
|
2037
|
+
}
|
|
2038
|
+
const definitions = Object.create(null) as VnextPinnedGates["definitions"] extends Readonly<infer T> ? T : Record<string, never>;
|
|
2039
|
+
const registryValue = bundle.gateRegistry?.value;
|
|
2040
|
+
const gatesObject = registryValue && typeof registryValue.gates === "object" && registryValue.gates && !Array.isArray(registryValue.gates)
|
|
2041
|
+
? registryValue.gates as Record<string, unknown>
|
|
2042
|
+
: {};
|
|
2043
|
+
for (const gateId of referenced) {
|
|
2044
|
+
(definitions as Record<string, unknown>)[gateId] = parseGateDefinition(gatesObject[gateId], gateId, plan.workflowId);
|
|
2045
|
+
}
|
|
2046
|
+
return {
|
|
2047
|
+
registry: referenced.size > 0 && bundle.gateRegistry
|
|
2048
|
+
? { schema: "kxm.gate-registry.v1", hash: gateRegistryHash(bundle.gateRegistry.value) }
|
|
2049
|
+
: bundle.gateRegistry && referenced.size === 0
|
|
2050
|
+
? { schema: "kxm.gate-registry.v1", hash: gateRegistryHash(bundle.gateRegistry.value) }
|
|
2051
|
+
: null,
|
|
2052
|
+
definitions: definitions as VnextPinnedGates["definitions"],
|
|
2053
|
+
controlRoot: { repositoryId: "control", projectKey: projectRuntimeKey(projectRoot) },
|
|
2054
|
+
};
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2057
|
+
export interface GateRecoveryPreflightResult {
|
|
2058
|
+
owned: number;
|
|
2059
|
+
blocking: Array<{ runId: string; attemptId: string }>;
|
|
2060
|
+
}
|
|
2061
|
+
|
|
2062
|
+
export function gateRecoveryPreflight(context: VnextRuntimeContext): GateRecoveryPreflightResult {
|
|
2063
|
+
const attempts = context.eventStore.gateAttemptsForProject(context.projectId);
|
|
2064
|
+
const capabilities = context.eventStore.issuedOrRevokedCapabilities();
|
|
2065
|
+
const seenAttempts = new Set<string>();
|
|
2066
|
+
const blocking: Array<{ runId: string; attemptId: string }> = [];
|
|
2067
|
+
let owned = 0;
|
|
2068
|
+
|
|
2069
|
+
for (const row of attempts) {
|
|
2070
|
+
seenAttempts.add(row.attemptId);
|
|
2071
|
+
const classified = classifyRecoverableGateAttempt(context, row, context.eventStore.capabilityByAttempt(row.attemptId));
|
|
2072
|
+
if (classified === "owned") owned += 1;
|
|
2073
|
+
else if (classified === "blocking") blocking.push({ runId: row.runId, attemptId: row.attemptId });
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
for (const capability of capabilities) {
|
|
2077
|
+
if (seenAttempts.has(capability.attemptId)) continue;
|
|
2078
|
+
const row = context.eventStore.gateAttempt(capability.attemptId);
|
|
2079
|
+
if (capability.producerId === "driver-simulated" || capability.producerId === "pi" || capability.producerId === "oneshot") {
|
|
2080
|
+
if (row) {
|
|
2081
|
+
throw runtimeError(
|
|
2082
|
+
"gate_recovery_corrupt",
|
|
2083
|
+
capability.attemptId,
|
|
2084
|
+
"gate recovery preflight: capability producer contradicts gate attempt",
|
|
2085
|
+
);
|
|
2086
|
+
}
|
|
2087
|
+
continue;
|
|
2088
|
+
}
|
|
2089
|
+
if (capability.producerId !== "kxm-gate") {
|
|
2090
|
+
throw runtimeError(
|
|
2091
|
+
"gate_recovery_corrupt",
|
|
2092
|
+
capability.attemptId,
|
|
2093
|
+
`gate recovery preflight: unknown capability producer ${capability.producerId}`,
|
|
2094
|
+
);
|
|
2095
|
+
}
|
|
2096
|
+
if (!row) {
|
|
2097
|
+
throw runtimeError("gate_recovery_corrupt", capability.attemptId, "gate recovery preflight: missing gate_attempts row");
|
|
2098
|
+
}
|
|
2099
|
+
const classified = classifyRecoverableGateAttempt(context, row, capability);
|
|
2100
|
+
if (classified === "owned") owned += 1;
|
|
2101
|
+
else if (classified === "blocking") blocking.push({ runId: row.runId, attemptId: row.attemptId });
|
|
2102
|
+
}
|
|
2103
|
+
|
|
2104
|
+
return { owned, blocking };
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
function classifyRecoverableGateAttempt(
|
|
2108
|
+
context: VnextRuntimeContext,
|
|
2109
|
+
row: VnextGateAttemptRow,
|
|
2110
|
+
capability: VnextAttemptCapabilityRow | undefined,
|
|
2111
|
+
): "excluded" | "owned" | "blocking" {
|
|
2112
|
+
if (row.producerId !== "kxm-gate") {
|
|
2113
|
+
throw runtimeError(
|
|
2114
|
+
"gate_recovery_corrupt",
|
|
2115
|
+
row.attemptId,
|
|
2116
|
+
"gate recovery preflight: capability producer contradicts gate attempt",
|
|
2117
|
+
);
|
|
2118
|
+
}
|
|
2119
|
+
if (capability && capability.producerId !== "kxm-gate") {
|
|
2120
|
+
throw runtimeError(
|
|
2121
|
+
"gate_recovery_corrupt",
|
|
2122
|
+
row.attemptId,
|
|
2123
|
+
"gate recovery preflight: capability producer contradicts gate attempt",
|
|
2124
|
+
);
|
|
2125
|
+
}
|
|
2126
|
+
if (
|
|
2127
|
+
capability
|
|
2128
|
+
&& (
|
|
2129
|
+
row.runId !== capability.runId
|
|
2130
|
+
|| row.assignmentId !== capability.assignmentId
|
|
2131
|
+
|| row.stepId !== capability.stepId
|
|
2132
|
+
|| row.stepAttempt !== capability.stepAttempt
|
|
2133
|
+
|| row.attemptId !== capability.attemptId
|
|
2134
|
+
)
|
|
2135
|
+
) {
|
|
2136
|
+
throw runtimeError("gate_recovery_corrupt", capability.attemptId, "gate recovery preflight: capability identity does not match gate attempt");
|
|
2137
|
+
}
|
|
2138
|
+
const run = context.eventStore.run(row.runId);
|
|
2139
|
+
if (!run) {
|
|
2140
|
+
throw runtimeError("gate_recovery_corrupt", row.runId, "gate recovery preflight: run does not exist");
|
|
2141
|
+
}
|
|
2142
|
+
if (run.projectId !== context.projectId || row.projectId !== context.projectId) {
|
|
2143
|
+
throw runtimeError("gate_recovery_corrupt", row.runId, "gate recovery preflight: run project mismatch");
|
|
2144
|
+
}
|
|
2145
|
+
let state;
|
|
2146
|
+
try {
|
|
2147
|
+
state = foldStoredVnextRun(context, run);
|
|
2148
|
+
} catch (error) {
|
|
2149
|
+
throw runtimeError(
|
|
2150
|
+
"gate_recovery_corrupt",
|
|
2151
|
+
row.runId,
|
|
2152
|
+
`gate recovery preflight: fold error: ${error instanceof Error ? error.message : String(error)}`,
|
|
2153
|
+
);
|
|
2154
|
+
}
|
|
2155
|
+
const observation = context.eventStore.gateObservationForAttempt(row.attemptId);
|
|
2156
|
+
const evidence = context.eventStore.gateEvidenceForAttempt(row.attemptId);
|
|
2157
|
+
const events = context.eventStore.events(row.runId, 0, 1_000_000);
|
|
2158
|
+
if (isExcludedTerminalProof(state, row, observation, evidence, events)) return "excluded";
|
|
2159
|
+
if (isEvaluatedSettledProof(row, observation, evidence)) {
|
|
2160
|
+
if (
|
|
2161
|
+
capability
|
|
2162
|
+
&& (capability.state === "issued" || capability.state === "revoked")
|
|
2163
|
+
&& state.currentStep?.attemptId === row.attemptId
|
|
2164
|
+
&& state.currentStep.effectState === "settled"
|
|
2165
|
+
) {
|
|
2166
|
+
throw runtimeError(
|
|
2167
|
+
"gate_recovery_corrupt",
|
|
2168
|
+
row.runId,
|
|
2169
|
+
"gate recovery preflight: capability remains issued or revoked after evaluated settlement",
|
|
2170
|
+
);
|
|
2171
|
+
}
|
|
2172
|
+
return "excluded";
|
|
2173
|
+
}
|
|
2174
|
+
if (
|
|
2175
|
+
!capability
|
|
2176
|
+
|| capability.producerId !== "kxm-gate"
|
|
2177
|
+
|| (capability.state !== "issued" && capability.state !== "revoked")
|
|
2178
|
+
) {
|
|
2179
|
+
throw runtimeError(
|
|
2180
|
+
"gate_recovery_corrupt",
|
|
2181
|
+
row.attemptId,
|
|
2182
|
+
"gate recovery preflight: unfinished gate attempt is missing issued or revoked kxm-gate capability",
|
|
2183
|
+
);
|
|
2184
|
+
}
|
|
2185
|
+
const current = state.currentStep;
|
|
2186
|
+
const sameAttempt = current?.attemptId === row.attemptId
|
|
2187
|
+
&& current.assignmentId === row.assignmentId
|
|
2188
|
+
&& current.effectId === row.effectId
|
|
2189
|
+
&& current.stepId === row.stepId;
|
|
2190
|
+
const controller = vnextAttemptController(context.eventStore.path, row.runId, row.attemptId);
|
|
2191
|
+
if (row.gateKind === "command") {
|
|
2192
|
+
const hold = vnextGateHold(context.eventStore.path, row.runId);
|
|
2193
|
+
const admittedToken = vnextAdmittedToken(context.eventStore.path, row.runId);
|
|
2194
|
+
if (hold && hold.attemptId !== row.attemptId) {
|
|
2195
|
+
throw runtimeError("gate_recovery_corrupt", row.attemptId, "gate recovery preflight: gate hold does not match the attempt");
|
|
2196
|
+
}
|
|
2197
|
+
if (hold && admittedToken !== undefined && hold.token !== admittedToken) {
|
|
2198
|
+
throw runtimeError("gate_recovery_corrupt", row.attemptId, "gate recovery preflight: gate hold token does not match admission");
|
|
2199
|
+
}
|
|
2200
|
+
if (hold?.state === "unsettled") return "blocking";
|
|
2201
|
+
const ownedCommand = run.homeRuntimeId === context.homeRuntimeId
|
|
2202
|
+
&& sameAttempt
|
|
2203
|
+
&& hold?.state === "active"
|
|
2204
|
+
&& hold.attemptId === row.attemptId
|
|
2205
|
+
&& hold.token === admittedToken
|
|
2206
|
+
&& controller?.attemptId === row.attemptId
|
|
2207
|
+
&& current?.effectState !== "blocked_uncertain";
|
|
2208
|
+
if (ownedCommand) return "owned";
|
|
2209
|
+
return "blocking";
|
|
2210
|
+
}
|
|
2211
|
+
const ownedHere = run.homeRuntimeId === context.homeRuntimeId
|
|
2212
|
+
&& sameAttempt
|
|
2213
|
+
&& controller?.attemptId === row.attemptId
|
|
2214
|
+
&& current?.effectState !== "blocked_uncertain";
|
|
2215
|
+
if (ownedHere) return "owned";
|
|
2216
|
+
return "blocking";
|
|
2217
|
+
}
|
|
2218
|
+
|
|
2219
|
+
function gateRowIdentityMatches(
|
|
2220
|
+
row: { attemptId: string; assignmentId: string; effectId: string; stepId: string; runId: string },
|
|
2221
|
+
other: { attemptId: string; assignmentId: string; effectId: string; stepId: string; runId: string },
|
|
2222
|
+
): boolean {
|
|
2223
|
+
return other.attemptId === row.attemptId
|
|
2224
|
+
&& other.assignmentId === row.assignmentId
|
|
2225
|
+
&& other.effectId === row.effectId
|
|
2226
|
+
&& other.stepId === row.stepId
|
|
2227
|
+
&& other.runId === row.runId;
|
|
2228
|
+
}
|
|
2229
|
+
|
|
2230
|
+
function isEvaluatedSettledProof(
|
|
2231
|
+
row: { attemptId: string; assignmentId: string; effectId: string; stepId: string; runId: string },
|
|
2232
|
+
observation: VnextGateObservationRow | undefined,
|
|
2233
|
+
evidence: VnextGateEvidenceRow | undefined,
|
|
2234
|
+
): boolean {
|
|
2235
|
+
if (!evidence || !observation) return false;
|
|
2236
|
+
if (!gateRowIdentityMatches(row, observation) || !gateRowIdentityMatches(row, evidence)) return false;
|
|
2237
|
+
if (evidence.observationId !== observation.observationId) return false;
|
|
2238
|
+
return observation.completeness === "complete";
|
|
2239
|
+
}
|
|
2240
|
+
|
|
2241
|
+
function proofOnlySettledEvent(
|
|
2242
|
+
events: readonly VnextRunEvent[],
|
|
2243
|
+
row: { attemptId: string; assignmentId: string; effectId: string; stepId: string; runId: string },
|
|
2244
|
+
): VnextRunEvent | undefined {
|
|
2245
|
+
return events.find((event) => {
|
|
2246
|
+
if (event.eventType !== "effect.settled" || event.runId !== row.runId) return false;
|
|
2247
|
+
if (
|
|
2248
|
+
event.payload.attemptId !== row.attemptId
|
|
2249
|
+
|| event.payload.assignmentId !== row.assignmentId
|
|
2250
|
+
|| event.payload.effectId !== row.effectId
|
|
2251
|
+
|| event.payload.stepId !== row.stepId
|
|
2252
|
+
) {
|
|
2253
|
+
return false;
|
|
2254
|
+
}
|
|
2255
|
+
const refs = event.payload.evidenceRefs;
|
|
2256
|
+
return !Array.isArray(refs) || refs.length === 0;
|
|
2257
|
+
});
|
|
2258
|
+
}
|
|
2259
|
+
|
|
2260
|
+
function isExcludedTerminalProof(
|
|
2261
|
+
state: VnextRunState,
|
|
2262
|
+
row: { attemptId: string; assignmentId: string; effectId: string; stepId: string; runId: string; stepAttempt?: number },
|
|
2263
|
+
observation: VnextGateObservationRow | undefined,
|
|
2264
|
+
evidence: VnextGateEvidenceRow | undefined,
|
|
2265
|
+
events: readonly VnextRunEvent[],
|
|
2266
|
+
): boolean {
|
|
2267
|
+
if (evidence) return false;
|
|
2268
|
+
if (!observation) return false;
|
|
2269
|
+
if (!gateRowIdentityMatches(row, observation)) return false;
|
|
2270
|
+
if (observation.completeness === "incomplete") {
|
|
2271
|
+
if (state.status === "failed" || state.status === "cancelled") return true;
|
|
2272
|
+
const currentAttemptNumber = state.stepAttempts[row.stepId] ?? 0;
|
|
2273
|
+
if (row.stepAttempt !== undefined && currentAttemptNumber > row.stepAttempt) return true;
|
|
2274
|
+
if (state.pendingStepId !== undefined && (state.pendingStepId !== row.stepId || state.currentStep === undefined)) return true;
|
|
2275
|
+
return false;
|
|
2276
|
+
}
|
|
2277
|
+
if (observation.completeness !== "no-start" && observation.completeness !== "complete") return false;
|
|
2278
|
+
if (proofOnlySettledEvent(events, row)) {
|
|
2279
|
+
return observation.completeness === "no-start" || observation.completeness === "complete";
|
|
2280
|
+
}
|
|
2281
|
+
const current = state.currentStep;
|
|
2282
|
+
if (current) {
|
|
2283
|
+
if (
|
|
2284
|
+
current.attemptId !== row.attemptId
|
|
2285
|
+
|| current.assignmentId !== row.assignmentId
|
|
2286
|
+
|| current.effectId !== row.effectId
|
|
2287
|
+
|| current.stepId !== row.stepId
|
|
2288
|
+
|| current.effectState !== "settled-proof"
|
|
2289
|
+
) {
|
|
2290
|
+
return false;
|
|
2291
|
+
}
|
|
2292
|
+
if (observation.completeness === "no-start" && current.observationCompleteness === "no-start") return true;
|
|
2293
|
+
return observation.completeness === "complete"
|
|
2294
|
+
&& current.observationCompleteness === "complete"
|
|
2295
|
+
&& current.status === "cancelled";
|
|
2296
|
+
}
|
|
2297
|
+
if (observation.completeness === "no-start" && (state.status === "failed" || state.status === "cancelled")) return true;
|
|
2298
|
+
return observation.completeness === "complete" && (state.status === "cancelled" || state.status === "failed");
|
|
2299
|
+
}
|
|
2300
|
+
|
|
2301
|
+
function settlePreparedGate(
|
|
2302
|
+
context: VnextRuntimeContext,
|
|
2303
|
+
prepared: VnextPreparedGateDispatch,
|
|
2304
|
+
observation: VnextGateObservationInput,
|
|
2305
|
+
) {
|
|
2306
|
+
const run = requireRun(context, prepared.run.runId);
|
|
2307
|
+
const folded = foldStoredVnextRun(context, run);
|
|
2308
|
+
const current = folded.currentStep;
|
|
2309
|
+
if (
|
|
2310
|
+
!current
|
|
2311
|
+
|| current.stepId !== prepared.stepId
|
|
2312
|
+
|| current.stepAttempt !== prepared.stepAttempt
|
|
2313
|
+
|| current.assignmentId !== prepared.assignmentId
|
|
2314
|
+
|| current.attemptId !== prepared.attemptId
|
|
2315
|
+
|| current.effectId !== prepared.effectId
|
|
2316
|
+
) {
|
|
2317
|
+
throw runtimeError("run_events_illegal", run.runId, "settlement identity does not match the prepared gate attempt");
|
|
2318
|
+
}
|
|
2319
|
+
const capability = context.eventStore.capabilityByAttempt(prepared.attemptId);
|
|
2320
|
+
if (
|
|
2321
|
+
!capability
|
|
2322
|
+
|| capability.runId !== run.runId
|
|
2323
|
+
|| capability.attemptId !== prepared.attemptId
|
|
2324
|
+
|| capability.assignmentId !== prepared.assignmentId
|
|
2325
|
+
|| capability.stepId !== prepared.stepId
|
|
2326
|
+
|| capability.stepAttempt !== prepared.stepAttempt
|
|
2327
|
+
|| capability.producerId !== "kxm-gate"
|
|
2328
|
+
|| (capability.state !== "issued" && capability.state !== "revoked")
|
|
2329
|
+
) {
|
|
2330
|
+
throw runtimeError("run_events_illegal", run.runId, "prepared gate capability is not the issued or revoked attempt");
|
|
2331
|
+
}
|
|
2332
|
+
const cancelled = folded.status === "cancelling" || capability.state === "revoked";
|
|
2333
|
+
if (observation.completeness !== "complete") {
|
|
2334
|
+
throw runtimeError("gate_row_invalid", run.runId, "gate settlement requires a complete observation");
|
|
2335
|
+
}
|
|
2336
|
+
const settled = cancelled
|
|
2337
|
+
? recordGateCancelObservedInTransaction(context, run, prepared.envelope, prepared.stepId, observation)
|
|
2338
|
+
: recordGateSettlementInTransaction(context, run, prepared.envelope, prepared.stepId, observation);
|
|
2339
|
+
vnextGateDispatchSeams.afterObservationWrite?.(prepared);
|
|
2340
|
+
return settled;
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
function requireRun(context: VnextRuntimeContext, runId: string): VnextRunRecord {
|
|
2344
|
+
const run = context.eventStore.run(runId);
|
|
2345
|
+
if (!run) throw runtimeError("run_unknown", runId, "run does not exist in this event store");
|
|
2346
|
+
if (run.projectId !== context.projectId || run.homeRuntimeId !== context.homeRuntimeId) {
|
|
2347
|
+
throw runtimeError("run_owner_mismatch", runId, "run is not owned by this runtime context");
|
|
2348
|
+
}
|
|
2349
|
+
return run;
|
|
2350
|
+
}
|
|
2351
|
+
|
|
2352
|
+
export interface VnextGateRecoveryRequest {
|
|
2353
|
+
action: "retry" | "fail" | "cancel" | "unblock";
|
|
2354
|
+
reason?: string;
|
|
2355
|
+
commandId?: string;
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2358
|
+
export function recoverVnextRun(
|
|
2359
|
+
context: VnextRuntimeContext,
|
|
2360
|
+
runId: string,
|
|
2361
|
+
request: VnextGateRecoveryRequest,
|
|
2362
|
+
): { state: VnextRunState; unblocked: boolean } {
|
|
2363
|
+
return context.eventStore.transaction(() => {
|
|
2364
|
+
const run = requireRun(context, runId);
|
|
2365
|
+
let state = foldStoredVnextRun(context, run);
|
|
2366
|
+
|
|
2367
|
+
if (state.status !== "blocked_uncertain" && state.currentStep?.effectState !== "blocked_uncertain") {
|
|
2368
|
+
throw runtimeError("run_events_illegal", runId, `run ${runId} is not in blocked_uncertain status`);
|
|
2369
|
+
}
|
|
2370
|
+
|
|
2371
|
+
const now = new Date().toISOString();
|
|
2372
|
+
let sequence = context.eventStore.nextSequence(runId);
|
|
2373
|
+
let mono = vnextMonotonicNs();
|
|
2374
|
+
const events: VnextRunEvent[] = [];
|
|
2375
|
+
const push = (eventType: string, payload: Record<string, unknown>): void => {
|
|
2376
|
+
events.push({
|
|
2377
|
+
...vnextEventBase(context, run, now, mono, request.commandId),
|
|
2378
|
+
eventId: newVnextEventId(),
|
|
2379
|
+
eventType,
|
|
2380
|
+
sequence: sequence++,
|
|
2381
|
+
payload,
|
|
2382
|
+
});
|
|
2383
|
+
mono = vnextIncrementMonotonicNs(mono);
|
|
2384
|
+
};
|
|
2385
|
+
|
|
2386
|
+
if (state.status === "running" && state.currentStep?.effectState === "blocked_uncertain") {
|
|
2387
|
+
push("run.status_changed", {
|
|
2388
|
+
status: "blocked_uncertain",
|
|
2389
|
+
previousStatus: "running",
|
|
2390
|
+
reason: request.reason ?? "uncertain gate effect",
|
|
2391
|
+
});
|
|
2392
|
+
for (const ev of events) context.eventStore.appendEvent(ev);
|
|
2393
|
+
events.length = 0;
|
|
2394
|
+
state = foldStoredVnextRun(context, run);
|
|
2395
|
+
context.eventStore.updateRunStatus(runId, "blocked_uncertain", now);
|
|
2396
|
+
}
|
|
2397
|
+
|
|
2398
|
+
const attemptId = state.currentStep?.attemptId;
|
|
2399
|
+
if (attemptId) {
|
|
2400
|
+
resolveVnextGateHold(context.eventStore.path, runId, attemptId);
|
|
2401
|
+
unregisterVnextAttemptController(context.eventStore.path, runId, attemptId);
|
|
2402
|
+
const cap = context.eventStore.capabilityByAttempt(attemptId);
|
|
2403
|
+
if (cap && cap.state !== "settled") {
|
|
2404
|
+
context.eventStore.settleCapability(attemptId, "settled");
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
|
|
2408
|
+
if (request.action === "retry" || request.action === "unblock") {
|
|
2409
|
+
push("run.status_changed", {
|
|
2410
|
+
status: "running",
|
|
2411
|
+
previousStatus: "blocked_uncertain",
|
|
2412
|
+
reason: request.reason ?? "operator_retry",
|
|
2413
|
+
});
|
|
2414
|
+
for (const ev of events) context.eventStore.appendEvent(ev);
|
|
2415
|
+
const nextState = foldStoredVnextRun(context, run);
|
|
2416
|
+
persistVnextRunState(context, runId, nextState, events[events.length - 1]!.sequence);
|
|
2417
|
+
context.eventStore.updateRunStatus(runId, "running", now);
|
|
2418
|
+
return { state: nextState, unblocked: true };
|
|
2419
|
+
}
|
|
2420
|
+
|
|
2421
|
+
if (request.action === "fail") {
|
|
2422
|
+
push("run.status_changed", {
|
|
2423
|
+
status: "failed",
|
|
2424
|
+
previousStatus: "blocked_uncertain",
|
|
2425
|
+
reason: request.reason ?? "operator_fail",
|
|
2426
|
+
});
|
|
2427
|
+
for (const ev of events) context.eventStore.appendEvent(ev);
|
|
2428
|
+
const nextState = foldStoredVnextRun(context, run);
|
|
2429
|
+
persistVnextRunState(context, runId, nextState, events[events.length - 1]!.sequence);
|
|
2430
|
+
context.eventStore.updateRunStatus(runId, "failed", now);
|
|
2431
|
+
return { state: nextState, unblocked: true };
|
|
2432
|
+
}
|
|
2433
|
+
|
|
2434
|
+
if (request.action === "cancel") {
|
|
2435
|
+
push("run.cancel_requested", {
|
|
2436
|
+
actor: { kind: "runtime", id: context.homeRuntimeId },
|
|
2437
|
+
reason: request.reason ?? "operator_cancel",
|
|
2438
|
+
});
|
|
2439
|
+
push("run.status_changed", {
|
|
2440
|
+
status: "cancelling",
|
|
2441
|
+
previousStatus: "blocked_uncertain",
|
|
2442
|
+
reason: request.reason ?? "operator_cancel",
|
|
2443
|
+
});
|
|
2444
|
+
push("run.status_changed", {
|
|
2445
|
+
status: "cancelled",
|
|
2446
|
+
previousStatus: "cancelling",
|
|
2447
|
+
reason: request.reason ?? "operator_cancel",
|
|
2448
|
+
});
|
|
2449
|
+
for (const ev of events) context.eventStore.appendEvent(ev);
|
|
2450
|
+
const nextState = foldStoredVnextRun(context, run);
|
|
2451
|
+
persistVnextRunState(context, runId, nextState, events[events.length - 1]!.sequence);
|
|
2452
|
+
context.eventStore.updateRunStatus(runId, "cancelled", now);
|
|
2453
|
+
return { state: nextState, unblocked: true };
|
|
2454
|
+
}
|
|
2455
|
+
|
|
2456
|
+
throw runtimeError("run_events_illegal", runId, `unsupported recovery action ${(request as { action: string }).action}`);
|
|
2457
|
+
});
|
|
2458
|
+
}
|