@cat-factory/orchestration 0.102.0 → 0.102.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/execution/ExecutionService.d.ts.map +1 -1
- package/dist/modules/execution/ExecutionService.js +34 -21
- package/dist/modules/execution/ExecutionService.js.map +1 -1
- package/dist/modules/execution/RunDispatcher.d.ts +28 -6
- package/dist/modules/execution/RunDispatcher.d.ts.map +1 -1
- package/dist/modules/execution/RunDispatcher.js +217 -127
- package/dist/modules/execution/RunDispatcher.js.map +1 -1
- package/dist/modules/execution/RunStateMachine.d.ts +21 -0
- package/dist/modules/execution/RunStateMachine.d.ts.map +1 -1
- package/dist/modules/execution/RunStateMachine.js +32 -2
- package/dist/modules/execution/RunStateMachine.js.map +1 -1
- package/package.json +8 -8
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ConflictError, DEFAULT_RISK_POLICY, getErrorMessage, getErrorReason, getProvider, INITIATIVE_ANALYST_AGENT_KIND, INITIATIVE_COMMITTER_AGENT_KIND, INITIATIVE_PLANNER_AGENT_KIND, isAsyncAgentExecutor, NotFoundError, parseLocalModelId, recordGateAttempt, registeredGateFactories, registeredStepResolverFactories, requireProvider, sameSubtasks, } from '@cat-factory/kernel';
|
|
1
|
+
import { ConflictError, DEFAULT_RISK_POLICY, getErrorMessage, getErrorReason, getProvider, INITIATIVE_ANALYST_AGENT_KIND, INITIATIVE_COMMITTER_AGENT_KIND, INITIATIVE_PLANNER_AGENT_KIND, isAsyncAgentExecutor, NotFoundError, parseLocalModelId, recordGateAttempt, registeredGateFactories, registeredStepResolverFactories, requireProvider, RunContendedError, sameSubtasks, } from '@cat-factory/kernel';
|
|
2
2
|
import { frontendOriginsForService, parseBlueprintService, parseSpecDoc, } from '@cat-factory/contracts';
|
|
3
3
|
import { blueprintPostOp, commitInitiativeTracker, hasTrait, isCompanionKind, isContainerBackedCompanion, INTERVIEW_GATE_TRAIT, moduleSlug, runRepoOps, specPostOp, TASK_ESTIMATOR_AGENT_KIND, } from '@cat-factory/agents';
|
|
4
4
|
import { DEPLOYER_AGENT_KIND, isDeployStep } from '@cat-factory/integrations';
|
|
@@ -178,6 +178,27 @@ export class RunDispatcher {
|
|
|
178
178
|
this.resolveRiskPolicy = deps.resolveRiskPolicy;
|
|
179
179
|
this.modelIdIsMetered = deps.modelIdIsMetered;
|
|
180
180
|
}
|
|
181
|
+
/**
|
|
182
|
+
* Run a durable-driver entry point, turning a lost optimistic-concurrency race into a
|
|
183
|
+
* re-drive. A driver write ({@link RunStateMachine.casPersist}) throws {@link RunContendedError}
|
|
184
|
+
* when a concurrent human action moved the row or a `cancel`/`stopRun` removed/terminated it;
|
|
185
|
+
* we swallow that and return `{ kind: 'continue' }` so the durable loop re-enters
|
|
186
|
+
* `advanceInstance`, reloads FRESH state, and either re-applies the mechanical step on the
|
|
187
|
+
* winning snapshot or no-ops on a gone/terminal run — never clobbering the winner or
|
|
188
|
+
* resurrecting a cancelled run (race-audit 2.2 driver-half / 2.3). This MUST run inside each
|
|
189
|
+
* entry point (ahead of the drivers' generic `catch`→`failRun` and Cloudflare's `step.do`
|
|
190
|
+
* retry); every other error propagates so real failures still fail the run.
|
|
191
|
+
*/
|
|
192
|
+
async redriveOnContention(run) {
|
|
193
|
+
try {
|
|
194
|
+
return await run();
|
|
195
|
+
}
|
|
196
|
+
catch (error) {
|
|
197
|
+
if (error instanceof RunContendedError)
|
|
198
|
+
return { kind: 'continue' };
|
|
199
|
+
throw error;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
181
202
|
/**
|
|
182
203
|
* The generic container/inline-agent step — the lowest-priority StepHandler, claiming
|
|
183
204
|
* every step no more-specific handler did (coder, architect, spec-writer, merger,
|
|
@@ -220,7 +241,7 @@ export class RunDispatcher {
|
|
|
220
241
|
// Surface the block's ephemeral environment (if any) alongside the cold-boot
|
|
221
242
|
// phase, so a run's details show the env spinning up next to the container.
|
|
222
243
|
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step);
|
|
223
|
-
await this.
|
|
244
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
224
245
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
225
246
|
let handle;
|
|
226
247
|
try {
|
|
@@ -233,7 +254,7 @@ export class RunDispatcher {
|
|
|
233
254
|
// start") so the run details say the container never started — not a generic
|
|
234
255
|
// "run failed". A dispatch-time eviction still routes to the evicted framing.
|
|
235
256
|
step.container = { status: 'errored' };
|
|
236
|
-
await this.
|
|
257
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
237
258
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
238
259
|
const message = getErrorMessage(error);
|
|
239
260
|
const evicted = isContainerEvictionError(message);
|
|
@@ -261,7 +282,7 @@ export class RunDispatcher {
|
|
|
261
282
|
// The dispatch returned, so the container is up and the job is accepted; the
|
|
262
283
|
// live phase + the container id/url arrive on the first poll.
|
|
263
284
|
step.container = { status: 'up' };
|
|
264
|
-
await this.
|
|
285
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
265
286
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
266
287
|
}
|
|
267
288
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
@@ -272,7 +293,7 @@ export class RunDispatcher {
|
|
|
272
293
|
const previewModel = await this.previewStepModel(context);
|
|
273
294
|
if (previewModel && previewModel !== step.model) {
|
|
274
295
|
step.model = previewModel;
|
|
275
|
-
await this.
|
|
296
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
276
297
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
277
298
|
}
|
|
278
299
|
const result = await this.runAgent(context, options);
|
|
@@ -397,6 +418,9 @@ export class RunDispatcher {
|
|
|
397
418
|
* simply lets the driver advance the now-current step.
|
|
398
419
|
*/
|
|
399
420
|
async pollAgentJob(workspaceId, executionId) {
|
|
421
|
+
return this.redriveOnContention(() => this.pollAgentJobInner(workspaceId, executionId));
|
|
422
|
+
}
|
|
423
|
+
async pollAgentJobInner(workspaceId, executionId) {
|
|
400
424
|
const instance = await this.executionRepository.get(workspaceId, executionId);
|
|
401
425
|
if (!instance || (instance.status !== 'running' && instance.status !== 'paused')) {
|
|
402
426
|
return { kind: 'noop' };
|
|
@@ -430,34 +454,60 @@ export class RunDispatcher {
|
|
|
430
454
|
agentKind: step.agentKind,
|
|
431
455
|
});
|
|
432
456
|
if (update.state === 'running') {
|
|
433
|
-
// A successful poll proves the container is up, so the cold-boot phase is
|
|
434
|
-
//
|
|
435
|
-
//
|
|
436
|
-
//
|
|
437
|
-
//
|
|
438
|
-
|
|
439
|
-
//
|
|
440
|
-
//
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
457
|
+
// A successful poll proves the container is up, so the cold-boot phase is over
|
|
458
|
+
// (defensive: a replay may have left the flag set). Surface live subtask progress
|
|
459
|
+
// (e.g. 3/8 todos done) without advancing the step. Only persist + emit when something
|
|
460
|
+
// actually changed so an idle poll doesn't churn storage or the event stream.
|
|
461
|
+
//
|
|
462
|
+
// Fold the poll's delta onto a target instance, returning whether anything changed. The
|
|
463
|
+
// captured `update` is the source, so re-applying on a freshly-reloaded instance is
|
|
464
|
+
// idempotent for the set-to-latest folds AND correct for the APPEND fold: the harness
|
|
465
|
+
// follow-up buffer is drain-on-read (`runner.ts`), so those streamed items exist ONLY on
|
|
466
|
+
// this `update` — abort-and-redrive would lose them, whereas re-applying under
|
|
467
|
+
// `mutateInstance` (which reloads a clean base each attempt) preserves them.
|
|
468
|
+
const applyRunningFold = async (target) => {
|
|
469
|
+
const s = target.steps[target.currentStep];
|
|
470
|
+
// The step advanced (or the job was superseded) under a concurrent write — nothing to fold.
|
|
471
|
+
if (!s || s.jobId !== step.jobId)
|
|
472
|
+
return false;
|
|
473
|
+
let changed = false;
|
|
474
|
+
if (this.applyContainerRunning(s, update))
|
|
475
|
+
changed = true;
|
|
476
|
+
if (this.applySubtaskProgress(s, update.subtasks))
|
|
477
|
+
changed = true;
|
|
478
|
+
// The transport reports WHICH backend served the job on the first poll (native host
|
|
479
|
+
// process vs. sandboxed container) — record it in the run diagnostics.
|
|
480
|
+
if (this.recordBackendDiagnostics(target, update.backend))
|
|
481
|
+
changed = true;
|
|
482
|
+
// Append any forward-looking items the Coder streamed since the last poll so the
|
|
483
|
+
// Follow-up companion lights up + accrues items LIVE while the container still runs.
|
|
484
|
+
if (this.appendStreamedFollowUps(s, update.followUps))
|
|
485
|
+
changed = true;
|
|
486
|
+
// Refresh the env projection so its status transitions (provisioning→ready→
|
|
487
|
+
// expired/torn_down) and any error stay live in the run details during the run.
|
|
488
|
+
if (await this.attachEnvironmentProjection(workspaceId, target.blockId, s))
|
|
489
|
+
changed = true;
|
|
490
|
+
return changed;
|
|
491
|
+
};
|
|
492
|
+
// Cheap pre-check against the loaded snapshot: skip the write entirely on an idle poll
|
|
493
|
+
// (the common case). The mutation is discarded — the authoritative write re-applies the
|
|
494
|
+
// same fold on fresh state under CAS below.
|
|
495
|
+
if (await applyRunningFold(instance)) {
|
|
496
|
+
try {
|
|
497
|
+
const persisted = await this.runStateMachine.mutateInstance(workspaceId, executionId, async (fresh) => {
|
|
498
|
+
await applyRunningFold(fresh);
|
|
499
|
+
});
|
|
500
|
+
await this.runStateMachine.emitInstance(workspaceId, persisted);
|
|
501
|
+
}
|
|
502
|
+
catch (error) {
|
|
503
|
+
// The run was cancelled/removed mid-poll (`NotFoundError`) or stayed hot-contended
|
|
504
|
+
// past the retry budget (`ConflictError`) — re-drive on fresh state rather than
|
|
505
|
+
// failing the run; the next entry no-ops on a gone/terminal run.
|
|
506
|
+
if (error instanceof NotFoundError || error instanceof ConflictError) {
|
|
507
|
+
throw new RunContendedError(executionId);
|
|
508
|
+
}
|
|
509
|
+
throw error;
|
|
510
|
+
}
|
|
461
511
|
}
|
|
462
512
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
463
513
|
}
|
|
@@ -545,7 +595,7 @@ export class RunDispatcher {
|
|
|
545
595
|
step.subtasks = undefined;
|
|
546
596
|
if (step.gate)
|
|
547
597
|
step.gate.phase = 'checking';
|
|
548
|
-
await this.
|
|
598
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
549
599
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
550
600
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
551
601
|
}
|
|
@@ -708,7 +758,7 @@ export class RunDispatcher {
|
|
|
708
758
|
// The container vanished and a fresh one is about to boot for the re-dispatch, so the
|
|
709
759
|
// details show it spinning up again rather than a stale "up".
|
|
710
760
|
step.container = { status: 'starting' };
|
|
711
|
-
await this.
|
|
761
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
712
762
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
713
763
|
return { kind: 'continue' };
|
|
714
764
|
}
|
|
@@ -734,7 +784,7 @@ export class RunDispatcher {
|
|
|
734
784
|
*/
|
|
735
785
|
async markContainerErrored(workspaceId, instance, step) {
|
|
736
786
|
step.container = { ...step.container, status: 'errored' };
|
|
737
|
-
await this.
|
|
787
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
738
788
|
}
|
|
739
789
|
/**
|
|
740
790
|
* Re-run a polling gate step's precheck from the durable driver's `awaiting_gate`
|
|
@@ -745,6 +795,9 @@ export class RunDispatcher {
|
|
|
745
795
|
* step is a gate actively in its `checking` phase.
|
|
746
796
|
*/
|
|
747
797
|
async pollGate(workspaceId, executionId) {
|
|
798
|
+
return this.redriveOnContention(() => this.pollGateInner(workspaceId, executionId));
|
|
799
|
+
}
|
|
800
|
+
async pollGateInner(workspaceId, executionId) {
|
|
748
801
|
const instance = await this.executionRepository.get(workspaceId, executionId);
|
|
749
802
|
if (!instance || (instance.status !== 'running' && instance.status !== 'paused')) {
|
|
750
803
|
return { kind: 'noop' };
|
|
@@ -778,6 +831,9 @@ export class RunDispatcher {
|
|
|
778
831
|
* gate; it returns a `job_failed` the driver funnels through its single `failRun`).
|
|
779
832
|
*/
|
|
780
833
|
async resolveGatePollExhaustion(workspaceId, executionId) {
|
|
834
|
+
return this.redriveOnContention(() => this.resolveGatePollExhaustionInner(workspaceId, executionId));
|
|
835
|
+
}
|
|
836
|
+
async resolveGatePollExhaustionInner(workspaceId, executionId) {
|
|
781
837
|
const instance = await this.executionRepository.get(workspaceId, executionId);
|
|
782
838
|
if (!instance || (instance.status !== 'running' && instance.status !== 'paused')) {
|
|
783
839
|
return { kind: 'noop' };
|
|
@@ -798,7 +854,7 @@ export class RunDispatcher {
|
|
|
798
854
|
if (step && gate && gate.pollExhaustion === 'rearm') {
|
|
799
855
|
if (step.gate)
|
|
800
856
|
step.gate.phase = 'checking';
|
|
801
|
-
await this.
|
|
857
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
802
858
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
803
859
|
}
|
|
804
860
|
if (!step || !gate || gate.pollExhaustion !== 'pass') {
|
|
@@ -817,7 +873,7 @@ export class RunDispatcher {
|
|
|
817
873
|
if (!windowElapsed) {
|
|
818
874
|
if (step.gate)
|
|
819
875
|
step.gate.phase = 'checking';
|
|
820
|
-
await this.
|
|
876
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
821
877
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
822
878
|
}
|
|
823
879
|
}
|
|
@@ -894,7 +950,7 @@ export class RunDispatcher {
|
|
|
894
950
|
if (isFinalStep) {
|
|
895
951
|
instance.status = 'done';
|
|
896
952
|
await this.runStateMachine.finalizeBlock(workspaceId, instance, undefined);
|
|
897
|
-
await this.
|
|
953
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
898
954
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
899
955
|
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
900
956
|
return { kind: 'done' };
|
|
@@ -904,7 +960,7 @@ export class RunDispatcher {
|
|
|
904
960
|
if (next)
|
|
905
961
|
this.stepGraph.startStep(next);
|
|
906
962
|
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
907
|
-
await this.
|
|
963
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
908
964
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
909
965
|
return { kind: 'continue' };
|
|
910
966
|
}
|
|
@@ -942,7 +998,7 @@ export class RunDispatcher {
|
|
|
942
998
|
this.stepGraph.pauseStepForInput(step);
|
|
943
999
|
instance.status = 'blocked';
|
|
944
1000
|
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
945
|
-
await this.
|
|
1001
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
946
1002
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
947
1003
|
return { kind: 'awaiting_decision', decisionId: step.decision.id };
|
|
948
1004
|
}
|
|
@@ -1066,7 +1122,7 @@ export class RunDispatcher {
|
|
|
1066
1122
|
this.stepGraph.pauseStepForInput(step);
|
|
1067
1123
|
instance.status = 'blocked';
|
|
1068
1124
|
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
1069
|
-
await this.
|
|
1125
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1070
1126
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1071
1127
|
return { kind: 'awaiting_decision', decisionId: step.approval.id };
|
|
1072
1128
|
}
|
|
@@ -1114,7 +1170,7 @@ export class RunDispatcher {
|
|
|
1114
1170
|
// real merge via the step-completion resolver registry (so a trailing
|
|
1115
1171
|
// post-release-health gate doesn't disable auto-merge). Nothing merge-specific here.
|
|
1116
1172
|
await this.runStateMachine.finalizeBlock(workspaceId, instance, result.confidence);
|
|
1117
|
-
await this.
|
|
1173
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1118
1174
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1119
1175
|
// The run is finished: reclaim its per-run container now instead of letting it
|
|
1120
1176
|
// idle out its sleepAfter window (~10 min of billed-but-useless compute). All
|
|
@@ -1137,7 +1193,7 @@ export class RunDispatcher {
|
|
|
1137
1193
|
else {
|
|
1138
1194
|
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
1139
1195
|
}
|
|
1140
|
-
await this.
|
|
1196
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1141
1197
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1142
1198
|
return { kind: 'continue' };
|
|
1143
1199
|
}
|
|
@@ -1227,7 +1283,7 @@ export class RunDispatcher {
|
|
|
1227
1283
|
// mid-fan-out resumes at the first un-settled frame rather than re-doing an already-settled
|
|
1228
1284
|
// one (which, on the synchronous REST path, would re-hit the provider — no idempotency guard
|
|
1229
1285
|
// there, unlike the deterministic async job ref).
|
|
1230
|
-
await this.
|
|
1286
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1231
1287
|
return this.advanceDeployerFrames(workspaceId, instance, step, block, isFinalStep, targets);
|
|
1232
1288
|
}
|
|
1233
1289
|
// Start provisioning the next frame: a raw-manifest config provisions SYNCHRONOUSLY over REST
|
|
@@ -1265,7 +1321,7 @@ export class RunDispatcher {
|
|
|
1265
1321
|
// mid-flight). Absent for the undeclared legacy path, which re-resolution handles harmlessly.
|
|
1266
1322
|
step.deployProvisioning = next.provisioning;
|
|
1267
1323
|
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step, next.frameId);
|
|
1268
|
-
await this.
|
|
1324
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1269
1325
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1270
1326
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
1271
1327
|
}
|
|
@@ -1352,7 +1408,7 @@ export class RunDispatcher {
|
|
|
1352
1408
|
step.deployEnvs = { ...done, [target.frameId]: { status: 'ready', url: handle.url } };
|
|
1353
1409
|
// Persist this frame's TERMINAL outcome BEFORE provisioning the next frame (see the infraless
|
|
1354
1410
|
// branch) so a crash/replay resumes at the first un-settled frame, not re-provisioning this one.
|
|
1355
|
-
await this.
|
|
1411
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1356
1412
|
return this.advanceDeployerFrames(workspaceId, instance, step, block, isFinalStep, targets);
|
|
1357
1413
|
}
|
|
1358
1414
|
/**
|
|
@@ -1373,7 +1429,7 @@ export class RunDispatcher {
|
|
|
1373
1429
|
}
|
|
1374
1430
|
// A PEER failure is non-terminal — persist it BEFORE moving to the next frame so a replay
|
|
1375
1431
|
// doesn't re-attempt this failed peer (same rationale as the ready/infraless settle paths).
|
|
1376
|
-
await this.
|
|
1432
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1377
1433
|
return this.advanceDeployerFrames(workspaceId, instance, step, block, isFinalStep, targets);
|
|
1378
1434
|
}
|
|
1379
1435
|
/**
|
|
@@ -1405,7 +1461,7 @@ export class RunDispatcher {
|
|
|
1405
1461
|
changed = true;
|
|
1406
1462
|
}
|
|
1407
1463
|
if (changed) {
|
|
1408
|
-
await this.
|
|
1464
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1409
1465
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1410
1466
|
}
|
|
1411
1467
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
@@ -1618,7 +1674,7 @@ export class RunDispatcher {
|
|
|
1618
1674
|
// single-frame deploy that is the own env; for a failed involved-service env it surfaces the
|
|
1619
1675
|
// peer's error rather than a sibling's healthy env.
|
|
1620
1676
|
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step, frameId);
|
|
1621
|
-
await this.
|
|
1677
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1622
1678
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1623
1679
|
return {
|
|
1624
1680
|
kind: 'job_failed',
|
|
@@ -1741,7 +1797,7 @@ export class RunDispatcher {
|
|
|
1741
1797
|
progress: 1,
|
|
1742
1798
|
});
|
|
1743
1799
|
}
|
|
1744
|
-
await this.
|
|
1800
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1745
1801
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1746
1802
|
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
1747
1803
|
return { kind: 'done' };
|
|
@@ -2459,7 +2515,7 @@ export class RunDispatcher {
|
|
|
2459
2515
|
if (step.gate.pendingFix && isAsyncAgentExecutor(this.agentExecutor)) {
|
|
2460
2516
|
const fix = step.gate.pendingFix;
|
|
2461
2517
|
step.gate.pendingFix = null;
|
|
2462
|
-
await this.
|
|
2518
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
2463
2519
|
return this.dispatchGateHelper(workspaceId, instance, step, block, isFinalStep, gate, fix.instructions);
|
|
2464
2520
|
}
|
|
2465
2521
|
// A time-windowed gate (post-release-health) marks when it began watching, on first
|
|
@@ -2494,7 +2550,7 @@ export class RunDispatcher {
|
|
|
2494
2550
|
if (probe.status === 'pending') {
|
|
2495
2551
|
// Keep polling. Persist the head sha + phase so the board can reflect it.
|
|
2496
2552
|
step.gate.phase = 'checking';
|
|
2497
|
-
await this.
|
|
2553
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
2498
2554
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2499
2555
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
2500
2556
|
}
|
|
@@ -2569,7 +2625,7 @@ export class RunDispatcher {
|
|
|
2569
2625
|
// `pendingFix.instructions`), which both arrive here as `failureSummary`.
|
|
2570
2626
|
lastDispatchedInstructions: failureSummary ?? step.gate?.lastDispatchedInstructions ?? null,
|
|
2571
2627
|
};
|
|
2572
|
-
await this.
|
|
2628
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
2573
2629
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2574
2630
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
2575
2631
|
}
|
|
@@ -2621,7 +2677,7 @@ export class RunDispatcher {
|
|
|
2621
2677
|
if (shouldLoopCoder(state)) {
|
|
2622
2678
|
this.loopCoderForFollowUps(instance, step);
|
|
2623
2679
|
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2624
|
-
await this.
|
|
2680
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
2625
2681
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2626
2682
|
return { kind: 'continue' };
|
|
2627
2683
|
}
|
|
@@ -2714,7 +2770,9 @@ export class RunDispatcher {
|
|
|
2714
2770
|
}
|
|
2715
2771
|
/** File a `follow_up` item as a tracker issue (GitHub / Jira), recording the ticket ref. */
|
|
2716
2772
|
async fileFollowUp(workspaceId, executionId, itemId) {
|
|
2717
|
-
|
|
2773
|
+
// Snapshot read for validation + frame resolution before the non-idempotent ticket
|
|
2774
|
+
// creation; the item's ticket refs are then recorded under CAS in applyFollowUpDecision.
|
|
2775
|
+
const { instance, item } = await this.loadFollowUpItem(workspaceId, executionId, itemId);
|
|
2718
2776
|
if (item.kind !== 'follow_up') {
|
|
2719
2777
|
throw new ConflictError('Only follow-up items can be filed as issues');
|
|
2720
2778
|
}
|
|
@@ -2738,91 +2796,123 @@ export class RunDispatcher {
|
|
|
2738
2796
|
if (!ticket) {
|
|
2739
2797
|
throw new ConflictError('No issue tracker is configured for this workspace');
|
|
2740
2798
|
}
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2799
|
+
return this.applyFollowUpDecision(workspaceId, executionId, itemId, (target) => {
|
|
2800
|
+
// Re-validated on the fresh snapshot inside the CAS (the ticket was already created above).
|
|
2801
|
+
if (target.kind !== 'follow_up') {
|
|
2802
|
+
throw new ConflictError('Only follow-up items can be filed as issues');
|
|
2803
|
+
}
|
|
2804
|
+
target.status = 'filed';
|
|
2805
|
+
target.ticketExternalId = ticket.externalId;
|
|
2806
|
+
target.ticketUrl = ticket.url;
|
|
2807
|
+
target.updatedAt = this.clock.now();
|
|
2808
|
+
});
|
|
2747
2809
|
}
|
|
2748
2810
|
/** Queue a `follow_up` item to send back to the Coder on its next pass. */
|
|
2749
2811
|
async queueFollowUp(workspaceId, executionId, itemId) {
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
return step.followUps;
|
|
2812
|
+
return this.applyFollowUpDecision(workspaceId, executionId, itemId, (item) => {
|
|
2813
|
+
if (item.kind !== 'follow_up') {
|
|
2814
|
+
throw new ConflictError('Only follow-up items can be sent back to the Coder');
|
|
2815
|
+
}
|
|
2816
|
+
item.status = 'queued';
|
|
2817
|
+
item.sentToCoder = false;
|
|
2818
|
+
item.updatedAt = this.clock.now();
|
|
2819
|
+
});
|
|
2759
2820
|
}
|
|
2760
2821
|
/** Answer a `question` item; the answer is folded into the Coder's next pass. */
|
|
2761
2822
|
async answerFollowUp(workspaceId, executionId, itemId, answer) {
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
return step.followUps;
|
|
2823
|
+
return this.applyFollowUpDecision(workspaceId, executionId, itemId, (item) => {
|
|
2824
|
+
if (item.kind !== 'question') {
|
|
2825
|
+
throw new ConflictError('Only question items can be answered');
|
|
2826
|
+
}
|
|
2827
|
+
item.status = 'answered';
|
|
2828
|
+
item.answer = answer;
|
|
2829
|
+
item.sentToCoder = false;
|
|
2830
|
+
item.updatedAt = this.clock.now();
|
|
2831
|
+
});
|
|
2772
2832
|
}
|
|
2773
2833
|
/** Dismiss a follow-up / question item without acting on it. */
|
|
2774
2834
|
async dismissFollowUp(workspaceId, executionId, itemId) {
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
return step.followUps;
|
|
2835
|
+
return this.applyFollowUpDecision(workspaceId, executionId, itemId, (item) => {
|
|
2836
|
+
item.status = 'dismissed';
|
|
2837
|
+
item.updatedAt = this.clock.now();
|
|
2838
|
+
});
|
|
2780
2839
|
}
|
|
2781
2840
|
/**
|
|
2782
|
-
*
|
|
2783
|
-
* every item
|
|
2784
|
-
* items
|
|
2785
|
-
*
|
|
2841
|
+
* Apply a human follow-up item decision and, when the run is PARKED on this step's follow-up
|
|
2842
|
+
* gate with every item now decided, drive it forward — loop the Coder for the queued/answered
|
|
2843
|
+
* items, hand off to a co-located approval gate, or advance past the gate — all under
|
|
2844
|
+
* OPTIMISTIC CONCURRENCY. A follow-up decision can race the driver's running-poll fold (which
|
|
2845
|
+
* appends newly-streamed items) or another decision on a sibling item, so it RE-READS +
|
|
2846
|
+
* RE-APPLIES on a lost CAS race instead of clobbering — the human-action dual of the driver's
|
|
2847
|
+
* abort-and-redrive (race-audit 2.2). The item mutation + the in-memory gate transition run
|
|
2848
|
+
* INSIDE the CAS callback (idempotent, re-runnable on reload); the non-idempotent side effects
|
|
2849
|
+
* (notifications, `signalDecision`, emit) run once AFTER, on the winning snapshot. `decide`
|
|
2850
|
+
* validates + mutates the item, throwing a `ConflictError`/`NotFoundError` that propagates
|
|
2851
|
+
* immediately (a domain error is not retried).
|
|
2786
2852
|
*/
|
|
2787
|
-
async
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
const
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2853
|
+
async applyFollowUpDecision(workspaceId, executionId, itemId, decide) {
|
|
2854
|
+
// Captured inside the (re-runnable) callback for the last winning attempt; the
|
|
2855
|
+
// non-idempotent side effects below act on them.
|
|
2856
|
+
let outcome = 'record';
|
|
2857
|
+
let index = -1;
|
|
2858
|
+
let loopDecisionId;
|
|
2859
|
+
const persisted = await this.runStateMachine.mutateInstance(workspaceId, executionId, (fresh) => {
|
|
2860
|
+
outcome = 'record';
|
|
2861
|
+
loopDecisionId = undefined;
|
|
2862
|
+
index = fresh.steps.findIndex((s) => s.followUps?.enabled && s.followUps.items.some((i) => i.id === itemId));
|
|
2863
|
+
if (index < 0)
|
|
2864
|
+
throw new NotFoundError('Follow-up item', itemId);
|
|
2865
|
+
const step = fresh.steps[index];
|
|
2866
|
+
decide(step.followUps.items.find((i) => i.id === itemId));
|
|
2867
|
+
const parkedHere = fresh.status === 'blocked' &&
|
|
2868
|
+
step.approval?.status === 'pending' &&
|
|
2869
|
+
fresh.currentStep === index;
|
|
2870
|
+
// Still collecting decisions (or the run isn't parked on this gate): only record it.
|
|
2871
|
+
if (!parkedHere || hasPendingFollowUps(step.followUps))
|
|
2872
|
+
return;
|
|
2873
|
+
// Every item decided and the run is parked here: loop the Coder for the send-back items,
|
|
2874
|
+
// hand off to a co-located approval gate, or advance past the gate.
|
|
2875
|
+
if (shouldLoopCoder(step.followUps)) {
|
|
2876
|
+
loopDecisionId = step.approval.id;
|
|
2877
|
+
this.loopCoderForFollowUps(fresh, step);
|
|
2878
|
+
outcome = 'loop';
|
|
2879
|
+
return;
|
|
2880
|
+
}
|
|
2881
|
+
const isFinalStep = index === fresh.steps.length - 1;
|
|
2882
|
+
if (step.requiresApproval && !isFinalStep && step.approval?.status === 'pending') {
|
|
2883
|
+
// The follow-up park reused `step.approval`; advancing here would silently SKIP the
|
|
2884
|
+
// approval. Refresh the proposal and hand off to the standard approval gate (the
|
|
2885
|
+
// follow-up card is cleared + the "waiting for input" card re-raised below), preserving
|
|
2886
|
+
// the follow-up-before-approval ordering recordStepResult established across the park.
|
|
2887
|
+
step.approval = { ...step.approval, proposal: step.output ?? '' };
|
|
2888
|
+
outcome = 'handoff';
|
|
2889
|
+
return;
|
|
2890
|
+
}
|
|
2891
|
+
this.runStateMachine.advanceRunPastGate(fresh, index);
|
|
2892
|
+
outcome = 'advance';
|
|
2893
|
+
});
|
|
2894
|
+
// Non-idempotent side effects on the winning snapshot (the CAS write is the source of truth,
|
|
2895
|
+
// so nothing re-persists here). The settled paths first clear the follow-up waiting card.
|
|
2896
|
+
if (outcome === 'record') {
|
|
2897
|
+
await this.runStateMachine.emitInstance(workspaceId, persisted);
|
|
2808
2898
|
}
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2899
|
+
else {
|
|
2900
|
+
await this.runStateMachine.clearWaitingNotification(workspaceId, persisted);
|
|
2901
|
+
if (outcome === 'loop') {
|
|
2902
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, persisted, 'in_progress');
|
|
2903
|
+
await this.workRunner.signalDecision(workspaceId, persisted.id, loopDecisionId, 'approved');
|
|
2904
|
+
await this.runStateMachine.emitInstance(workspaceId, persisted);
|
|
2905
|
+
}
|
|
2906
|
+
else if (outcome === 'handoff') {
|
|
2907
|
+
await this.runStateMachine.ensureWaitingNotification(workspaceId, persisted);
|
|
2908
|
+
await this.runStateMachine.emitInstance(workspaceId, persisted);
|
|
2909
|
+
}
|
|
2910
|
+
else {
|
|
2911
|
+
// advance: block writes + `signalDecision('approved')` + emit — the approveStep template.
|
|
2912
|
+
await this.runStateMachine.settleAdvancedGate(workspaceId, persisted, index);
|
|
2913
|
+
}
|
|
2824
2914
|
}
|
|
2825
|
-
|
|
2915
|
+
return persisted.steps[index].followUps;
|
|
2826
2916
|
}
|
|
2827
2917
|
/** Provision inputs (`{{input.*}}`) derived from the block under deployment. */
|
|
2828
2918
|
deployInputs(block) {
|