@cat-factory/orchestration 0.101.0 → 0.102.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/container.d.ts +6 -6
- package/dist/container.d.ts.map +1 -1
- package/dist/container.js +8 -8
- package/dist/container.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/modules/board/BoardService.d.ts.map +1 -1
- package/dist/modules/board/BoardService.js +13 -3
- package/dist/modules/board/BoardService.js.map +1 -1
- package/dist/modules/execution/ExecutionService.d.ts +7 -7
- package/dist/modules/execution/ExecutionService.d.ts.map +1 -1
- package/dist/modules/execution/ExecutionService.js +66 -41
- package/dist/modules/execution/ExecutionService.js.map +1 -1
- package/dist/modules/execution/HumanTestController.d.ts +1 -1
- package/dist/modules/execution/HumanTestController.d.ts.map +1 -1
- package/dist/modules/execution/HumanTestController.js +1 -1
- package/dist/modules/execution/HumanTestController.js.map +1 -1
- package/dist/modules/execution/MergeResolver.d.ts +1 -1
- package/dist/modules/execution/MergeResolver.d.ts.map +1 -1
- package/dist/modules/execution/MergeResolver.js +1 -1
- package/dist/modules/execution/MergeResolver.js.map +1 -1
- package/dist/modules/execution/ReviewGateController.d.ts +27 -8
- package/dist/modules/execution/ReviewGateController.d.ts.map +1 -1
- package/dist/modules/execution/ReviewGateController.js +59 -9
- package/dist/modules/execution/ReviewGateController.js.map +1 -1
- package/dist/modules/execution/RunDispatcher.d.ts +33 -11
- package/dist/modules/execution/RunDispatcher.d.ts.map +1 -1
- package/dist/modules/execution/RunDispatcher.js +229 -134
- package/dist/modules/execution/RunDispatcher.js.map +1 -1
- package/dist/modules/execution/RunStateMachine.d.ts +21 -0
- package/dist/modules/execution/RunStateMachine.d.ts.map +1 -1
- package/dist/modules/execution/RunStateMachine.js +32 -2
- package/dist/modules/execution/RunStateMachine.js.map +1 -1
- package/dist/modules/execution/TesterController.d.ts +1 -1
- package/dist/modules/execution/TesterController.d.ts.map +1 -1
- package/dist/modules/execution/TesterController.js +4 -4
- package/dist/modules/execution/TesterController.js.map +1 -1
- package/dist/modules/execution/VisualConfirmationController.d.ts +1 -1
- package/dist/modules/execution/VisualConfirmationController.d.ts.map +1 -1
- package/dist/modules/execution/VisualConfirmationController.js +1 -1
- package/dist/modules/execution/VisualConfirmationController.js.map +1 -1
- package/dist/modules/execution/gate-window-facades.d.ts +2 -2
- package/dist/modules/execution/gate-window-facades.d.ts.map +1 -1
- package/dist/modules/execution/gate-window-facades.js +2 -2
- package/dist/modules/execution/gate-window-facades.js.map +1 -1
- package/dist/modules/merge/{MergePresetService.d.ts → RiskPolicyService.d.ts} +12 -12
- package/dist/modules/merge/RiskPolicyService.d.ts.map +1 -0
- package/dist/modules/merge/{MergePresetService.js → RiskPolicyService.js} +10 -10
- package/dist/modules/merge/RiskPolicyService.js.map +1 -0
- package/dist/modules/pipelines/PipelineService.d.ts.map +1 -1
- package/dist/modules/pipelines/PipelineService.js +13 -0
- package/dist/modules/pipelines/PipelineService.js.map +1 -1
- package/dist/modules/requirements/RequirementReviewService.d.ts +29 -7
- package/dist/modules/requirements/RequirementReviewService.d.ts.map +1 -1
- package/dist/modules/requirements/RequirementReviewService.js +61 -13
- package/dist/modules/requirements/RequirementReviewService.js.map +1 -1
- package/dist/modules/requirements/requirements.logic.d.ts.map +1 -1
- package/dist/modules/requirements/requirements.logic.js +10 -1
- package/dist/modules/requirements/requirements.logic.js.map +1 -1
- package/package.json +11 -11
- package/dist/modules/merge/MergePresetService.d.ts.map +0 -1
- package/dist/modules/merge/MergePresetService.js.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ConflictError,
|
|
1
|
+
import { ConflictError, DEFAULT_RISK_POLICY, getErrorMessage, getErrorReason, getProvider, INITIATIVE_ANALYST_AGENT_KIND, INITIATIVE_COMMITTER_AGENT_KIND, INITIATIVE_PLANNER_AGENT_KIND, isAsyncAgentExecutor, NotFoundError, parseLocalModelId, recordGateAttempt, registeredGateFactories, registeredStepResolverFactories, requireProvider, RunContendedError, sameSubtasks, } from '@cat-factory/kernel';
|
|
2
2
|
import { frontendOriginsForService, parseBlueprintService, parseSpecDoc, } from '@cat-factory/contracts';
|
|
3
3
|
import { blueprintPostOp, commitInitiativeTracker, hasTrait, isCompanionKind, isContainerBackedCompanion, INTERVIEW_GATE_TRAIT, moduleSlug, runRepoOps, specPostOp, TASK_ESTIMATOR_AGENT_KIND, } from '@cat-factory/agents';
|
|
4
4
|
import { DEPLOYER_AGENT_KIND, isDeployStep } from '@cat-factory/integrations';
|
|
@@ -92,7 +92,7 @@ function buildPeerEnvUrls(targets, done) {
|
|
|
92
92
|
* a fat per-callback bag. It composes the existing collaborators ({@link RunStateMachine} /
|
|
93
93
|
* {@link StepGraph} / the five gate controllers / {@link MergeResolver}); the merge/auto-start
|
|
94
94
|
* subgraph deliberately STAYS on the engine, reached only through the injected
|
|
95
|
-
* `
|
|
95
|
+
* `resolveRiskPolicy` callback + the {@link MergeResolver} (which itself closes over the
|
|
96
96
|
* engine's `finalizeMerge`). `ExecutionService.stepInstance` / `pollAgentJob` / `pollGate`
|
|
97
97
|
* delegate here; no behaviour changes in the move.
|
|
98
98
|
*/
|
|
@@ -131,7 +131,7 @@ export class RunDispatcher {
|
|
|
131
131
|
initiativeService;
|
|
132
132
|
resolveRunRepoContext;
|
|
133
133
|
resolveProviderCapabilities;
|
|
134
|
-
|
|
134
|
+
resolveRiskPolicy;
|
|
135
135
|
modelIdIsMetered;
|
|
136
136
|
/** Lazily-built polling-gate registry, keyed by `agentKind`. See {@link gateFor}. */
|
|
137
137
|
gateRegistryCache;
|
|
@@ -175,9 +175,30 @@ export class RunDispatcher {
|
|
|
175
175
|
this.initiativeService = deps.initiativeService;
|
|
176
176
|
this.resolveRunRepoContext = deps.resolveRunRepoContext;
|
|
177
177
|
this.resolveProviderCapabilities = deps.resolveProviderCapabilities;
|
|
178
|
-
this.
|
|
178
|
+
this.resolveRiskPolicy = deps.resolveRiskPolicy;
|
|
179
179
|
this.modelIdIsMetered = deps.modelIdIsMetered;
|
|
180
180
|
}
|
|
181
|
+
/**
|
|
182
|
+
* Run a durable-driver entry point, turning a lost optimistic-concurrency race into a
|
|
183
|
+
* re-drive. A driver write ({@link RunStateMachine.casPersist}) throws {@link RunContendedError}
|
|
184
|
+
* when a concurrent human action moved the row or a `cancel`/`stopRun` removed/terminated it;
|
|
185
|
+
* we swallow that and return `{ kind: 'continue' }` so the durable loop re-enters
|
|
186
|
+
* `advanceInstance`, reloads FRESH state, and either re-applies the mechanical step on the
|
|
187
|
+
* winning snapshot or no-ops on a gone/terminal run — never clobbering the winner or
|
|
188
|
+
* resurrecting a cancelled run (race-audit 2.2 driver-half / 2.3). This MUST run inside each
|
|
189
|
+
* entry point (ahead of the drivers' generic `catch`→`failRun` and Cloudflare's `step.do`
|
|
190
|
+
* retry); every other error propagates so real failures still fail the run.
|
|
191
|
+
*/
|
|
192
|
+
async redriveOnContention(run) {
|
|
193
|
+
try {
|
|
194
|
+
return await run();
|
|
195
|
+
}
|
|
196
|
+
catch (error) {
|
|
197
|
+
if (error instanceof RunContendedError)
|
|
198
|
+
return { kind: 'continue' };
|
|
199
|
+
throw error;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
181
202
|
/**
|
|
182
203
|
* The generic container/inline-agent step — the lowest-priority StepHandler, claiming
|
|
183
204
|
* every step no more-specific handler did (coder, architect, spec-writer, merger,
|
|
@@ -220,7 +241,7 @@ export class RunDispatcher {
|
|
|
220
241
|
// Surface the block's ephemeral environment (if any) alongside the cold-boot
|
|
221
242
|
// phase, so a run's details show the env spinning up next to the container.
|
|
222
243
|
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step);
|
|
223
|
-
await this.
|
|
244
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
224
245
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
225
246
|
let handle;
|
|
226
247
|
try {
|
|
@@ -233,7 +254,7 @@ export class RunDispatcher {
|
|
|
233
254
|
// start") so the run details say the container never started — not a generic
|
|
234
255
|
// "run failed". A dispatch-time eviction still routes to the evicted framing.
|
|
235
256
|
step.container = { status: 'errored' };
|
|
236
|
-
await this.
|
|
257
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
237
258
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
238
259
|
const message = getErrorMessage(error);
|
|
239
260
|
const evicted = isContainerEvictionError(message);
|
|
@@ -261,7 +282,7 @@ export class RunDispatcher {
|
|
|
261
282
|
// The dispatch returned, so the container is up and the job is accepted; the
|
|
262
283
|
// live phase + the container id/url arrive on the first poll.
|
|
263
284
|
step.container = { status: 'up' };
|
|
264
|
-
await this.
|
|
285
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
265
286
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
266
287
|
}
|
|
267
288
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
@@ -272,7 +293,7 @@ export class RunDispatcher {
|
|
|
272
293
|
const previewModel = await this.previewStepModel(context);
|
|
273
294
|
if (previewModel && previewModel !== step.model) {
|
|
274
295
|
step.model = previewModel;
|
|
275
|
-
await this.
|
|
296
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
276
297
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
277
298
|
}
|
|
278
299
|
const result = await this.runAgent(context, options);
|
|
@@ -397,6 +418,9 @@ export class RunDispatcher {
|
|
|
397
418
|
* simply lets the driver advance the now-current step.
|
|
398
419
|
*/
|
|
399
420
|
async pollAgentJob(workspaceId, executionId) {
|
|
421
|
+
return this.redriveOnContention(() => this.pollAgentJobInner(workspaceId, executionId));
|
|
422
|
+
}
|
|
423
|
+
async pollAgentJobInner(workspaceId, executionId) {
|
|
400
424
|
const instance = await this.executionRepository.get(workspaceId, executionId);
|
|
401
425
|
if (!instance || (instance.status !== 'running' && instance.status !== 'paused')) {
|
|
402
426
|
return { kind: 'noop' };
|
|
@@ -430,34 +454,60 @@ export class RunDispatcher {
|
|
|
430
454
|
agentKind: step.agentKind,
|
|
431
455
|
});
|
|
432
456
|
if (update.state === 'running') {
|
|
433
|
-
// A successful poll proves the container is up, so the cold-boot phase is
|
|
434
|
-
//
|
|
435
|
-
//
|
|
436
|
-
//
|
|
437
|
-
//
|
|
438
|
-
|
|
439
|
-
//
|
|
440
|
-
//
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
457
|
+
// A successful poll proves the container is up, so the cold-boot phase is over
|
|
458
|
+
// (defensive: a replay may have left the flag set). Surface live subtask progress
|
|
459
|
+
// (e.g. 3/8 todos done) without advancing the step. Only persist + emit when something
|
|
460
|
+
// actually changed so an idle poll doesn't churn storage or the event stream.
|
|
461
|
+
//
|
|
462
|
+
// Fold the poll's delta onto a target instance, returning whether anything changed. The
|
|
463
|
+
// captured `update` is the source, so re-applying on a freshly-reloaded instance is
|
|
464
|
+
// idempotent for the set-to-latest folds AND correct for the APPEND fold: the harness
|
|
465
|
+
// follow-up buffer is drain-on-read (`runner.ts`), so those streamed items exist ONLY on
|
|
466
|
+
// this `update` — abort-and-redrive would lose them, whereas re-applying under
|
|
467
|
+
// `mutateInstance` (which reloads a clean base each attempt) preserves them.
|
|
468
|
+
const applyRunningFold = async (target) => {
|
|
469
|
+
const s = target.steps[target.currentStep];
|
|
470
|
+
// The step advanced (or the job was superseded) under a concurrent write — nothing to fold.
|
|
471
|
+
if (!s || s.jobId !== step.jobId)
|
|
472
|
+
return false;
|
|
473
|
+
let changed = false;
|
|
474
|
+
if (this.applyContainerRunning(s, update))
|
|
475
|
+
changed = true;
|
|
476
|
+
if (this.applySubtaskProgress(s, update.subtasks))
|
|
477
|
+
changed = true;
|
|
478
|
+
// The transport reports WHICH backend served the job on the first poll (native host
|
|
479
|
+
// process vs. sandboxed container) — record it in the run diagnostics.
|
|
480
|
+
if (this.recordBackendDiagnostics(target, update.backend))
|
|
481
|
+
changed = true;
|
|
482
|
+
// Append any forward-looking items the Coder streamed since the last poll so the
|
|
483
|
+
// Follow-up companion lights up + accrues items LIVE while the container still runs.
|
|
484
|
+
if (this.appendStreamedFollowUps(s, update.followUps))
|
|
485
|
+
changed = true;
|
|
486
|
+
// Refresh the env projection so its status transitions (provisioning→ready→
|
|
487
|
+
// expired/torn_down) and any error stay live in the run details during the run.
|
|
488
|
+
if (await this.attachEnvironmentProjection(workspaceId, target.blockId, s))
|
|
489
|
+
changed = true;
|
|
490
|
+
return changed;
|
|
491
|
+
};
|
|
492
|
+
// Cheap pre-check against the loaded snapshot: skip the write entirely on an idle poll
|
|
493
|
+
// (the common case). The mutation is discarded — the authoritative write re-applies the
|
|
494
|
+
// same fold on fresh state under CAS below.
|
|
495
|
+
if (await applyRunningFold(instance)) {
|
|
496
|
+
try {
|
|
497
|
+
const persisted = await this.runStateMachine.mutateInstance(workspaceId, executionId, async (fresh) => {
|
|
498
|
+
await applyRunningFold(fresh);
|
|
499
|
+
});
|
|
500
|
+
await this.runStateMachine.emitInstance(workspaceId, persisted);
|
|
501
|
+
}
|
|
502
|
+
catch (error) {
|
|
503
|
+
// The run was cancelled/removed mid-poll (`NotFoundError`) or stayed hot-contended
|
|
504
|
+
// past the retry budget (`ConflictError`) — re-drive on fresh state rather than
|
|
505
|
+
// failing the run; the next entry no-ops on a gone/terminal run.
|
|
506
|
+
if (error instanceof NotFoundError || error instanceof ConflictError) {
|
|
507
|
+
throw new RunContendedError(executionId);
|
|
508
|
+
}
|
|
509
|
+
throw error;
|
|
510
|
+
}
|
|
461
511
|
}
|
|
462
512
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
463
513
|
}
|
|
@@ -545,7 +595,7 @@ export class RunDispatcher {
|
|
|
545
595
|
step.subtasks = undefined;
|
|
546
596
|
if (step.gate)
|
|
547
597
|
step.gate.phase = 'checking';
|
|
548
|
-
await this.
|
|
598
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
549
599
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
550
600
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
551
601
|
}
|
|
@@ -708,7 +758,7 @@ export class RunDispatcher {
|
|
|
708
758
|
// The container vanished and a fresh one is about to boot for the re-dispatch, so the
|
|
709
759
|
// details show it spinning up again rather than a stale "up".
|
|
710
760
|
step.container = { status: 'starting' };
|
|
711
|
-
await this.
|
|
761
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
712
762
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
713
763
|
return { kind: 'continue' };
|
|
714
764
|
}
|
|
@@ -734,7 +784,7 @@ export class RunDispatcher {
|
|
|
734
784
|
*/
|
|
735
785
|
async markContainerErrored(workspaceId, instance, step) {
|
|
736
786
|
step.container = { ...step.container, status: 'errored' };
|
|
737
|
-
await this.
|
|
787
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
738
788
|
}
|
|
739
789
|
/**
|
|
740
790
|
* Re-run a polling gate step's precheck from the durable driver's `awaiting_gate`
|
|
@@ -745,6 +795,9 @@ export class RunDispatcher {
|
|
|
745
795
|
* step is a gate actively in its `checking` phase.
|
|
746
796
|
*/
|
|
747
797
|
async pollGate(workspaceId, executionId) {
|
|
798
|
+
return this.redriveOnContention(() => this.pollGateInner(workspaceId, executionId));
|
|
799
|
+
}
|
|
800
|
+
async pollGateInner(workspaceId, executionId) {
|
|
748
801
|
const instance = await this.executionRepository.get(workspaceId, executionId);
|
|
749
802
|
if (!instance || (instance.status !== 'running' && instance.status !== 'paused')) {
|
|
750
803
|
return { kind: 'noop' };
|
|
@@ -778,6 +831,9 @@ export class RunDispatcher {
|
|
|
778
831
|
* gate; it returns a `job_failed` the driver funnels through its single `failRun`).
|
|
779
832
|
*/
|
|
780
833
|
async resolveGatePollExhaustion(workspaceId, executionId) {
|
|
834
|
+
return this.redriveOnContention(() => this.resolveGatePollExhaustionInner(workspaceId, executionId));
|
|
835
|
+
}
|
|
836
|
+
async resolveGatePollExhaustionInner(workspaceId, executionId) {
|
|
781
837
|
const instance = await this.executionRepository.get(workspaceId, executionId);
|
|
782
838
|
if (!instance || (instance.status !== 'running' && instance.status !== 'paused')) {
|
|
783
839
|
return { kind: 'noop' };
|
|
@@ -798,7 +854,7 @@ export class RunDispatcher {
|
|
|
798
854
|
if (step && gate && gate.pollExhaustion === 'rearm') {
|
|
799
855
|
if (step.gate)
|
|
800
856
|
step.gate.phase = 'checking';
|
|
801
|
-
await this.
|
|
857
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
802
858
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
803
859
|
}
|
|
804
860
|
if (!step || !gate || gate.pollExhaustion !== 'pass') {
|
|
@@ -817,7 +873,7 @@ export class RunDispatcher {
|
|
|
817
873
|
if (!windowElapsed) {
|
|
818
874
|
if (step.gate)
|
|
819
875
|
step.gate.phase = 'checking';
|
|
820
|
-
await this.
|
|
876
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
821
877
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
822
878
|
}
|
|
823
879
|
}
|
|
@@ -894,7 +950,7 @@ export class RunDispatcher {
|
|
|
894
950
|
if (isFinalStep) {
|
|
895
951
|
instance.status = 'done';
|
|
896
952
|
await this.runStateMachine.finalizeBlock(workspaceId, instance, undefined);
|
|
897
|
-
await this.
|
|
953
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
898
954
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
899
955
|
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
900
956
|
return { kind: 'done' };
|
|
@@ -904,7 +960,7 @@ export class RunDispatcher {
|
|
|
904
960
|
if (next)
|
|
905
961
|
this.stepGraph.startStep(next);
|
|
906
962
|
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
907
|
-
await this.
|
|
963
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
908
964
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
909
965
|
return { kind: 'continue' };
|
|
910
966
|
}
|
|
@@ -915,8 +971,11 @@ export class RunDispatcher {
|
|
|
915
971
|
* Shared by the inline path and the async-job poll path.
|
|
916
972
|
*/
|
|
917
973
|
async recordStepResult(workspaceId, instance, step, isFinalStep, result) {
|
|
918
|
-
// Meter the LLM call
|
|
919
|
-
//
|
|
974
|
+
// Meter the LLM call into the usage ledger. Recorded whether the step completed or
|
|
975
|
+
// raised a decision — both consumed tokens. A subscription-harness result is tagged
|
|
976
|
+
// `'subscription'` so it's counted for the usage report but EXCLUDED from the budget
|
|
977
|
+
// rollups (a flat-rate quota plan costs nothing per token); an inline metered call
|
|
978
|
+
// defaults to `'metered'` and is summed by the spend gate as before.
|
|
920
979
|
if (result.usage) {
|
|
921
980
|
await this.spend.record({
|
|
922
981
|
workspaceId,
|
|
@@ -924,6 +983,8 @@ export class RunDispatcher {
|
|
|
924
983
|
agentKind: step.agentKind,
|
|
925
984
|
model: result.model ?? 'unknown',
|
|
926
985
|
usage: result.usage,
|
|
986
|
+
billing: result.usageBilling ?? 'metered',
|
|
987
|
+
vendor: result.usageVendor ?? null,
|
|
927
988
|
});
|
|
928
989
|
}
|
|
929
990
|
// The agent asked for a human decision and this step hasn't resolved one yet.
|
|
@@ -937,7 +998,7 @@ export class RunDispatcher {
|
|
|
937
998
|
this.stepGraph.pauseStepForInput(step);
|
|
938
999
|
instance.status = 'blocked';
|
|
939
1000
|
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
940
|
-
await this.
|
|
1001
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
941
1002
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
942
1003
|
return { kind: 'awaiting_decision', decisionId: step.decision.id };
|
|
943
1004
|
}
|
|
@@ -1061,7 +1122,7 @@ export class RunDispatcher {
|
|
|
1061
1122
|
this.stepGraph.pauseStepForInput(step);
|
|
1062
1123
|
instance.status = 'blocked';
|
|
1063
1124
|
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
1064
|
-
await this.
|
|
1125
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1065
1126
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1066
1127
|
return { kind: 'awaiting_decision', decisionId: step.approval.id };
|
|
1067
1128
|
}
|
|
@@ -1109,7 +1170,7 @@ export class RunDispatcher {
|
|
|
1109
1170
|
// real merge via the step-completion resolver registry (so a trailing
|
|
1110
1171
|
// post-release-health gate doesn't disable auto-merge). Nothing merge-specific here.
|
|
1111
1172
|
await this.runStateMachine.finalizeBlock(workspaceId, instance, result.confidence);
|
|
1112
|
-
await this.
|
|
1173
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1113
1174
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1114
1175
|
// The run is finished: reclaim its per-run container now instead of letting it
|
|
1115
1176
|
// idle out its sleepAfter window (~10 min of billed-but-useless compute). All
|
|
@@ -1132,7 +1193,7 @@ export class RunDispatcher {
|
|
|
1132
1193
|
else {
|
|
1133
1194
|
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
1134
1195
|
}
|
|
1135
|
-
await this.
|
|
1196
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1136
1197
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1137
1198
|
return { kind: 'continue' };
|
|
1138
1199
|
}
|
|
@@ -1222,7 +1283,7 @@ export class RunDispatcher {
|
|
|
1222
1283
|
// mid-fan-out resumes at the first un-settled frame rather than re-doing an already-settled
|
|
1223
1284
|
// one (which, on the synchronous REST path, would re-hit the provider — no idempotency guard
|
|
1224
1285
|
// there, unlike the deterministic async job ref).
|
|
1225
|
-
await this.
|
|
1286
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1226
1287
|
return this.advanceDeployerFrames(workspaceId, instance, step, block, isFinalStep, targets);
|
|
1227
1288
|
}
|
|
1228
1289
|
// Start provisioning the next frame: a raw-manifest config provisions SYNCHRONOUSLY over REST
|
|
@@ -1260,7 +1321,7 @@ export class RunDispatcher {
|
|
|
1260
1321
|
// mid-flight). Absent for the undeclared legacy path, which re-resolution handles harmlessly.
|
|
1261
1322
|
step.deployProvisioning = next.provisioning;
|
|
1262
1323
|
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step, next.frameId);
|
|
1263
|
-
await this.
|
|
1324
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1264
1325
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1265
1326
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
1266
1327
|
}
|
|
@@ -1347,7 +1408,7 @@ export class RunDispatcher {
|
|
|
1347
1408
|
step.deployEnvs = { ...done, [target.frameId]: { status: 'ready', url: handle.url } };
|
|
1348
1409
|
// Persist this frame's TERMINAL outcome BEFORE provisioning the next frame (see the infraless
|
|
1349
1410
|
// branch) so a crash/replay resumes at the first un-settled frame, not re-provisioning this one.
|
|
1350
|
-
await this.
|
|
1411
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1351
1412
|
return this.advanceDeployerFrames(workspaceId, instance, step, block, isFinalStep, targets);
|
|
1352
1413
|
}
|
|
1353
1414
|
/**
|
|
@@ -1368,7 +1429,7 @@ export class RunDispatcher {
|
|
|
1368
1429
|
}
|
|
1369
1430
|
// A PEER failure is non-terminal — persist it BEFORE moving to the next frame so a replay
|
|
1370
1431
|
// doesn't re-attempt this failed peer (same rationale as the ready/infraless settle paths).
|
|
1371
|
-
await this.
|
|
1432
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1372
1433
|
return this.advanceDeployerFrames(workspaceId, instance, step, block, isFinalStep, targets);
|
|
1373
1434
|
}
|
|
1374
1435
|
/**
|
|
@@ -1400,7 +1461,7 @@ export class RunDispatcher {
|
|
|
1400
1461
|
changed = true;
|
|
1401
1462
|
}
|
|
1402
1463
|
if (changed) {
|
|
1403
|
-
await this.
|
|
1464
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1404
1465
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1405
1466
|
}
|
|
1406
1467
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
@@ -1613,7 +1674,7 @@ export class RunDispatcher {
|
|
|
1613
1674
|
// single-frame deploy that is the own env; for a failed involved-service env it surfaces the
|
|
1614
1675
|
// peer's error rather than a sibling's healthy env.
|
|
1615
1676
|
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step, frameId);
|
|
1616
|
-
await this.
|
|
1677
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1617
1678
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1618
1679
|
return {
|
|
1619
1680
|
kind: 'job_failed',
|
|
@@ -1736,7 +1797,7 @@ export class RunDispatcher {
|
|
|
1736
1797
|
progress: 1,
|
|
1737
1798
|
});
|
|
1738
1799
|
}
|
|
1739
|
-
await this.
|
|
1800
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1740
1801
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1741
1802
|
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
1742
1803
|
return { kind: 'done' };
|
|
@@ -2432,7 +2493,7 @@ export class RunDispatcher {
|
|
|
2432
2493
|
// Initialise the gate's state on first entry, resolving the attempt budget from the
|
|
2433
2494
|
// task's merge preset (stable across polls once set).
|
|
2434
2495
|
if (!step.gate) {
|
|
2435
|
-
const preset = await this.
|
|
2496
|
+
const preset = await this.resolveRiskPolicy(workspaceId, block);
|
|
2436
2497
|
step.gate = {
|
|
2437
2498
|
phase: 'checking',
|
|
2438
2499
|
attempts: 0,
|
|
@@ -2454,7 +2515,7 @@ export class RunDispatcher {
|
|
|
2454
2515
|
if (step.gate.pendingFix && isAsyncAgentExecutor(this.agentExecutor)) {
|
|
2455
2516
|
const fix = step.gate.pendingFix;
|
|
2456
2517
|
step.gate.pendingFix = null;
|
|
2457
|
-
await this.
|
|
2518
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
2458
2519
|
return this.dispatchGateHelper(workspaceId, instance, step, block, isFinalStep, gate, fix.instructions);
|
|
2459
2520
|
}
|
|
2460
2521
|
// A time-windowed gate (post-release-health) marks when it began watching, on first
|
|
@@ -2489,7 +2550,7 @@ export class RunDispatcher {
|
|
|
2489
2550
|
if (probe.status === 'pending') {
|
|
2490
2551
|
// Keep polling. Persist the head sha + phase so the board can reflect it.
|
|
2491
2552
|
step.gate.phase = 'checking';
|
|
2492
|
-
await this.
|
|
2553
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
2493
2554
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2494
2555
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
2495
2556
|
}
|
|
@@ -2555,7 +2616,7 @@ export class RunDispatcher {
|
|
|
2555
2616
|
...step.gate,
|
|
2556
2617
|
phase: 'working',
|
|
2557
2618
|
attempts: (step.gate?.attempts ?? 0) + 1,
|
|
2558
|
-
maxAttempts: step.gate?.maxAttempts ??
|
|
2619
|
+
maxAttempts: step.gate?.maxAttempts ?? DEFAULT_RISK_POLICY.ciMaxAttempts,
|
|
2559
2620
|
headSha: step.gate?.headSha ?? null,
|
|
2560
2621
|
// Stash the instructions this helper was handed (the failing-check summary / conflict
|
|
2561
2622
|
// reason / human fix prompt) so the attempt recorded at its completion can show WHAT
|
|
@@ -2564,7 +2625,7 @@ export class RunDispatcher {
|
|
|
2564
2625
|
// `pendingFix.instructions`), which both arrive here as `failureSummary`.
|
|
2565
2626
|
lastDispatchedInstructions: failureSummary ?? step.gate?.lastDispatchedInstructions ?? null,
|
|
2566
2627
|
};
|
|
2567
|
-
await this.
|
|
2628
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
2568
2629
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2569
2630
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
2570
2631
|
}
|
|
@@ -2616,7 +2677,7 @@ export class RunDispatcher {
|
|
|
2616
2677
|
if (shouldLoopCoder(state)) {
|
|
2617
2678
|
this.loopCoderForFollowUps(instance, step);
|
|
2618
2679
|
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2619
|
-
await this.
|
|
2680
|
+
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
2620
2681
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2621
2682
|
return { kind: 'continue' };
|
|
2622
2683
|
}
|
|
@@ -2709,7 +2770,9 @@ export class RunDispatcher {
|
|
|
2709
2770
|
}
|
|
2710
2771
|
/** File a `follow_up` item as a tracker issue (GitHub / Jira), recording the ticket ref. */
|
|
2711
2772
|
async fileFollowUp(workspaceId, executionId, itemId) {
|
|
2712
|
-
|
|
2773
|
+
// Snapshot read for validation + frame resolution before the non-idempotent ticket
|
|
2774
|
+
// creation; the item's ticket refs are then recorded under CAS in applyFollowUpDecision.
|
|
2775
|
+
const { instance, item } = await this.loadFollowUpItem(workspaceId, executionId, itemId);
|
|
2713
2776
|
if (item.kind !== 'follow_up') {
|
|
2714
2777
|
throw new ConflictError('Only follow-up items can be filed as issues');
|
|
2715
2778
|
}
|
|
@@ -2733,91 +2796,123 @@ export class RunDispatcher {
|
|
|
2733
2796
|
if (!ticket) {
|
|
2734
2797
|
throw new ConflictError('No issue tracker is configured for this workspace');
|
|
2735
2798
|
}
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2799
|
+
return this.applyFollowUpDecision(workspaceId, executionId, itemId, (target) => {
|
|
2800
|
+
// Re-validated on the fresh snapshot inside the CAS (the ticket was already created above).
|
|
2801
|
+
if (target.kind !== 'follow_up') {
|
|
2802
|
+
throw new ConflictError('Only follow-up items can be filed as issues');
|
|
2803
|
+
}
|
|
2804
|
+
target.status = 'filed';
|
|
2805
|
+
target.ticketExternalId = ticket.externalId;
|
|
2806
|
+
target.ticketUrl = ticket.url;
|
|
2807
|
+
target.updatedAt = this.clock.now();
|
|
2808
|
+
});
|
|
2742
2809
|
}
|
|
2743
2810
|
/** Queue a `follow_up` item to send back to the Coder on its next pass. */
|
|
2744
2811
|
async queueFollowUp(workspaceId, executionId, itemId) {
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
return step.followUps;
|
|
2812
|
+
return this.applyFollowUpDecision(workspaceId, executionId, itemId, (item) => {
|
|
2813
|
+
if (item.kind !== 'follow_up') {
|
|
2814
|
+
throw new ConflictError('Only follow-up items can be sent back to the Coder');
|
|
2815
|
+
}
|
|
2816
|
+
item.status = 'queued';
|
|
2817
|
+
item.sentToCoder = false;
|
|
2818
|
+
item.updatedAt = this.clock.now();
|
|
2819
|
+
});
|
|
2754
2820
|
}
|
|
2755
2821
|
/** Answer a `question` item; the answer is folded into the Coder's next pass. */
|
|
2756
2822
|
async answerFollowUp(workspaceId, executionId, itemId, answer) {
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
return step.followUps;
|
|
2823
|
+
return this.applyFollowUpDecision(workspaceId, executionId, itemId, (item) => {
|
|
2824
|
+
if (item.kind !== 'question') {
|
|
2825
|
+
throw new ConflictError('Only question items can be answered');
|
|
2826
|
+
}
|
|
2827
|
+
item.status = 'answered';
|
|
2828
|
+
item.answer = answer;
|
|
2829
|
+
item.sentToCoder = false;
|
|
2830
|
+
item.updatedAt = this.clock.now();
|
|
2831
|
+
});
|
|
2767
2832
|
}
|
|
2768
2833
|
/** Dismiss a follow-up / question item without acting on it. */
|
|
2769
2834
|
async dismissFollowUp(workspaceId, executionId, itemId) {
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
return step.followUps;
|
|
2835
|
+
return this.applyFollowUpDecision(workspaceId, executionId, itemId, (item) => {
|
|
2836
|
+
item.status = 'dismissed';
|
|
2837
|
+
item.updatedAt = this.clock.now();
|
|
2838
|
+
});
|
|
2775
2839
|
}
|
|
2776
2840
|
/**
|
|
2777
|
-
*
|
|
2778
|
-
* every item
|
|
2779
|
-
* items
|
|
2780
|
-
*
|
|
2841
|
+
* Apply a human follow-up item decision and, when the run is PARKED on this step's follow-up
|
|
2842
|
+
* gate with every item now decided, drive it forward — loop the Coder for the queued/answered
|
|
2843
|
+
* items, hand off to a co-located approval gate, or advance past the gate — all under
|
|
2844
|
+
* OPTIMISTIC CONCURRENCY. A follow-up decision can race the driver's running-poll fold (which
|
|
2845
|
+
* appends newly-streamed items) or another decision on a sibling item, so it RE-READS +
|
|
2846
|
+
* RE-APPLIES on a lost CAS race instead of clobbering — the human-action dual of the driver's
|
|
2847
|
+
* abort-and-redrive (race-audit 2.2). The item mutation + the in-memory gate transition run
|
|
2848
|
+
* INSIDE the CAS callback (idempotent, re-runnable on reload); the non-idempotent side effects
|
|
2849
|
+
* (notifications, `signalDecision`, emit) run once AFTER, on the winning snapshot. `decide`
|
|
2850
|
+
* validates + mutates the item, throwing a `ConflictError`/`NotFoundError` that propagates
|
|
2851
|
+
* immediately (a domain error is not retried).
|
|
2781
2852
|
*/
|
|
2782
|
-
async
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
const
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2853
|
+
async applyFollowUpDecision(workspaceId, executionId, itemId, decide) {
|
|
2854
|
+
// Captured inside the (re-runnable) callback for the last winning attempt; the
|
|
2855
|
+
// non-idempotent side effects below act on them.
|
|
2856
|
+
let outcome = 'record';
|
|
2857
|
+
let index = -1;
|
|
2858
|
+
let loopDecisionId;
|
|
2859
|
+
const persisted = await this.runStateMachine.mutateInstance(workspaceId, executionId, (fresh) => {
|
|
2860
|
+
outcome = 'record';
|
|
2861
|
+
loopDecisionId = undefined;
|
|
2862
|
+
index = fresh.steps.findIndex((s) => s.followUps?.enabled && s.followUps.items.some((i) => i.id === itemId));
|
|
2863
|
+
if (index < 0)
|
|
2864
|
+
throw new NotFoundError('Follow-up item', itemId);
|
|
2865
|
+
const step = fresh.steps[index];
|
|
2866
|
+
decide(step.followUps.items.find((i) => i.id === itemId));
|
|
2867
|
+
const parkedHere = fresh.status === 'blocked' &&
|
|
2868
|
+
step.approval?.status === 'pending' &&
|
|
2869
|
+
fresh.currentStep === index;
|
|
2870
|
+
// Still collecting decisions (or the run isn't parked on this gate): only record it.
|
|
2871
|
+
if (!parkedHere || hasPendingFollowUps(step.followUps))
|
|
2872
|
+
return;
|
|
2873
|
+
// Every item decided and the run is parked here: loop the Coder for the send-back items,
|
|
2874
|
+
// hand off to a co-located approval gate, or advance past the gate.
|
|
2875
|
+
if (shouldLoopCoder(step.followUps)) {
|
|
2876
|
+
loopDecisionId = step.approval.id;
|
|
2877
|
+
this.loopCoderForFollowUps(fresh, step);
|
|
2878
|
+
outcome = 'loop';
|
|
2879
|
+
return;
|
|
2880
|
+
}
|
|
2881
|
+
const isFinalStep = index === fresh.steps.length - 1;
|
|
2882
|
+
if (step.requiresApproval && !isFinalStep && step.approval?.status === 'pending') {
|
|
2883
|
+
// The follow-up park reused `step.approval`; advancing here would silently SKIP the
|
|
2884
|
+
// approval. Refresh the proposal and hand off to the standard approval gate (the
|
|
2885
|
+
// follow-up card is cleared + the "waiting for input" card re-raised below), preserving
|
|
2886
|
+
// the follow-up-before-approval ordering recordStepResult established across the park.
|
|
2887
|
+
step.approval = { ...step.approval, proposal: step.output ?? '' };
|
|
2888
|
+
outcome = 'handoff';
|
|
2889
|
+
return;
|
|
2890
|
+
}
|
|
2891
|
+
this.runStateMachine.advanceRunPastGate(fresh, index);
|
|
2892
|
+
outcome = 'advance';
|
|
2893
|
+
});
|
|
2894
|
+
// Non-idempotent side effects on the winning snapshot (the CAS write is the source of truth,
|
|
2895
|
+
// so nothing re-persists here). The settled paths first clear the follow-up waiting card.
|
|
2896
|
+
if (outcome === 'record') {
|
|
2897
|
+
await this.runStateMachine.emitInstance(workspaceId, persisted);
|
|
2803
2898
|
}
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2899
|
+
else {
|
|
2900
|
+
await this.runStateMachine.clearWaitingNotification(workspaceId, persisted);
|
|
2901
|
+
if (outcome === 'loop') {
|
|
2902
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, persisted, 'in_progress');
|
|
2903
|
+
await this.workRunner.signalDecision(workspaceId, persisted.id, loopDecisionId, 'approved');
|
|
2904
|
+
await this.runStateMachine.emitInstance(workspaceId, persisted);
|
|
2905
|
+
}
|
|
2906
|
+
else if (outcome === 'handoff') {
|
|
2907
|
+
await this.runStateMachine.ensureWaitingNotification(workspaceId, persisted);
|
|
2908
|
+
await this.runStateMachine.emitInstance(workspaceId, persisted);
|
|
2909
|
+
}
|
|
2910
|
+
else {
|
|
2911
|
+
// advance: block writes + `signalDecision('approved')` + emit — the approveStep template.
|
|
2912
|
+
await this.runStateMachine.settleAdvancedGate(workspaceId, persisted, index);
|
|
2913
|
+
}
|
|
2819
2914
|
}
|
|
2820
|
-
|
|
2915
|
+
return persisted.steps[index].followUps;
|
|
2821
2916
|
}
|
|
2822
2917
|
/** Provision inputs (`{{input.*}}`) derived from the block under deployment. */
|
|
2823
2918
|
deployInputs(block) {
|