@cat-factory/orchestration 0.37.1 → 0.37.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/execution/CompanionController.d.ts +6 -11
- package/dist/modules/execution/CompanionController.d.ts.map +1 -1
- package/dist/modules/execution/CompanionController.js +20 -20
- package/dist/modules/execution/CompanionController.js.map +1 -1
- package/dist/modules/execution/ExecutionService.d.ts +3 -100
- package/dist/modules/execution/ExecutionService.d.ts.map +1 -1
- package/dist/modules/execution/ExecutionService.js +84 -460
- package/dist/modules/execution/ExecutionService.js.map +1 -1
- package/dist/modules/execution/HumanTestController.d.ts +6 -8
- package/dist/modules/execution/HumanTestController.d.ts.map +1 -1
- package/dist/modules/execution/HumanTestController.js +26 -26
- package/dist/modules/execution/HumanTestController.js.map +1 -1
- package/dist/modules/execution/ReviewGateController.d.ts +11 -19
- package/dist/modules/execution/ReviewGateController.d.ts.map +1 -1
- package/dist/modules/execution/ReviewGateController.js +19 -19
- package/dist/modules/execution/ReviewGateController.js.map +1 -1
- package/dist/modules/execution/RunStateMachine.d.ts +131 -0
- package/dist/modules/execution/RunStateMachine.d.ts.map +1 -0
- package/dist/modules/execution/RunStateMachine.js +389 -0
- package/dist/modules/execution/RunStateMachine.js.map +1 -0
- package/dist/modules/execution/TesterController.d.ts +3 -4
- package/dist/modules/execution/TesterController.d.ts.map +1 -1
- package/dist/modules/execution/TesterController.js +6 -6
- package/dist/modules/execution/TesterController.js.map +1 -1
- package/dist/modules/execution/VisualConfirmationController.d.ts +6 -8
- package/dist/modules/execution/VisualConfirmationController.d.ts.map +1 -1
- package/dist/modules/execution/VisualConfirmationController.js +18 -18
- package/dist/modules/execution/VisualConfirmationController.js.map +1 -1
- package/package.json +1 -1
|
@@ -12,6 +12,7 @@ import { DEFAULT_FOLLOW_UP_MAX_LOOPS, FOLLOW_UP_PRODUCER_KIND, followUpsToSendBa
|
|
|
12
12
|
import { AgentContextBuilder, } from './AgentContextBuilder.js';
|
|
13
13
|
import { CompanionController } from './CompanionController.js';
|
|
14
14
|
import { StepGraph } from './StepGraph.js';
|
|
15
|
+
import { RunStateMachine } from './RunStateMachine.js';
|
|
15
16
|
import { inferTechnicalLabel } from './technical.logic.js';
|
|
16
17
|
import { MergeResolver } from './MergeResolver.js';
|
|
17
18
|
import { ReviewGateController } from './ReviewGateController.js';
|
|
@@ -27,22 +28,6 @@ import { requireWorkspace } from '@cat-factory/kernel';
|
|
|
27
28
|
import { planResumedSteps, planRestartFromStep } from './retry.logic.js';
|
|
28
29
|
import { isContainerEvictionError, isTransientEviction, MAX_EVICTION_RECOVERIES, MAX_TRANSIENT_EVICTION_RECOVERIES, } from './job.logic.js';
|
|
29
30
|
import { decideTesterInfra, resolveTesterEnvironment, TESTER_INFRA_MESSAGES, } from './tester-infra.logic.js';
|
|
30
|
-
/**
|
|
31
|
-
* "What to do next" guidance per failure kind a pipeline run can produce, shown
|
|
32
|
-
* under the failure banner on the board (mirrors bootstrap's FAILURE_HINTS). Only
|
|
33
|
-
* the execution-relevant subset of {@link AgentFailureKind} is keyed.
|
|
34
|
-
*/
|
|
35
|
-
const EXECUTION_FAILURE_HINTS = {
|
|
36
|
-
agent: 'An agent step failed after its automatic retries. Review the run, then retry to re-run the pipeline.',
|
|
37
|
-
job_failed: 'The implementation container reported a failure. Inspect its logs (Cloudflare Workers Observability, filtered by the run id), then retry to spin a fresh container.',
|
|
38
|
-
evicted: 'The implementation container kept vanishing mid-run even after automatic fresh-container restarts. Most often this is transient: a deploy / new-version rollout draining the container, in which case simply retrying once the rollout has finished succeeds. If it persists, it points at a memory or crash issue on the run — inspect its logs (Cloudflare Workers Observability, filtered by the run id) and consider a heavier container instance type. Retry to try again.',
|
|
39
|
-
timeout: 'The run exceeded its time budget — a step or the implementation job did not finish in time. Retry to start it again.',
|
|
40
|
-
rejected: 'You rejected this step’s proposal, stopping the run. Retry to re-run the pipeline from the rejected step.',
|
|
41
|
-
companion_rejected: 'A companion agent could not return a usable quality assessment (its reply was truncated or malformed) even after a repair retry. Review the companion’s raw output on the run, then retry.',
|
|
42
|
-
cancelled: 'You stopped this run; its container was killed. Retry to start it again.',
|
|
43
|
-
dispatch: 'The agent’s container could not be started — the run never began executing. The provider/runtime’s verbatim response is shown below. Most often this is transient (a capacity blip or a new-version rollout); retrying spins a fresh container. If it persists it points at a misconfigured container binding/image or runner pool. Retry to try again.',
|
|
44
|
-
unknown: 'The run failed for an unclassified reason. Review the run, then retry.',
|
|
45
|
-
};
|
|
46
31
|
/**
|
|
47
32
|
* Step kinds whose run details surface the ephemeral-environment lifecycle: the
|
|
48
33
|
* `deployer` provisions it and the `tester`/`playwright` exercise it. Used to gate
|
|
@@ -96,13 +81,14 @@ export class ExecutionService {
|
|
|
96
81
|
clock;
|
|
97
82
|
/** The pure step/cursor mutators (start/finish/park/reset + the companion rework loop). */
|
|
98
83
|
stepGraph;
|
|
84
|
+
/** The async instance/block state-machine spine (persist/emit/park/advance/finalize/fail). */
|
|
85
|
+
runStateMachine;
|
|
99
86
|
agentExecutor;
|
|
100
87
|
workRunner;
|
|
101
88
|
events;
|
|
102
89
|
board;
|
|
103
90
|
spend;
|
|
104
91
|
requirementReviewService;
|
|
105
|
-
kaizenScheduler;
|
|
106
92
|
clarityReviewService;
|
|
107
93
|
brainstormServices;
|
|
108
94
|
environmentProvisioning;
|
|
@@ -132,7 +118,6 @@ export class ExecutionService {
|
|
|
132
118
|
blueprintReconciler;
|
|
133
119
|
notificationService;
|
|
134
120
|
workspaceSettingsService;
|
|
135
|
-
llmObservability;
|
|
136
121
|
prMerger;
|
|
137
122
|
mergePresetRepository;
|
|
138
123
|
ticketTrackerProvider;
|
|
@@ -181,13 +166,26 @@ export class ExecutionService {
|
|
|
181
166
|
this.idGenerator = idGenerator;
|
|
182
167
|
this.clock = clock;
|
|
183
168
|
this.stepGraph = new StepGraph(clock);
|
|
169
|
+
this.runStateMachine = new RunStateMachine({
|
|
170
|
+
executionRepository,
|
|
171
|
+
blockRepository,
|
|
172
|
+
events: executionEventPublisher,
|
|
173
|
+
workRunner,
|
|
174
|
+
agentExecutor,
|
|
175
|
+
idGenerator,
|
|
176
|
+
clock,
|
|
177
|
+
stepGraph: this.stepGraph,
|
|
178
|
+
notificationService,
|
|
179
|
+
kaizenScheduler,
|
|
180
|
+
subscriptionActivations: subscriptionActivationRepository,
|
|
181
|
+
llmObservability,
|
|
182
|
+
});
|
|
184
183
|
this.agentExecutor = agentExecutor;
|
|
185
184
|
this.workRunner = workRunner;
|
|
186
185
|
this.events = executionEventPublisher;
|
|
187
186
|
this.board = boardService;
|
|
188
187
|
this.spend = spendService;
|
|
189
188
|
this.requirementReviewService = requirementReviewService;
|
|
190
|
-
this.kaizenScheduler = kaizenScheduler;
|
|
191
189
|
this.clarityReviewService = clarityReviewService;
|
|
192
190
|
this.brainstormServices = brainstormServices;
|
|
193
191
|
this.environmentProvisioning = environmentProvisioning;
|
|
@@ -219,17 +217,8 @@ export class ExecutionService {
|
|
|
219
217
|
idGenerator,
|
|
220
218
|
previewStepModel: (ctx) => this.previewStepModel(ctx),
|
|
221
219
|
runAgent: (ctx, opts) => this.runAgent(ctx, opts),
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
pauseStepForInput: (s) => this.stepGraph.pauseStepForInput(s),
|
|
225
|
-
updateBlockProgress: (ws, i, st) => this.updateBlockProgress(ws, i, st),
|
|
226
|
-
persistInstance: (ws, i) => this.executionRepository.upsert(ws, i),
|
|
227
|
-
emitInstance: (ws, i) => this.emitInstance(ws, i),
|
|
228
|
-
stopRunContainer: (ws, i) => this.stopRunContainer(ws, i),
|
|
229
|
-
finalizeBlock: (ws, i, c) => this.finalizeBlock(ws, i, c),
|
|
230
|
-
parkStepOnDecision: (ws, i, s, p) => this.parkStepOnDecision(ws, i, s, p),
|
|
231
|
-
raiseDecisionRequired: (ws, i) => this.raiseDecisionRequired(ws, i),
|
|
232
|
-
loopCompanionProducer: (i, ci, rw) => this.stepGraph.loopCompanionProducer(i, ci, rw),
|
|
220
|
+
stateMachine: this.runStateMachine,
|
|
221
|
+
stepGraph: this.stepGraph,
|
|
233
222
|
inferTechnicalLabel: (ws, block, producer, companionStep) => this.inferBlockTechnical(ws, block, producer, companionStep),
|
|
234
223
|
});
|
|
235
224
|
this.testerController = new TesterController({
|
|
@@ -238,9 +227,7 @@ export class ExecutionService {
|
|
|
238
227
|
agentExecutor,
|
|
239
228
|
contextBuilder: this.contextBuilder,
|
|
240
229
|
resolveMergePreset: (ws, block) => this.resolveMergePreset(ws, block),
|
|
241
|
-
|
|
242
|
-
persistInstance: (ws, i) => this.executionRepository.upsert(ws, i),
|
|
243
|
-
emitInstance: (ws, i) => this.emitInstance(ws, i),
|
|
230
|
+
stateMachine: this.runStateMachine,
|
|
244
231
|
});
|
|
245
232
|
this.humanTestController = new HumanTestController({
|
|
246
233
|
blockRepository,
|
|
@@ -273,14 +260,8 @@ export class ExecutionService {
|
|
|
273
260
|
: {}),
|
|
274
261
|
...(branchUpdater ? { branchUpdater } : {}),
|
|
275
262
|
resolveMergePreset: (ws, block) => this.resolveMergePreset(ws, block),
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
startStep: (s) => this.stepGraph.startStep(s),
|
|
279
|
-
updateBlockProgress: (ws, i, st) => this.updateBlockProgress(ws, i, st),
|
|
280
|
-
finalizeBlock: (ws, i, c) => this.finalizeBlock(ws, i, c),
|
|
281
|
-
stopRunContainer: (ws, i) => this.stopRunContainer(ws, i),
|
|
282
|
-
persistInstance: (ws, i) => this.executionRepository.upsert(ws, i),
|
|
283
|
-
emitInstance: (ws, i) => this.emitInstance(ws, i),
|
|
263
|
+
stateMachine: this.runStateMachine,
|
|
264
|
+
stepGraph: this.stepGraph,
|
|
284
265
|
clockNow: () => this.clock.now(),
|
|
285
266
|
});
|
|
286
267
|
this.visualConfirmationController = new VisualConfirmationController({
|
|
@@ -292,32 +273,18 @@ export class ExecutionService {
|
|
|
292
273
|
notificationService,
|
|
293
274
|
...(resolveBinaryArtifactStore ? { resolveBinaryArtifactStore } : {}),
|
|
294
275
|
resolveMergePreset: (ws, block) => this.resolveMergePreset(ws, block),
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
startStep: (s) => this.stepGraph.startStep(s),
|
|
298
|
-
updateBlockProgress: (ws, i, st) => this.updateBlockProgress(ws, i, st),
|
|
299
|
-
finalizeBlock: (ws, i, c) => this.finalizeBlock(ws, i, c),
|
|
300
|
-
stopRunContainer: (ws, i) => this.stopRunContainer(ws, i),
|
|
301
|
-
persistInstance: (ws, i) => this.executionRepository.upsert(ws, i),
|
|
302
|
-
emitInstance: (ws, i) => this.emitInstance(ws, i),
|
|
276
|
+
stateMachine: this.runStateMachine,
|
|
277
|
+
stepGraph: this.stepGraph,
|
|
303
278
|
clockNow: () => this.clock.now(),
|
|
304
279
|
});
|
|
305
280
|
this.reviewGate = new ReviewGateController({
|
|
306
281
|
blockRepository,
|
|
307
282
|
executionRepository,
|
|
308
283
|
workRunner,
|
|
284
|
+
stateMachine: this.runStateMachine,
|
|
285
|
+
stepGraph: this.stepGraph,
|
|
309
286
|
resolveMergePreset: (ws, block) => this.resolveMergePreset(ws, block),
|
|
310
|
-
parkStepOnDecision: (ws, i, s, p) => this.parkStepOnDecision(ws, i, s, p),
|
|
311
|
-
advancePastResolvedGate: (ws, i, idx) => this.advancePastResolvedGate(ws, i, idx),
|
|
312
287
|
dispatchIterationCap: (ws, blockId, choice, handlers) => this.dispatchIterationCap(ws, blockId, choice, handlers),
|
|
313
|
-
raiseDecisionRequired: (ws, i) => this.raiseDecisionRequired(ws, i),
|
|
314
|
-
finishStep: (s) => this.stepGraph.finishStep(s),
|
|
315
|
-
startStep: (s) => this.stepGraph.startStep(s),
|
|
316
|
-
updateBlockProgress: (ws, i, st) => this.updateBlockProgress(ws, i, st),
|
|
317
|
-
finalizeBlock: (ws, i, c) => this.finalizeBlock(ws, i, c),
|
|
318
|
-
stopRunContainer: (ws, i) => this.stopRunContainer(ws, i),
|
|
319
|
-
persistInstance: (ws, i) => this.executionRepository.upsert(ws, i),
|
|
320
|
-
emitInstance: (ws, i) => this.emitInstance(ws, i),
|
|
321
288
|
});
|
|
322
289
|
this.requirementsKind = this.buildRequirementsKind();
|
|
323
290
|
this.clarityKind = this.buildClarityKind();
|
|
@@ -326,7 +293,6 @@ export class ExecutionService {
|
|
|
326
293
|
this.blueprintReconciler = blueprintReconciler;
|
|
327
294
|
this.notificationService = notificationService;
|
|
328
295
|
this.workspaceSettingsService = workspaceSettingsService;
|
|
329
|
-
this.llmObservability = llmObservability;
|
|
330
296
|
this.prMerger = pullRequestMerger;
|
|
331
297
|
this.mergePresetRepository = mergePresetRepository;
|
|
332
298
|
this.ticketTrackerProvider = ticketTrackerProvider;
|
|
@@ -636,7 +602,7 @@ export class ExecutionService {
|
|
|
636
602
|
// a browser open. With the no-op runner (tests) this does nothing and the run
|
|
637
603
|
// is advanced directly via advanceInstance.
|
|
638
604
|
await this.workRunner.startRun(workspaceId, instance.id);
|
|
639
|
-
await this.emitInstance(workspaceId, instance);
|
|
605
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
640
606
|
return instance;
|
|
641
607
|
}
|
|
642
608
|
/**
|
|
@@ -759,10 +725,10 @@ export class ExecutionService {
|
|
|
759
725
|
// auto-passed, or the run reached a terminal state) clear that waiting card so the
|
|
760
726
|
// escalation sweep can't later flip a settled decision red ("Overdue").
|
|
761
727
|
if (result.kind === 'awaiting_decision') {
|
|
762
|
-
await this.ensureWaitingNotification(workspaceId, instance);
|
|
728
|
+
await this.runStateMachine.ensureWaitingNotification(workspaceId, instance);
|
|
763
729
|
}
|
|
764
730
|
else {
|
|
765
|
-
await this.clearWaitingNotification(workspaceId, instance);
|
|
731
|
+
await this.runStateMachine.clearWaitingNotification(workspaceId, instance);
|
|
766
732
|
}
|
|
767
733
|
return result;
|
|
768
734
|
}
|
|
@@ -784,7 +750,7 @@ export class ExecutionService {
|
|
|
784
750
|
if (instance.status !== 'paused') {
|
|
785
751
|
instance.status = 'paused';
|
|
786
752
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
787
|
-
await this.emitInstance(workspaceId, instance);
|
|
753
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
788
754
|
}
|
|
789
755
|
return { kind: 'paused' };
|
|
790
756
|
}
|
|
@@ -818,7 +784,7 @@ export class ExecutionService {
|
|
|
818
784
|
if (pendingId) {
|
|
819
785
|
instance.status = 'blocked';
|
|
820
786
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
821
|
-
await this.emitInstance(workspaceId, instance);
|
|
787
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
822
788
|
return { kind: 'awaiting_decision', decisionId: pendingId };
|
|
823
789
|
}
|
|
824
790
|
}
|
|
@@ -891,7 +857,7 @@ export class ExecutionService {
|
|
|
891
857
|
// phase, so a run's details show the env spinning up next to the container.
|
|
892
858
|
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step);
|
|
893
859
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
894
|
-
await this.emitInstance(workspaceId, instance);
|
|
860
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
895
861
|
let handle;
|
|
896
862
|
try {
|
|
897
863
|
handle = await executor.startJob(context);
|
|
@@ -904,7 +870,7 @@ export class ExecutionService {
|
|
|
904
870
|
// "run failed". A dispatch-time eviction still routes to the evicted framing.
|
|
905
871
|
step.startingContainer = false;
|
|
906
872
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
907
|
-
await this.emitInstance(workspaceId, instance);
|
|
873
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
908
874
|
const message = getErrorMessage(error);
|
|
909
875
|
const evicted = isContainerEvictionError(message);
|
|
910
876
|
return {
|
|
@@ -921,7 +887,7 @@ export class ExecutionService {
|
|
|
921
887
|
// The dispatch returned, so the container is up and execution has begun.
|
|
922
888
|
step.startingContainer = false;
|
|
923
889
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
924
|
-
await this.emitInstance(workspaceId, instance);
|
|
890
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
925
891
|
}
|
|
926
892
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
927
893
|
}
|
|
@@ -932,7 +898,7 @@ export class ExecutionService {
|
|
|
932
898
|
if (previewModel && previewModel !== step.model) {
|
|
933
899
|
step.model = previewModel;
|
|
934
900
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
935
|
-
await this.emitInstance(workspaceId, instance);
|
|
901
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
936
902
|
}
|
|
937
903
|
const result = await this.runAgent(context, options);
|
|
938
904
|
return this.recordStepResult(workspaceId, instance, step, isFinalStep, result);
|
|
@@ -1069,7 +1035,7 @@ export class ExecutionService {
|
|
|
1069
1035
|
}
|
|
1070
1036
|
if (changed) {
|
|
1071
1037
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1072
|
-
await this.emitInstance(workspaceId, instance);
|
|
1038
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1073
1039
|
}
|
|
1074
1040
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
1075
1041
|
}
|
|
@@ -1158,7 +1124,7 @@ export class ExecutionService {
|
|
|
1158
1124
|
if (step.gate)
|
|
1159
1125
|
step.gate.phase = 'checking';
|
|
1160
1126
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1161
|
-
await this.emitInstance(workspaceId, instance);
|
|
1127
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1162
1128
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
1163
1129
|
}
|
|
1164
1130
|
// A `tester` step in its `fixing` phase has a Fixer job in flight, NOT the
|
|
@@ -1175,7 +1141,7 @@ export class ExecutionService {
|
|
|
1175
1141
|
// Reclaim the finished Fixer container before re-dispatching the Tester so it
|
|
1176
1142
|
// boots fresh against the just-pushed fixes (rather than re-attaching to the
|
|
1177
1143
|
// completed job by run id).
|
|
1178
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
1144
|
+
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
1179
1145
|
return this.testerController.dispatchTester(workspaceId, instance, step, block);
|
|
1180
1146
|
}
|
|
1181
1147
|
// A `human-test` gate in its `fixing` / `resolving_conflicts` phase has a helper job
|
|
@@ -1225,7 +1191,7 @@ export class ExecutionService {
|
|
|
1225
1191
|
step.subtasks = undefined;
|
|
1226
1192
|
step.progress = 0;
|
|
1227
1193
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1228
|
-
await this.emitInstance(workspaceId, instance);
|
|
1194
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1229
1195
|
return { kind: 'continue' };
|
|
1230
1196
|
}
|
|
1231
1197
|
return {
|
|
@@ -1407,19 +1373,19 @@ export class ExecutionService {
|
|
|
1407
1373
|
this.stepGraph.finishStep(step);
|
|
1408
1374
|
if (isFinalStep) {
|
|
1409
1375
|
instance.status = 'done';
|
|
1410
|
-
await this.finalizeBlock(workspaceId, instance, undefined);
|
|
1376
|
+
await this.runStateMachine.finalizeBlock(workspaceId, instance, undefined);
|
|
1411
1377
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1412
|
-
await this.emitInstance(workspaceId, instance);
|
|
1413
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
1378
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1379
|
+
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
1414
1380
|
return { kind: 'done' };
|
|
1415
1381
|
}
|
|
1416
1382
|
instance.currentStep += 1;
|
|
1417
1383
|
const next = instance.steps[instance.currentStep];
|
|
1418
1384
|
if (next)
|
|
1419
1385
|
this.stepGraph.startStep(next);
|
|
1420
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
1386
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
1421
1387
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1422
|
-
await this.emitInstance(workspaceId, instance);
|
|
1388
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1423
1389
|
return { kind: 'continue' };
|
|
1424
1390
|
}
|
|
1425
1391
|
/**
|
|
@@ -1467,9 +1433,9 @@ export class ExecutionService {
|
|
|
1467
1433
|
};
|
|
1468
1434
|
this.stepGraph.pauseStepForInput(step);
|
|
1469
1435
|
instance.status = 'blocked';
|
|
1470
|
-
await this.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
1436
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
1471
1437
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1472
|
-
await this.emitInstance(workspaceId, instance);
|
|
1438
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1473
1439
|
return { kind: 'awaiting_decision', decisionId: step.decision.id };
|
|
1474
1440
|
}
|
|
1475
1441
|
// Completion-path interceptors short-circuit before the normal finish/advance for the
|
|
@@ -1586,9 +1552,9 @@ export class ExecutionService {
|
|
|
1586
1552
|
};
|
|
1587
1553
|
this.stepGraph.pauseStepForInput(step);
|
|
1588
1554
|
instance.status = 'blocked';
|
|
1589
|
-
await this.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
1555
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
1590
1556
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1591
|
-
await this.emitInstance(workspaceId, instance);
|
|
1557
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1592
1558
|
return { kind: 'awaiting_decision', decisionId: step.approval.id };
|
|
1593
1559
|
}
|
|
1594
1560
|
// Persist the agent's reported confidence whenever a step reports it, for board
|
|
@@ -1634,14 +1600,14 @@ export class ExecutionService {
|
|
|
1634
1600
|
// POSITION-INDEPENDENTLY: confidence at the top of recordStepResult and the merger's
|
|
1635
1601
|
// real merge via the step-completion resolver registry (so a trailing
|
|
1636
1602
|
// post-release-health gate doesn't disable auto-merge). Nothing merge-specific here.
|
|
1637
|
-
await this.finalizeBlock(workspaceId, instance, result.confidence);
|
|
1603
|
+
await this.runStateMachine.finalizeBlock(workspaceId, instance, result.confidence);
|
|
1638
1604
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1639
|
-
await this.emitInstance(workspaceId, instance);
|
|
1605
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1640
1606
|
// The run is finished: reclaim its per-run container now instead of letting it
|
|
1641
1607
|
// idle out its sleepAfter window (~10 min of billed-but-useless compute). All
|
|
1642
1608
|
// pipeline steps share the one container keyed by the execution id, so this is
|
|
1643
1609
|
// only safe on the FINAL step — never between steps. Best-effort/idempotent.
|
|
1644
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
1610
|
+
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
1645
1611
|
return { kind: 'done' };
|
|
1646
1612
|
}
|
|
1647
1613
|
instance.currentStep += 1;
|
|
@@ -1653,13 +1619,13 @@ export class ExecutionService {
|
|
|
1653
1619
|
// advance to a trailing step — refresh progress only, preserving that status. (The
|
|
1654
1620
|
// final step's `finalizeBlock` then leaves a `done` block alone.)
|
|
1655
1621
|
if (resolverOwnsTerminalStatus) {
|
|
1656
|
-
await this.refreshBlockProgress(workspaceId, instance);
|
|
1622
|
+
await this.runStateMachine.refreshBlockProgress(workspaceId, instance);
|
|
1657
1623
|
}
|
|
1658
1624
|
else {
|
|
1659
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
1625
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
1660
1626
|
}
|
|
1661
1627
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1662
|
-
await this.emitInstance(workspaceId, instance);
|
|
1628
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1663
1629
|
return { kind: 'continue' };
|
|
1664
1630
|
}
|
|
1665
1631
|
/**
|
|
@@ -2282,7 +2248,7 @@ export class ExecutionService {
|
|
|
2282
2248
|
// Keep polling. Persist the head sha + phase so the board can reflect it.
|
|
2283
2249
|
step.gate.phase = 'checking';
|
|
2284
2250
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2285
|
-
await this.emitInstance(workspaceId, instance);
|
|
2251
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2286
2252
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
2287
2253
|
}
|
|
2288
2254
|
// probe.status === 'fail'.
|
|
@@ -2337,75 +2303,9 @@ export class ExecutionService {
|
|
|
2337
2303
|
headSha: step.gate?.headSha ?? null,
|
|
2338
2304
|
};
|
|
2339
2305
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2340
|
-
await this.emitInstance(workspaceId, instance);
|
|
2306
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2341
2307
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
2342
2308
|
}
|
|
2343
|
-
/**
|
|
2344
|
-
* Raise a `decision_required` notification when a run parks on an iteration-cap gate
|
|
2345
|
-
* after spending its automatic budget — a quality companion at its rework cap or an
|
|
2346
|
-
* iterative reviewer (requirements / clarity) at its iteration cap. Without it the
|
|
2347
|
-
* three-choice decision is reachable only by drilling into the parked step, so the run
|
|
2348
|
-
* looks silently stuck. Best-effort: a missing notification service (tests) or block is
|
|
2349
|
-
* a no-op.
|
|
2350
|
-
*/
|
|
2351
|
-
async raiseDecisionRequired(workspaceId, instance) {
|
|
2352
|
-
if (!this.notificationService)
|
|
2353
|
-
return;
|
|
2354
|
-
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
2355
|
-
if (!block)
|
|
2356
|
-
return;
|
|
2357
|
-
await this.notificationService.raise(workspaceId, {
|
|
2358
|
-
type: 'decision_required',
|
|
2359
|
-
blockId: block.id,
|
|
2360
|
-
executionId: instance.id,
|
|
2361
|
-
title: `"${block.title}" ran out of automatic iterations and needs your decision`,
|
|
2362
|
-
body: 'An automatic review loop reached its iteration cap without converging. Open the ' +
|
|
2363
|
-
'task to choose: one more round, proceed with the current result, or stop and reset.',
|
|
2364
|
-
payload: { pipelineName: instance.pipelineName },
|
|
2365
|
-
});
|
|
2366
|
-
}
|
|
2367
|
-
/**
|
|
2368
|
-
* Ensure an open notification exists for a run that has just parked waiting for a human
|
|
2369
|
-
* (an agent-raised decision, an approval gate, or an iterative review gate). Without
|
|
2370
|
-
* the old decision timeout the run waits indefinitely, so the inbox card — which the
|
|
2371
|
-
* periodic sweep escalates yellow → red — is the only signal a human is needed.
|
|
2372
|
-
*
|
|
2373
|
-
* Non-clobbering: if ANY open notification is already on the block (a more specific
|
|
2374
|
-
* `merge_review`, iteration-cap `decision_required`, etc.), it is left untouched and we
|
|
2375
|
-
* raise nothing — so the richer message wins. Best-effort: no notification service
|
|
2376
|
-
* (tests) or a missing block is a no-op.
|
|
2377
|
-
*/
|
|
2378
|
-
async ensureWaitingNotification(workspaceId, instance) {
|
|
2379
|
-
const svc = this.notificationService;
|
|
2380
|
-
if (!svc)
|
|
2381
|
-
return;
|
|
2382
|
-
const open = await svc.listOpen(workspaceId);
|
|
2383
|
-
if (open.some((n) => n.blockId === instance.blockId))
|
|
2384
|
-
return;
|
|
2385
|
-
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
2386
|
-
if (!block)
|
|
2387
|
-
return;
|
|
2388
|
-
await svc.raise(workspaceId, {
|
|
2389
|
-
type: 'decision_required',
|
|
2390
|
-
blockId: block.id,
|
|
2391
|
-
executionId: instance.id,
|
|
2392
|
-
title: `"${block.title}" is waiting for your input`,
|
|
2393
|
-
body: 'A pipeline step is parked awaiting a human decision. Open the task to respond.',
|
|
2394
|
-
payload: { pipelineName: instance.pipelineName },
|
|
2395
|
-
});
|
|
2396
|
-
}
|
|
2397
|
-
/**
|
|
2398
|
-
* Clear the auto-raised "waiting for a human decision" card once a run advances past
|
|
2399
|
-
* the decision it was parked on (so the escalation sweep can't flip a settled decision
|
|
2400
|
-
* red). Scoped to the `decision_required` type, so the human-actionable cards a stopped
|
|
2401
|
-
* run leaves behind are untouched. Best-effort: no notification service (tests) is a no-op.
|
|
2402
|
-
*/
|
|
2403
|
-
async clearWaitingNotification(workspaceId, instance) {
|
|
2404
|
-
const svc = this.notificationService;
|
|
2405
|
-
if (!svc)
|
|
2406
|
-
return;
|
|
2407
|
-
await svc.clearWaitingDecision(workspaceId, instance.blockId);
|
|
2408
|
-
}
|
|
2409
2309
|
// ---- Follow-up companion (future-looking Coder) -------------------------
|
|
2410
2310
|
// The Coder streams forward-looking items (loose ends / side-tasks / questions) which
|
|
2411
2311
|
// accrue on its `step.followUps` live (see pollAgentJob). At the Coder's completion the
|
|
@@ -2449,13 +2349,13 @@ export class ExecutionService {
|
|
|
2449
2349
|
return undefined;
|
|
2450
2350
|
if (hasPendingFollowUps(state)) {
|
|
2451
2351
|
await this.raiseFollowUpPending(workspaceId, instance, state);
|
|
2452
|
-
return this.parkStepOnDecision(workspaceId, instance, step);
|
|
2352
|
+
return this.runStateMachine.parkStepOnDecision(workspaceId, instance, step);
|
|
2453
2353
|
}
|
|
2454
2354
|
if (shouldLoopCoder(state)) {
|
|
2455
2355
|
this.loopCoderForFollowUps(instance, step);
|
|
2456
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2356
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2457
2357
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2458
|
-
await this.emitInstance(workspaceId, instance);
|
|
2358
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2459
2359
|
return { kind: 'continue' };
|
|
2460
2360
|
}
|
|
2461
2361
|
return undefined;
|
|
@@ -2624,19 +2524,19 @@ export class ExecutionService {
|
|
|
2624
2524
|
if (!parkedHere || hasPendingFollowUps(step.followUps)) {
|
|
2625
2525
|
// Still collecting decisions (or the run isn't parked on this gate): just record it.
|
|
2626
2526
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2627
|
-
await this.emitInstance(workspaceId, instance);
|
|
2527
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2628
2528
|
return;
|
|
2629
2529
|
}
|
|
2630
2530
|
// Every item is decided and the run is parked here: clear the waiting card and either
|
|
2631
2531
|
// loop the Coder for the send-back items or advance past the gate.
|
|
2632
|
-
await this.clearWaitingNotification(workspaceId, instance);
|
|
2532
|
+
await this.runStateMachine.clearWaitingNotification(workspaceId, instance);
|
|
2633
2533
|
if (shouldLoopCoder(step.followUps)) {
|
|
2634
2534
|
const decisionId = step.approval.id;
|
|
2635
2535
|
this.loopCoderForFollowUps(instance, step);
|
|
2636
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2536
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2637
2537
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2638
2538
|
await this.workRunner.signalDecision(workspaceId, instance.id, decisionId, 'approved');
|
|
2639
|
-
await this.emitInstance(workspaceId, instance);
|
|
2539
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2640
2540
|
return;
|
|
2641
2541
|
}
|
|
2642
2542
|
// The follow-up gate is settled and we won't loop. If this step ALSO carries a human
|
|
@@ -2651,11 +2551,11 @@ export class ExecutionService {
|
|
|
2651
2551
|
if (step.requiresApproval && !isFinalStep && step.approval?.status === 'pending') {
|
|
2652
2552
|
step.approval = { ...step.approval, proposal: step.output ?? '' };
|
|
2653
2553
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2654
|
-
await this.ensureWaitingNotification(workspaceId, instance);
|
|
2655
|
-
await this.emitInstance(workspaceId, instance);
|
|
2554
|
+
await this.runStateMachine.ensureWaitingNotification(workspaceId, instance);
|
|
2555
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2656
2556
|
return;
|
|
2657
2557
|
}
|
|
2658
|
-
await this.advancePastResolvedGate(workspaceId, instance, index);
|
|
2558
|
+
await this.runStateMachine.advancePastResolvedGate(workspaceId, instance, index);
|
|
2659
2559
|
}
|
|
2660
2560
|
/** Provision inputs (`{{input.*}}`) derived from the block under deployment. */
|
|
2661
2561
|
deployInputs(block) {
|
|
@@ -2765,23 +2665,6 @@ export class ExecutionService {
|
|
|
2765
2665
|
// state-machine primitives stay here — they are reused by the generic approval path and
|
|
2766
2666
|
// the companion iteration-cap gate, so they have a single home: {@link parkStepOnDecision},
|
|
2767
2667
|
// {@link advancePastResolvedGate} and {@link dispatchIterationCap}.
|
|
2768
|
-
/**
|
|
2769
|
-
* Park a step on the durable decision-wait the approval gate uses, so a human (or the
|
|
2770
|
-
* dedicated review window) can drive an iterative loop and resume the run. Shared by the
|
|
2771
|
-
* requirements gate and the companion iteration-cap gate: both reuse the SAME parking
|
|
2772
|
-
* mechanism rather than each rolling its own. `proposal` seeds the gate's stored text
|
|
2773
|
-
* (the companion's latest feedback; empty for the requirements window, which renders its
|
|
2774
|
-
* own structured surface via the universal result-view registry).
|
|
2775
|
-
*/
|
|
2776
|
-
async parkStepOnDecision(workspaceId, instance, step, proposal = '') {
|
|
2777
|
-
step.approval = { id: this.idGenerator.next('appr'), status: 'pending', proposal };
|
|
2778
|
-
this.stepGraph.pauseStepForInput(step);
|
|
2779
|
-
instance.status = 'blocked';
|
|
2780
|
-
await this.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
2781
|
-
await this.executionRepository.upsert(workspaceId, instance);
|
|
2782
|
-
await this.emitInstance(workspaceId, instance);
|
|
2783
|
-
return { kind: 'awaiting_decision', decisionId: step.approval.id };
|
|
2784
|
-
}
|
|
2785
2668
|
/**
|
|
2786
2669
|
* Two gates park on a `step.approval` but are NOT generic prose approvals — they are
|
|
2787
2670
|
* iterative gates driven by their own dedicated surface, never the generic
|
|
@@ -3061,10 +2944,10 @@ export class ExecutionService {
|
|
|
3061
2944
|
previousProposal: producer?.output ?? '',
|
|
3062
2945
|
feedback: step.companion.verdicts.at(-1)?.feedback ?? '',
|
|
3063
2946
|
});
|
|
3064
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2947
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
3065
2948
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
3066
2949
|
await this.workRunner.signalDecision(workspaceId, instance.id, approvalId, 'extra-round');
|
|
3067
|
-
await this.emitInstance(workspaceId, instance);
|
|
2950
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3068
2951
|
},
|
|
3069
2952
|
// Proceed: accept the producer's current output and advance past the gate.
|
|
3070
2953
|
proceed: async () => {
|
|
@@ -3080,43 +2963,11 @@ export class ExecutionService {
|
|
|
3080
2963
|
if (producer && block)
|
|
3081
2964
|
await this.inferBlockTechnical(workspaceId, block, producer, step);
|
|
3082
2965
|
}
|
|
3083
|
-
await this.advancePastResolvedGate(workspaceId, instance, stepIndex);
|
|
2966
|
+
await this.runStateMachine.advancePastResolvedGate(workspaceId, instance, stepIndex);
|
|
3084
2967
|
},
|
|
3085
2968
|
});
|
|
3086
2969
|
return instance;
|
|
3087
2970
|
}
|
|
3088
|
-
/**
|
|
3089
|
-
* Finish a gate step the human just resolved (its `approval` already marked `approved`),
|
|
3090
|
-
* then either finish the run (final step) or advance to the next step, persist, and wake
|
|
3091
|
-
* the parked durable driver. The single advance/finalize/signal path shared by every
|
|
3092
|
-
* gate-resume site — the generic approval ({@link approveStep}), the review gates (via
|
|
3093
|
-
* {@link ReviewGateController}) and the companion iteration-cap proceed
|
|
3094
|
-
* ({@link resolveCompanionExceeded}) — so the logic lives in exactly one place.
|
|
3095
|
-
*/
|
|
3096
|
-
async advancePastResolvedGate(workspaceId, instance, stepIndex) {
|
|
3097
|
-
const step = instance.steps[stepIndex];
|
|
3098
|
-
const decisionId = step.approval.id;
|
|
3099
|
-
this.stepGraph.finishStep(step);
|
|
3100
|
-
step.progress = 1;
|
|
3101
|
-
const isFinalStep = stepIndex === instance.steps.length - 1;
|
|
3102
|
-
if (isFinalStep) {
|
|
3103
|
-
instance.status = 'done';
|
|
3104
|
-
await this.finalizeBlock(workspaceId, instance, undefined);
|
|
3105
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
3106
|
-
}
|
|
3107
|
-
else {
|
|
3108
|
-
instance.currentStep = stepIndex + 1;
|
|
3109
|
-
const next = instance.steps[instance.currentStep];
|
|
3110
|
-
if (next)
|
|
3111
|
-
this.stepGraph.startStep(next);
|
|
3112
|
-
if (instance.status === 'blocked')
|
|
3113
|
-
instance.status = 'running';
|
|
3114
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
3115
|
-
}
|
|
3116
|
-
await this.executionRepository.upsert(workspaceId, instance);
|
|
3117
|
-
await this.workRunner.signalDecision(workspaceId, instance.id, decisionId, 'approved');
|
|
3118
|
-
await this.emitInstance(workspaceId, instance);
|
|
3119
|
-
}
|
|
3120
2971
|
// ---- clarity-review context helpers (bug-report triage) ------------------
|
|
3121
2972
|
// The clarity gate triages a block's bug report — optionally enriched by an upstream
|
|
3122
2973
|
// `bug-investigator` step's prose output — through the SAME {@link ReviewGateController}
|
|
@@ -3239,166 +3090,13 @@ export class ExecutionService {
|
|
|
3239
3090
|
if (instance.status === 'blocked')
|
|
3240
3091
|
instance.status = 'running';
|
|
3241
3092
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
3242
|
-
await this.emitInstance(workspaceId, instance);
|
|
3093
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3243
3094
|
// Ensure a driver is active to consume the pending fix (idempotent for a live run).
|
|
3244
3095
|
if (instance.status === 'running') {
|
|
3245
3096
|
await this.workRunner.startRun(workspaceId, instance.id);
|
|
3246
3097
|
}
|
|
3247
3098
|
return instance;
|
|
3248
3099
|
}
|
|
3249
|
-
/**
|
|
3250
|
-
* Push the run's latest state to subscribed clients, alongside its rolled-up
|
|
3251
|
-
* block so the board updates without a refetch. Best-effort: the publisher
|
|
3252
|
-
* swallows its own errors, and the persisted run remains the source of truth.
|
|
3253
|
-
*/
|
|
3254
|
-
async emitInstance(workspaceId, instance) {
|
|
3255
|
-
// Stamp each step with the run id so a lone step (in a pushed event, a log line, a
|
|
3256
|
-
// detail view) is self-describing for debugging; the value always equals the run id.
|
|
3257
|
-
for (const step of instance.steps)
|
|
3258
|
-
step.runId = instance.id;
|
|
3259
|
-
// The metrics rollup and the block fetch are independent, so run them concurrently
|
|
3260
|
-
// — the rollup adds no serial latency to the (frequent) emit path.
|
|
3261
|
-
const [, block] = await Promise.all([
|
|
3262
|
-
this.attachStepMetrics(workspaceId, instance),
|
|
3263
|
-
this.blockRepository.get(workspaceId, instance.blockId),
|
|
3264
|
-
]);
|
|
3265
|
-
await this.events.executionChanged(workspaceId, instance, block);
|
|
3266
|
-
// When a run reaches a terminal state, schedule a post-run Kaizen grading for each
|
|
3267
|
-
// completed agent step (the scheduler skips verified combos + already-graded steps).
|
|
3268
|
-
// Best-effort + idempotent: a failure here must never derail the emit, and a re-emit
|
|
3269
|
-
// of an already-scheduled run is a no-op. The actual LLM grading runs later in the
|
|
3270
|
-
// background sweep, so this only does cheap inserts.
|
|
3271
|
-
if (this.kaizenScheduler && (instance.status === 'done' || instance.status === 'failed')) {
|
|
3272
|
-
try {
|
|
3273
|
-
await this.kaizenScheduler.scheduleForRun(workspaceId, instance);
|
|
3274
|
-
}
|
|
3275
|
-
catch {
|
|
3276
|
-
// Swallow — grading is an observability concern and must never break a run.
|
|
3277
|
-
}
|
|
3278
|
-
}
|
|
3279
|
-
// When a run reaches a terminal state, delete its per-run personal-credential
|
|
3280
|
-
// activation immediately (individual-usage subscriptions) so the system-encrypted
|
|
3281
|
-
// token copy doesn't linger to its TTL. Best-effort + idempotent — a missing repo or
|
|
3282
|
-
// a re-emit of an already-cleared run is a no-op, and a failure here must never
|
|
3283
|
-
// derail the emit.
|
|
3284
|
-
if (this.subscriptionActivations &&
|
|
3285
|
-
(instance.status === 'done' || instance.status === 'failed')) {
|
|
3286
|
-
try {
|
|
3287
|
-
await this.subscriptionActivations.deleteByExecution(instance.id);
|
|
3288
|
-
}
|
|
3289
|
-
catch {
|
|
3290
|
-
// Swallow — a failure here must never derail the emit. This is not a silent
|
|
3291
|
-
// data-loss path: the TTL sweep reclaims the row as a backstop, and the sweep
|
|
3292
|
-
// (Worker cron / Node retention timer) logs its own errors, so a *systemic*
|
|
3293
|
-
// cleanup failure surfaces there rather than being lost here.
|
|
3294
|
-
}
|
|
3295
|
-
}
|
|
3296
|
-
}
|
|
3297
|
-
/**
|
|
3298
|
-
* Roll the run's recorded LLM calls into per-step `metrics` for the board, in
|
|
3299
|
-
* place on the emitted instance. The proxy keys calls by execution + agentKind
|
|
3300
|
-
* (not step index), so the aggregate is per-agent-kind within the run; steps
|
|
3301
|
-
* sharing a kind get the same rollup. Best-effort and a no-op when the sink is
|
|
3302
|
-
* not wired, so it never blocks an emit.
|
|
3303
|
-
*/
|
|
3304
|
-
async attachStepMetrics(workspaceId, instance) {
|
|
3305
|
-
if (!this.llmObservability)
|
|
3306
|
-
return;
|
|
3307
|
-
try {
|
|
3308
|
-
const summaries = await this.llmObservability.summarizeByExecution(workspaceId, instance.id);
|
|
3309
|
-
if (summaries.length === 0)
|
|
3310
|
-
return;
|
|
3311
|
-
const byKind = new Map(summaries.map((s) => [s.agentKind, s]));
|
|
3312
|
-
for (const step of instance.steps) {
|
|
3313
|
-
const s = byKind.get(step.agentKind);
|
|
3314
|
-
if (!s)
|
|
3315
|
-
continue;
|
|
3316
|
-
step.metrics = {
|
|
3317
|
-
calls: s.calls,
|
|
3318
|
-
promptTokens: s.promptTokens,
|
|
3319
|
-
cachedPromptTokens: s.cachedPromptTokens,
|
|
3320
|
-
completionTokens: s.completionTokens,
|
|
3321
|
-
peakCompletionTokens: s.peakCompletionTokens,
|
|
3322
|
-
maxOutputTokens: s.maxOutputTokens,
|
|
3323
|
-
truncatedCalls: s.truncatedCalls,
|
|
3324
|
-
upstreamMs: s.upstreamMs,
|
|
3325
|
-
overheadMs: s.overheadMs,
|
|
3326
|
-
errors: s.errors,
|
|
3327
|
-
warnings: s.warnings,
|
|
3328
|
-
};
|
|
3329
|
-
}
|
|
3330
|
-
}
|
|
3331
|
-
catch (error) {
|
|
3332
|
-
// Observability is best-effort; never block an emit on a metrics read.
|
|
3333
|
-
void error;
|
|
3334
|
-
}
|
|
3335
|
-
}
|
|
3336
|
-
/** Set the block's in-progress/blocked status and step-completion progress. */
|
|
3337
|
-
async updateBlockProgress(workspaceId, instance, status) {
|
|
3338
|
-
const total = instance.steps.length || 1;
|
|
3339
|
-
const done = instance.steps.filter((s) => s.state === 'done').length;
|
|
3340
|
-
await this.blockRepository.update(workspaceId, instance.blockId, {
|
|
3341
|
-
status,
|
|
3342
|
-
progress: Math.min(1, done / total),
|
|
3343
|
-
});
|
|
3344
|
-
}
|
|
3345
|
-
/**
|
|
3346
|
-
* Advance the block's step PROGRESS without touching its status — used when a step
|
|
3347
|
-
* resolver already owns the block's terminal status (the merger set `done`/`pr_ready`)
|
|
3348
|
-
* and a trailing step still follows, so the bar moves on without downgrading that status.
|
|
3349
|
-
*/
|
|
3350
|
-
async refreshBlockProgress(workspaceId, instance) {
|
|
3351
|
-
const total = instance.steps.length || 1;
|
|
3352
|
-
const done = instance.steps.filter((s) => s.state === 'done').length;
|
|
3353
|
-
await this.blockRepository.update(workspaceId, instance.blockId, {
|
|
3354
|
-
progress: Math.min(1, done / total),
|
|
3355
|
-
});
|
|
3356
|
-
}
|
|
3357
|
-
/**
|
|
3358
|
-
* A pipeline finished. A frame becomes `done` (a mapping-only run leaves it
|
|
3359
|
-
* `ready`). A *task* never auto-`done`s from a confidence score any more — that
|
|
3360
|
-
* looked merged when the PR was still open with red CI. Instead:
|
|
3361
|
-
* - if the pipeline has a `merger` step, it already owned the merge/notify
|
|
3362
|
-
* decision (see {@link resolveMergerStep}); we only backstop a missing one;
|
|
3363
|
-
* - otherwise the work is complete but unmerged: leave the PR open (`pr_ready`)
|
|
3364
|
-
* and raise a `pipeline_complete` notification for a human to confirm + merge.
|
|
3365
|
-
* `done` now strictly means the PR was merged (see {@link finalizeMerge}).
|
|
3366
|
-
*/
|
|
3367
|
-
async finalizeBlock(workspaceId, instance, confidence) {
|
|
3368
|
-
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
3369
|
-
if (!block || block.status === 'done')
|
|
3370
|
-
return;
|
|
3371
|
-
if ((block.level ?? 'frame') !== 'task') {
|
|
3372
|
-
// A mapping-only run (just the `blueprints` step, e.g. kicked off after a
|
|
3373
|
-
// bootstrap) leaves the service frame `ready` and droppable rather than
|
|
3374
|
-
// marking the whole service "done".
|
|
3375
|
-
const mappingOnly = instance.steps.every((s) => s.agentKind === 'blueprints');
|
|
3376
|
-
await this.blockRepository.update(workspaceId, block.id, {
|
|
3377
|
-
status: mappingOnly ? 'ready' : 'done',
|
|
3378
|
-
progress: 1,
|
|
3379
|
-
});
|
|
3380
|
-
return;
|
|
3381
|
-
}
|
|
3382
|
-
// Confidence is recorded by the caller (recordStepResult) before any merge, so
|
|
3383
|
-
// it persists on both the merge and review paths; `confidence` is unused here.
|
|
3384
|
-
void confidence;
|
|
3385
|
-
const hasMerger = instance.steps.some((s) => s.agentKind === MERGER_AGENT_KIND);
|
|
3386
|
-
if (hasMerger) {
|
|
3387
|
-
// The `merger` step already merged (→ `done`) or raised a review (→ `pr_ready`).
|
|
3388
|
-
// Only backstop the case where it produced no decision at all.
|
|
3389
|
-
const fresh = await this.blockRepository.get(workspaceId, block.id);
|
|
3390
|
-
if (fresh && fresh.status !== 'done' && fresh.status !== 'pr_ready') {
|
|
3391
|
-
await this.blockRepository.update(workspaceId, block.id, {
|
|
3392
|
-
status: 'pr_ready',
|
|
3393
|
-
progress: 1,
|
|
3394
|
-
});
|
|
3395
|
-
}
|
|
3396
|
-
return;
|
|
3397
|
-
}
|
|
3398
|
-
// No merger in this pipeline: complete but unmerged — ask a human to confirm.
|
|
3399
|
-
await this.blockRepository.update(workspaceId, block.id, { status: 'pr_ready', progress: 1 });
|
|
3400
|
-
await this.raisePipelineComplete(workspaceId, instance, block);
|
|
3401
|
-
}
|
|
3402
3100
|
/**
|
|
3403
3101
|
* Merge a block's PR for real, then mark it `done`. The remote merge happens
|
|
3404
3102
|
* FIRST (via the {@link PullRequestMerger} port) and only on its success does the
|
|
@@ -3501,23 +3199,6 @@ export class ExecutionService {
|
|
|
3501
3199
|
}
|
|
3502
3200
|
return DEFAULT_MERGE_PRESET;
|
|
3503
3201
|
}
|
|
3504
|
-
/** Raise a `pipeline_complete` notification for a no-merger run awaiting confirmation. */
|
|
3505
|
-
async raisePipelineComplete(workspaceId, instance, block) {
|
|
3506
|
-
if (!this.notificationService)
|
|
3507
|
-
return;
|
|
3508
|
-
await this.notificationService.raise(workspaceId, {
|
|
3509
|
-
type: 'pipeline_complete',
|
|
3510
|
-
blockId: block.id,
|
|
3511
|
-
executionId: instance.id,
|
|
3512
|
-
title: `Confirm "${block.title}" is complete`,
|
|
3513
|
-
body: `The "${instance.pipelineName}" pipeline finished and opened a PR, but it has no ` +
|
|
3514
|
-
`merger step. Review the work and confirm it as complete (this merges the PR).`,
|
|
3515
|
-
payload: {
|
|
3516
|
-
...(block.pullRequest?.url ? { prUrl: block.pullRequest.url } : {}),
|
|
3517
|
-
pipelineName: instance.pipelineName,
|
|
3518
|
-
},
|
|
3519
|
-
});
|
|
3520
|
-
}
|
|
3521
3202
|
/**
|
|
3522
3203
|
* Implementing a task assigned to a module materialises that module: create it
|
|
3523
3204
|
* in the service if missing, then move the task inside it.
|
|
@@ -3559,13 +3240,13 @@ export class ExecutionService {
|
|
|
3559
3240
|
this.stepGraph.startStep(step);
|
|
3560
3241
|
if (instance.status === 'blocked')
|
|
3561
3242
|
instance.status = 'running';
|
|
3562
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
3243
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
3563
3244
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
3564
3245
|
// Wake the parked durable run, if any. The DB write above remains the source
|
|
3565
3246
|
// of truth (so the backstop sweeper can still re-drive it); the signal is an
|
|
3566
3247
|
// optimisation that lets the workflow continue immediately.
|
|
3567
3248
|
await this.workRunner.signalDecision(workspaceId, instance.id, decisionId, choice);
|
|
3568
|
-
await this.emitInstance(workspaceId, instance);
|
|
3249
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3569
3250
|
return instance;
|
|
3570
3251
|
}
|
|
3571
3252
|
/**
|
|
@@ -3593,7 +3274,7 @@ export class ExecutionService {
|
|
|
3593
3274
|
}
|
|
3594
3275
|
step.approval.status = 'approved';
|
|
3595
3276
|
// A gate is never raised on the final step, but the shared advance stays defensive.
|
|
3596
|
-
await this.advancePastResolvedGate(workspaceId, instance, stepIndex);
|
|
3277
|
+
await this.runStateMachine.advancePastResolvedGate(workspaceId, instance, stepIndex);
|
|
3597
3278
|
return instance;
|
|
3598
3279
|
}
|
|
3599
3280
|
/**
|
|
@@ -3654,7 +3335,7 @@ export class ExecutionService {
|
|
|
3654
3335
|
instance.status = 'running';
|
|
3655
3336
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
3656
3337
|
await this.workRunner.signalDecision(workspaceId, instance.id, approvalId, 'changes_requested');
|
|
3657
|
-
await this.emitInstance(workspaceId, instance);
|
|
3338
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3658
3339
|
return instance;
|
|
3659
3340
|
}
|
|
3660
3341
|
}
|
|
@@ -3670,7 +3351,7 @@ export class ExecutionService {
|
|
|
3670
3351
|
instance.status = 'running';
|
|
3671
3352
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
3672
3353
|
await this.workRunner.signalDecision(workspaceId, instance.id, approvalId, 'changes_requested');
|
|
3673
|
-
await this.emitInstance(workspaceId, instance);
|
|
3354
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3674
3355
|
return instance;
|
|
3675
3356
|
}
|
|
3676
3357
|
/**
|
|
@@ -3731,42 +3412,8 @@ export class ExecutionService {
|
|
|
3731
3412
|
* durable driver once a step has exhausted its retries (or a job/decision
|
|
3732
3413
|
* faulted); `kind` classifies the cause so the right hint is shown.
|
|
3733
3414
|
*/
|
|
3734
|
-
|
|
3735
|
-
|
|
3736
|
-
if (!instance)
|
|
3737
|
-
return;
|
|
3738
|
-
// Reclaim the per-run container on the failure path too: a failed run otherwise
|
|
3739
|
-
// leaves its container to idle out sleepAfter. This is the single funnel for
|
|
3740
|
-
// every failure kind (job_failed from the driver, the spend/decision timeouts,
|
|
3741
|
-
// and the user-facing stopRun, which already reclaimed — the call is idempotent).
|
|
3742
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
3743
|
-
// The FIRST recorded failure wins: a run already in a terminal `failed` state keeps
|
|
3744
|
-
// its existing (richest) failure rather than being overwritten. An inline gate that
|
|
3745
|
-
// knows the precise kind/detail returns a `job_failed` result the driver funnels here,
|
|
3746
|
-
// so there should only ever be one write — but this guards against a future path that
|
|
3747
|
-
// both records a failure and returns `job_failed`, which would otherwise clobber the
|
|
3748
|
-
// good record with a generic one (the companion-rejected regression).
|
|
3749
|
-
if (instance.status === 'failed')
|
|
3750
|
-
return;
|
|
3751
|
-
const failure = {
|
|
3752
|
-
kind,
|
|
3753
|
-
message,
|
|
3754
|
-
detail,
|
|
3755
|
-
hint: EXECUTION_FAILURE_HINTS[kind] ?? null,
|
|
3756
|
-
occurredAt: this.clock.now(),
|
|
3757
|
-
lastSubtasks: instance.steps[instance.currentStep]?.subtasks ?? null,
|
|
3758
|
-
};
|
|
3759
|
-
await this.executionRepository.markFailed(workspaceId, executionId, failure);
|
|
3760
|
-
// Progress reflects how far the pipeline got before failing.
|
|
3761
|
-
const done = instance.steps.filter((s) => s.state === 'done').length;
|
|
3762
|
-
const progress = instance.steps.length > 0 ? done / instance.steps.length : 0;
|
|
3763
|
-
await this.blockRepository.update(workspaceId, instance.blockId, {
|
|
3764
|
-
status: 'blocked',
|
|
3765
|
-
progress,
|
|
3766
|
-
});
|
|
3767
|
-
const failed = await this.executionRepository.get(workspaceId, executionId);
|
|
3768
|
-
if (failed)
|
|
3769
|
-
await this.emitInstance(workspaceId, failed);
|
|
3415
|
+
failRun(workspaceId, executionId, message, kind = 'agent', detail = null) {
|
|
3416
|
+
return this.runStateMachine.failRun(workspaceId, executionId, message, kind, detail);
|
|
3770
3417
|
}
|
|
3771
3418
|
/**
|
|
3772
3419
|
* Retry a failed run: re-drive the same pipeline on the same block, **resuming
|
|
@@ -3827,7 +3474,7 @@ export class ExecutionService {
|
|
|
3827
3474
|
executionId: instance.id,
|
|
3828
3475
|
});
|
|
3829
3476
|
await this.workRunner.startRun(workspaceId, instance.id);
|
|
3830
|
-
await this.emitInstance(workspaceId, instance);
|
|
3477
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3831
3478
|
return instance;
|
|
3832
3479
|
}
|
|
3833
3480
|
/**
|
|
@@ -3874,7 +3521,7 @@ export class ExecutionService {
|
|
|
3874
3521
|
// container AND its durable driver — before minting the restart. A `done`/`failed`
|
|
3875
3522
|
// run is already terminal (a no-op teardown), but a still-`running` run would
|
|
3876
3523
|
// otherwise leak a container and a live Workflows/pg-boss driver.
|
|
3877
|
-
await this.stopRunContainer(workspaceId, previous);
|
|
3524
|
+
await this.runStateMachine.stopRunContainer(workspaceId, previous);
|
|
3878
3525
|
await this.workRunner.cancelRun(workspaceId, executionId);
|
|
3879
3526
|
const { steps, currentStep } = planRestartFromStep(previous, fromStepIndex);
|
|
3880
3527
|
// Mint the activation before replacing the prior run, so a bad password aborts the
|
|
@@ -3900,7 +3547,7 @@ export class ExecutionService {
|
|
|
3900
3547
|
executionId: instance.id,
|
|
3901
3548
|
});
|
|
3902
3549
|
await this.workRunner.startRun(workspaceId, instance.id);
|
|
3903
|
-
await this.emitInstance(workspaceId, instance);
|
|
3550
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3904
3551
|
return instance;
|
|
3905
3552
|
}
|
|
3906
3553
|
/**
|
|
@@ -3916,7 +3563,7 @@ export class ExecutionService {
|
|
|
3916
3563
|
instance.status = 'running';
|
|
3917
3564
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
3918
3565
|
await this.workRunner.startRun(workspaceId, instance.id);
|
|
3919
|
-
await this.emitInstance(workspaceId, instance);
|
|
3566
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3920
3567
|
}
|
|
3921
3568
|
return this.executionRepository.listByWorkspace(workspaceId);
|
|
3922
3569
|
}
|
|
@@ -3928,7 +3575,7 @@ export class ExecutionService {
|
|
|
3928
3575
|
// the record, so a cancel never leaves a container running until its watchdog.
|
|
3929
3576
|
const existing = await this.executionRepository.getByBlock(workspaceId, blockId);
|
|
3930
3577
|
if (existing) {
|
|
3931
|
-
await this.stopRunContainer(workspaceId, existing);
|
|
3578
|
+
await this.runStateMachine.stopRunContainer(workspaceId, existing);
|
|
3932
3579
|
await this.workRunner.cancelRun(workspaceId, existing.id);
|
|
3933
3580
|
}
|
|
3934
3581
|
await this.executionRepository.deleteByBlock(workspaceId, blockId);
|
|
@@ -3956,7 +3603,7 @@ export class ExecutionService {
|
|
|
3956
3603
|
const instance = assertFound(await this.executionRepository.get(workspaceId, executionId), 'Execution', executionId);
|
|
3957
3604
|
if (instance.status === 'failed' || instance.status === 'done')
|
|
3958
3605
|
return instance;
|
|
3959
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
3606
|
+
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
3960
3607
|
await this.workRunner.cancelRun(workspaceId, executionId);
|
|
3961
3608
|
await this.failRun(workspaceId, executionId, opts.reason ?? 'Stopped by the user.', opts.kind ?? 'cancelled');
|
|
3962
3609
|
return assertFound(await this.executionRepository.get(workspaceId, executionId), 'Execution', executionId);
|
|
@@ -3973,33 +3620,10 @@ export class ExecutionService {
|
|
|
3973
3620
|
const run = await this.executionRepository.getByBlock(workspaceId, blockId);
|
|
3974
3621
|
if (!run)
|
|
3975
3622
|
continue;
|
|
3976
|
-
await this.stopRunContainer(workspaceId, run);
|
|
3623
|
+
await this.runStateMachine.stopRunContainer(workspaceId, run);
|
|
3977
3624
|
await this.workRunner.cancelRun(workspaceId, run.id);
|
|
3978
3625
|
await this.executionRepository.deleteByBlock(workspaceId, blockId);
|
|
3979
3626
|
}
|
|
3980
3627
|
}
|
|
3981
|
-
/**
|
|
3982
|
-
* Best-effort: reclaim the per-run container backing an execution. The container is
|
|
3983
|
-
* addressed by the run (execution) id, so a backend that shares one across the run
|
|
3984
|
-
* (Cloudflare, local Docker) tears the whole thing down. A per-job backend (a
|
|
3985
|
-
* self-hosted pool) has no run container, so it cancels the run's IN-FLIGHT step job
|
|
3986
|
-
* instead — hence we pass the current step's job id alongside the run id. A no-op for
|
|
3987
|
-
* inline executors (no `stopJob`) and for an already-gone container/job; never
|
|
3988
|
-
* throws, so it can't derail the teardown that calls it.
|
|
3989
|
-
*/
|
|
3990
|
-
async stopRunContainer(workspaceId, instance) {
|
|
3991
|
-
const executor = this.agentExecutor;
|
|
3992
|
-
if (!isAsyncAgentExecutor(executor) || !executor.stopJob)
|
|
3993
|
-
return;
|
|
3994
|
-
// The in-flight step's job id (when a job is parked), so a per-job backend can
|
|
3995
|
-
// cancel exactly it; the run-container backends ignore it and use the run id.
|
|
3996
|
-
const jobId = instance.steps[instance.currentStep]?.jobId ?? instance.id;
|
|
3997
|
-
try {
|
|
3998
|
-
await executor.stopJob({ jobId, runId: instance.id, workspaceId });
|
|
3999
|
-
}
|
|
4000
|
-
catch {
|
|
4001
|
-
// The container may already be gone (eviction/completion) — nothing to reclaim.
|
|
4002
|
-
}
|
|
4003
|
-
}
|
|
4004
3628
|
}
|
|
4005
3629
|
//# sourceMappingURL=ExecutionService.js.map
|