@cat-factory/orchestration 0.37.0 → 0.37.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/execution/CompanionController.d.ts +6 -11
- package/dist/modules/execution/CompanionController.d.ts.map +1 -1
- package/dist/modules/execution/CompanionController.js +20 -20
- package/dist/modules/execution/CompanionController.js.map +1 -1
- package/dist/modules/execution/ExecutionService.d.ts +5 -151
- package/dist/modules/execution/ExecutionService.d.ts.map +1 -1
- package/dist/modules/execution/ExecutionService.js +103 -578
- package/dist/modules/execution/ExecutionService.js.map +1 -1
- package/dist/modules/execution/HumanTestController.d.ts +6 -8
- package/dist/modules/execution/HumanTestController.d.ts.map +1 -1
- package/dist/modules/execution/HumanTestController.js +26 -26
- package/dist/modules/execution/HumanTestController.js.map +1 -1
- package/dist/modules/execution/ReviewGateController.d.ts +11 -19
- package/dist/modules/execution/ReviewGateController.d.ts.map +1 -1
- package/dist/modules/execution/ReviewGateController.js +19 -19
- package/dist/modules/execution/ReviewGateController.js.map +1 -1
- package/dist/modules/execution/RunStateMachine.d.ts +131 -0
- package/dist/modules/execution/RunStateMachine.d.ts.map +1 -0
- package/dist/modules/execution/RunStateMachine.js +389 -0
- package/dist/modules/execution/RunStateMachine.js.map +1 -0
- package/dist/modules/execution/StepGraph.d.ts +70 -0
- package/dist/modules/execution/StepGraph.d.ts.map +1 -0
- package/dist/modules/execution/StepGraph.js +124 -0
- package/dist/modules/execution/StepGraph.js.map +1 -0
- package/dist/modules/execution/TesterController.d.ts +3 -4
- package/dist/modules/execution/TesterController.d.ts.map +1 -1
- package/dist/modules/execution/TesterController.js +6 -6
- package/dist/modules/execution/TesterController.js.map +1 -1
- package/dist/modules/execution/VisualConfirmationController.d.ts +6 -8
- package/dist/modules/execution/VisualConfirmationController.d.ts.map +1 -1
- package/dist/modules/execution/VisualConfirmationController.js +18 -18
- package/dist/modules/execution/VisualConfirmationController.js.map +1 -1
- package/package.json +1 -1
|
@@ -11,6 +11,8 @@ import { CONFLICTS_AGENT_KIND, MERGER_AGENT_KIND, REQUIREMENTS_REVIEW_AGENT_KIND
|
|
|
11
11
|
import { DEFAULT_FOLLOW_UP_MAX_LOOPS, FOLLOW_UP_PRODUCER_KIND, followUpsToSendBack, hasPendingFollowUps, renderFollowUpRework, shouldLoopCoder, } from './followUp.logic.js';
|
|
12
12
|
import { AgentContextBuilder, } from './AgentContextBuilder.js';
|
|
13
13
|
import { CompanionController } from './CompanionController.js';
|
|
14
|
+
import { StepGraph } from './StepGraph.js';
|
|
15
|
+
import { RunStateMachine } from './RunStateMachine.js';
|
|
14
16
|
import { inferTechnicalLabel } from './technical.logic.js';
|
|
15
17
|
import { MergeResolver } from './MergeResolver.js';
|
|
16
18
|
import { ReviewGateController } from './ReviewGateController.js';
|
|
@@ -26,22 +28,6 @@ import { requireWorkspace } from '@cat-factory/kernel';
|
|
|
26
28
|
import { planResumedSteps, planRestartFromStep } from './retry.logic.js';
|
|
27
29
|
import { isContainerEvictionError, isTransientEviction, MAX_EVICTION_RECOVERIES, MAX_TRANSIENT_EVICTION_RECOVERIES, } from './job.logic.js';
|
|
28
30
|
import { decideTesterInfra, resolveTesterEnvironment, TESTER_INFRA_MESSAGES, } from './tester-infra.logic.js';
|
|
29
|
-
/**
|
|
30
|
-
* "What to do next" guidance per failure kind a pipeline run can produce, shown
|
|
31
|
-
* under the failure banner on the board (mirrors bootstrap's FAILURE_HINTS). Only
|
|
32
|
-
* the execution-relevant subset of {@link AgentFailureKind} is keyed.
|
|
33
|
-
*/
|
|
34
|
-
const EXECUTION_FAILURE_HINTS = {
|
|
35
|
-
agent: 'An agent step failed after its automatic retries. Review the run, then retry to re-run the pipeline.',
|
|
36
|
-
job_failed: 'The implementation container reported a failure. Inspect its logs (Cloudflare Workers Observability, filtered by the run id), then retry to spin a fresh container.',
|
|
37
|
-
evicted: 'The implementation container kept vanishing mid-run even after automatic fresh-container restarts. Most often this is transient: a deploy / new-version rollout draining the container, in which case simply retrying once the rollout has finished succeeds. If it persists, it points at a memory or crash issue on the run — inspect its logs (Cloudflare Workers Observability, filtered by the run id) and consider a heavier container instance type. Retry to try again.',
|
|
38
|
-
timeout: 'The run exceeded its time budget — a step or the implementation job did not finish in time. Retry to start it again.',
|
|
39
|
-
rejected: 'You rejected this step’s proposal, stopping the run. Retry to re-run the pipeline from the rejected step.',
|
|
40
|
-
companion_rejected: 'A companion agent could not return a usable quality assessment (its reply was truncated or malformed) even after a repair retry. Review the companion’s raw output on the run, then retry.',
|
|
41
|
-
cancelled: 'You stopped this run; its container was killed. Retry to start it again.',
|
|
42
|
-
dispatch: 'The agent’s container could not be started — the run never began executing. The provider/runtime’s verbatim response is shown below. Most often this is transient (a capacity blip or a new-version rollout); retrying spins a fresh container. If it persists it points at a misconfigured container binding/image or runner pool. Retry to try again.',
|
|
43
|
-
unknown: 'The run failed for an unclassified reason. Review the run, then retry.',
|
|
44
|
-
};
|
|
45
31
|
/**
|
|
46
32
|
* Step kinds whose run details surface the ephemeral-environment lifecycle: the
|
|
47
33
|
* `deployer` provisions it and the `tester`/`playwright` exercise it. Used to gate
|
|
@@ -93,13 +79,16 @@ export class ExecutionService {
|
|
|
93
79
|
accountRepository;
|
|
94
80
|
idGenerator;
|
|
95
81
|
clock;
|
|
82
|
+
/** The pure step/cursor mutators (start/finish/park/reset + the companion rework loop). */
|
|
83
|
+
stepGraph;
|
|
84
|
+
/** The async instance/block state-machine spine (persist/emit/park/advance/finalize/fail). */
|
|
85
|
+
runStateMachine;
|
|
96
86
|
agentExecutor;
|
|
97
87
|
workRunner;
|
|
98
88
|
events;
|
|
99
89
|
board;
|
|
100
90
|
spend;
|
|
101
91
|
requirementReviewService;
|
|
102
|
-
kaizenScheduler;
|
|
103
92
|
clarityReviewService;
|
|
104
93
|
brainstormServices;
|
|
105
94
|
environmentProvisioning;
|
|
@@ -129,7 +118,6 @@ export class ExecutionService {
|
|
|
129
118
|
blueprintReconciler;
|
|
130
119
|
notificationService;
|
|
131
120
|
workspaceSettingsService;
|
|
132
|
-
llmObservability;
|
|
133
121
|
prMerger;
|
|
134
122
|
mergePresetRepository;
|
|
135
123
|
ticketTrackerProvider;
|
|
@@ -177,13 +165,27 @@ export class ExecutionService {
|
|
|
177
165
|
this.accountRepository = accountRepository;
|
|
178
166
|
this.idGenerator = idGenerator;
|
|
179
167
|
this.clock = clock;
|
|
168
|
+
this.stepGraph = new StepGraph(clock);
|
|
169
|
+
this.runStateMachine = new RunStateMachine({
|
|
170
|
+
executionRepository,
|
|
171
|
+
blockRepository,
|
|
172
|
+
events: executionEventPublisher,
|
|
173
|
+
workRunner,
|
|
174
|
+
agentExecutor,
|
|
175
|
+
idGenerator,
|
|
176
|
+
clock,
|
|
177
|
+
stepGraph: this.stepGraph,
|
|
178
|
+
notificationService,
|
|
179
|
+
kaizenScheduler,
|
|
180
|
+
subscriptionActivations: subscriptionActivationRepository,
|
|
181
|
+
llmObservability,
|
|
182
|
+
});
|
|
180
183
|
this.agentExecutor = agentExecutor;
|
|
181
184
|
this.workRunner = workRunner;
|
|
182
185
|
this.events = executionEventPublisher;
|
|
183
186
|
this.board = boardService;
|
|
184
187
|
this.spend = spendService;
|
|
185
188
|
this.requirementReviewService = requirementReviewService;
|
|
186
|
-
this.kaizenScheduler = kaizenScheduler;
|
|
187
189
|
this.clarityReviewService = clarityReviewService;
|
|
188
190
|
this.brainstormServices = brainstormServices;
|
|
189
191
|
this.environmentProvisioning = environmentProvisioning;
|
|
@@ -215,17 +217,8 @@ export class ExecutionService {
|
|
|
215
217
|
idGenerator,
|
|
216
218
|
previewStepModel: (ctx) => this.previewStepModel(ctx),
|
|
217
219
|
runAgent: (ctx, opts) => this.runAgent(ctx, opts),
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
pauseStepForInput: (s) => this.pauseStepForInput(s),
|
|
221
|
-
updateBlockProgress: (ws, i, st) => this.updateBlockProgress(ws, i, st),
|
|
222
|
-
persistInstance: (ws, i) => this.executionRepository.upsert(ws, i),
|
|
223
|
-
emitInstance: (ws, i) => this.emitInstance(ws, i),
|
|
224
|
-
stopRunContainer: (ws, i) => this.stopRunContainer(ws, i),
|
|
225
|
-
finalizeBlock: (ws, i, c) => this.finalizeBlock(ws, i, c),
|
|
226
|
-
parkStepOnDecision: (ws, i, s, p) => this.parkStepOnDecision(ws, i, s, p),
|
|
227
|
-
raiseDecisionRequired: (ws, i) => this.raiseDecisionRequired(ws, i),
|
|
228
|
-
loopCompanionProducer: (i, ci, rw) => this.loopCompanionProducer(i, ci, rw),
|
|
220
|
+
stateMachine: this.runStateMachine,
|
|
221
|
+
stepGraph: this.stepGraph,
|
|
229
222
|
inferTechnicalLabel: (ws, block, producer, companionStep) => this.inferBlockTechnical(ws, block, producer, companionStep),
|
|
230
223
|
});
|
|
231
224
|
this.testerController = new TesterController({
|
|
@@ -234,9 +227,7 @@ export class ExecutionService {
|
|
|
234
227
|
agentExecutor,
|
|
235
228
|
contextBuilder: this.contextBuilder,
|
|
236
229
|
resolveMergePreset: (ws, block) => this.resolveMergePreset(ws, block),
|
|
237
|
-
|
|
238
|
-
persistInstance: (ws, i) => this.executionRepository.upsert(ws, i),
|
|
239
|
-
emitInstance: (ws, i) => this.emitInstance(ws, i),
|
|
230
|
+
stateMachine: this.runStateMachine,
|
|
240
231
|
});
|
|
241
232
|
this.humanTestController = new HumanTestController({
|
|
242
233
|
blockRepository,
|
|
@@ -269,14 +260,8 @@ export class ExecutionService {
|
|
|
269
260
|
: {}),
|
|
270
261
|
...(branchUpdater ? { branchUpdater } : {}),
|
|
271
262
|
resolveMergePreset: (ws, block) => this.resolveMergePreset(ws, block),
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
startStep: (s) => this.startStep(s),
|
|
275
|
-
updateBlockProgress: (ws, i, st) => this.updateBlockProgress(ws, i, st),
|
|
276
|
-
finalizeBlock: (ws, i, c) => this.finalizeBlock(ws, i, c),
|
|
277
|
-
stopRunContainer: (ws, i) => this.stopRunContainer(ws, i),
|
|
278
|
-
persistInstance: (ws, i) => this.executionRepository.upsert(ws, i),
|
|
279
|
-
emitInstance: (ws, i) => this.emitInstance(ws, i),
|
|
263
|
+
stateMachine: this.runStateMachine,
|
|
264
|
+
stepGraph: this.stepGraph,
|
|
280
265
|
clockNow: () => this.clock.now(),
|
|
281
266
|
});
|
|
282
267
|
this.visualConfirmationController = new VisualConfirmationController({
|
|
@@ -288,32 +273,18 @@ export class ExecutionService {
|
|
|
288
273
|
notificationService,
|
|
289
274
|
...(resolveBinaryArtifactStore ? { resolveBinaryArtifactStore } : {}),
|
|
290
275
|
resolveMergePreset: (ws, block) => this.resolveMergePreset(ws, block),
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
startStep: (s) => this.startStep(s),
|
|
294
|
-
updateBlockProgress: (ws, i, st) => this.updateBlockProgress(ws, i, st),
|
|
295
|
-
finalizeBlock: (ws, i, c) => this.finalizeBlock(ws, i, c),
|
|
296
|
-
stopRunContainer: (ws, i) => this.stopRunContainer(ws, i),
|
|
297
|
-
persistInstance: (ws, i) => this.executionRepository.upsert(ws, i),
|
|
298
|
-
emitInstance: (ws, i) => this.emitInstance(ws, i),
|
|
276
|
+
stateMachine: this.runStateMachine,
|
|
277
|
+
stepGraph: this.stepGraph,
|
|
299
278
|
clockNow: () => this.clock.now(),
|
|
300
279
|
});
|
|
301
280
|
this.reviewGate = new ReviewGateController({
|
|
302
281
|
blockRepository,
|
|
303
282
|
executionRepository,
|
|
304
283
|
workRunner,
|
|
284
|
+
stateMachine: this.runStateMachine,
|
|
285
|
+
stepGraph: this.stepGraph,
|
|
305
286
|
resolveMergePreset: (ws, block) => this.resolveMergePreset(ws, block),
|
|
306
|
-
parkStepOnDecision: (ws, i, s, p) => this.parkStepOnDecision(ws, i, s, p),
|
|
307
|
-
advancePastResolvedGate: (ws, i, idx) => this.advancePastResolvedGate(ws, i, idx),
|
|
308
287
|
dispatchIterationCap: (ws, blockId, choice, handlers) => this.dispatchIterationCap(ws, blockId, choice, handlers),
|
|
309
|
-
raiseDecisionRequired: (ws, i) => this.raiseDecisionRequired(ws, i),
|
|
310
|
-
finishStep: (s) => this.finishStep(s),
|
|
311
|
-
startStep: (s) => this.startStep(s),
|
|
312
|
-
updateBlockProgress: (ws, i, st) => this.updateBlockProgress(ws, i, st),
|
|
313
|
-
finalizeBlock: (ws, i, c) => this.finalizeBlock(ws, i, c),
|
|
314
|
-
stopRunContainer: (ws, i) => this.stopRunContainer(ws, i),
|
|
315
|
-
persistInstance: (ws, i) => this.executionRepository.upsert(ws, i),
|
|
316
|
-
emitInstance: (ws, i) => this.emitInstance(ws, i),
|
|
317
288
|
});
|
|
318
289
|
this.requirementsKind = this.buildRequirementsKind();
|
|
319
290
|
this.clarityKind = this.buildClarityKind();
|
|
@@ -322,7 +293,6 @@ export class ExecutionService {
|
|
|
322
293
|
this.blueprintReconciler = blueprintReconciler;
|
|
323
294
|
this.notificationService = notificationService;
|
|
324
295
|
this.workspaceSettingsService = workspaceSettingsService;
|
|
325
|
-
this.llmObservability = llmObservability;
|
|
326
296
|
this.prMerger = pullRequestMerger;
|
|
327
297
|
this.mergePresetRepository = mergePresetRepository;
|
|
328
298
|
this.ticketTrackerProvider = ticketTrackerProvider;
|
|
@@ -632,7 +602,7 @@ export class ExecutionService {
|
|
|
632
602
|
// a browser open. With the no-op runner (tests) this does nothing and the run
|
|
633
603
|
// is advanced directly via advanceInstance.
|
|
634
604
|
await this.workRunner.startRun(workspaceId, instance.id);
|
|
635
|
-
await this.emitInstance(workspaceId, instance);
|
|
605
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
636
606
|
return instance;
|
|
637
607
|
}
|
|
638
608
|
/**
|
|
@@ -755,10 +725,10 @@ export class ExecutionService {
|
|
|
755
725
|
// auto-passed, or the run reached a terminal state) clear that waiting card so the
|
|
756
726
|
// escalation sweep can't later flip a settled decision red ("Overdue").
|
|
757
727
|
if (result.kind === 'awaiting_decision') {
|
|
758
|
-
await this.ensureWaitingNotification(workspaceId, instance);
|
|
728
|
+
await this.runStateMachine.ensureWaitingNotification(workspaceId, instance);
|
|
759
729
|
}
|
|
760
730
|
else {
|
|
761
|
-
await this.clearWaitingNotification(workspaceId, instance);
|
|
731
|
+
await this.runStateMachine.clearWaitingNotification(workspaceId, instance);
|
|
762
732
|
}
|
|
763
733
|
return result;
|
|
764
734
|
}
|
|
@@ -780,7 +750,7 @@ export class ExecutionService {
|
|
|
780
750
|
if (instance.status !== 'paused') {
|
|
781
751
|
instance.status = 'paused';
|
|
782
752
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
783
|
-
await this.emitInstance(workspaceId, instance);
|
|
753
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
784
754
|
}
|
|
785
755
|
return { kind: 'paused' };
|
|
786
756
|
}
|
|
@@ -814,12 +784,12 @@ export class ExecutionService {
|
|
|
814
784
|
if (pendingId) {
|
|
815
785
|
instance.status = 'blocked';
|
|
816
786
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
817
|
-
await this.emitInstance(workspaceId, instance);
|
|
787
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
818
788
|
return { kind: 'awaiting_decision', decisionId: pendingId };
|
|
819
789
|
}
|
|
820
790
|
}
|
|
821
791
|
}
|
|
822
|
-
this.startStep(step);
|
|
792
|
+
this.stepGraph.startStep(step);
|
|
823
793
|
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
824
794
|
if (!block)
|
|
825
795
|
return { kind: 'noop' };
|
|
@@ -887,7 +857,7 @@ export class ExecutionService {
|
|
|
887
857
|
// phase, so a run's details show the env spinning up next to the container.
|
|
888
858
|
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step);
|
|
889
859
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
890
|
-
await this.emitInstance(workspaceId, instance);
|
|
860
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
891
861
|
let handle;
|
|
892
862
|
try {
|
|
893
863
|
handle = await executor.startJob(context);
|
|
@@ -900,7 +870,7 @@ export class ExecutionService {
|
|
|
900
870
|
// "run failed". A dispatch-time eviction still routes to the evicted framing.
|
|
901
871
|
step.startingContainer = false;
|
|
902
872
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
903
|
-
await this.emitInstance(workspaceId, instance);
|
|
873
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
904
874
|
const message = getErrorMessage(error);
|
|
905
875
|
const evicted = isContainerEvictionError(message);
|
|
906
876
|
return {
|
|
@@ -917,7 +887,7 @@ export class ExecutionService {
|
|
|
917
887
|
// The dispatch returned, so the container is up and execution has begun.
|
|
918
888
|
step.startingContainer = false;
|
|
919
889
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
920
|
-
await this.emitInstance(workspaceId, instance);
|
|
890
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
921
891
|
}
|
|
922
892
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
923
893
|
}
|
|
@@ -928,7 +898,7 @@ export class ExecutionService {
|
|
|
928
898
|
if (previewModel && previewModel !== step.model) {
|
|
929
899
|
step.model = previewModel;
|
|
930
900
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
931
|
-
await this.emitInstance(workspaceId, instance);
|
|
901
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
932
902
|
}
|
|
933
903
|
const result = await this.runAgent(context, options);
|
|
934
904
|
return this.recordStepResult(workspaceId, instance, step, isFinalStep, result);
|
|
@@ -1065,7 +1035,7 @@ export class ExecutionService {
|
|
|
1065
1035
|
}
|
|
1066
1036
|
if (changed) {
|
|
1067
1037
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1068
|
-
await this.emitInstance(workspaceId, instance);
|
|
1038
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1069
1039
|
}
|
|
1070
1040
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
1071
1041
|
}
|
|
@@ -1154,7 +1124,7 @@ export class ExecutionService {
|
|
|
1154
1124
|
if (step.gate)
|
|
1155
1125
|
step.gate.phase = 'checking';
|
|
1156
1126
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1157
|
-
await this.emitInstance(workspaceId, instance);
|
|
1127
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1158
1128
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
1159
1129
|
}
|
|
1160
1130
|
// A `tester` step in its `fixing` phase has a Fixer job in flight, NOT the
|
|
@@ -1171,7 +1141,7 @@ export class ExecutionService {
|
|
|
1171
1141
|
// Reclaim the finished Fixer container before re-dispatching the Tester so it
|
|
1172
1142
|
// boots fresh against the just-pushed fixes (rather than re-attaching to the
|
|
1173
1143
|
// completed job by run id).
|
|
1174
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
1144
|
+
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
1175
1145
|
return this.testerController.dispatchTester(workspaceId, instance, step, block);
|
|
1176
1146
|
}
|
|
1177
1147
|
// A `human-test` gate in its `fixing` / `resolving_conflicts` phase has a helper job
|
|
@@ -1221,7 +1191,7 @@ export class ExecutionService {
|
|
|
1221
1191
|
step.subtasks = undefined;
|
|
1222
1192
|
step.progress = 0;
|
|
1223
1193
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1224
|
-
await this.emitInstance(workspaceId, instance);
|
|
1194
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1225
1195
|
return { kind: 'continue' };
|
|
1226
1196
|
}
|
|
1227
1197
|
return {
|
|
@@ -1388,29 +1358,6 @@ export class ExecutionService {
|
|
|
1388
1358
|
return false;
|
|
1389
1359
|
}
|
|
1390
1360
|
}
|
|
1391
|
-
startStep(step) {
|
|
1392
|
-
step.state = 'working';
|
|
1393
|
-
if (step.startedAt == null)
|
|
1394
|
-
step.startedAt = this.clock.now();
|
|
1395
|
-
// (Re)entering `working` means the step is no longer parked on a human: resume
|
|
1396
|
-
// its duration clock (see {@link pauseStepForInput}).
|
|
1397
|
-
step.pausedAt = null;
|
|
1398
|
-
}
|
|
1399
|
-
/**
|
|
1400
|
-
* Transition a step into `done`, stamping its finish time once. Set-once so the
|
|
1401
|
-
* approval-gate flow (which re-asserts `done` after a human approves, long after
|
|
1402
|
-
* the agent actually finished) keeps the agent's true completion time, and so a
|
|
1403
|
-
* replay doesn't move it. With {@link startStep}'s `startedAt` this yields the
|
|
1404
|
-
* step's execution duration. A step finished directly out of a parked approval
|
|
1405
|
-
* stopped *working* when it parked, so its duration is billed to the pause instant
|
|
1406
|
-
* ({@link pauseStepForInput}), not the (later) moment the human decided.
|
|
1407
|
-
*/
|
|
1408
|
-
finishStep(step) {
|
|
1409
|
-
step.state = 'done';
|
|
1410
|
-
if (step.finishedAt == null)
|
|
1411
|
-
step.finishedAt = step.pausedAt ?? this.clock.now();
|
|
1412
|
-
step.pausedAt = null;
|
|
1413
|
-
}
|
|
1414
1361
|
/**
|
|
1415
1362
|
* Finish a gated step that was skipped (its estimate gate was not satisfied) and either
|
|
1416
1363
|
* complete the run or advance to the next step — the deterministic finish/advance tail
|
|
@@ -1423,36 +1370,24 @@ export class ExecutionService {
|
|
|
1423
1370
|
step.output = '';
|
|
1424
1371
|
step.progress = 1;
|
|
1425
1372
|
step.subtasks = undefined;
|
|
1426
|
-
this.finishStep(step);
|
|
1373
|
+
this.stepGraph.finishStep(step);
|
|
1427
1374
|
if (isFinalStep) {
|
|
1428
1375
|
instance.status = 'done';
|
|
1429
|
-
await this.finalizeBlock(workspaceId, instance, undefined);
|
|
1376
|
+
await this.runStateMachine.finalizeBlock(workspaceId, instance, undefined);
|
|
1430
1377
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1431
|
-
await this.emitInstance(workspaceId, instance);
|
|
1432
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
1378
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1379
|
+
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
1433
1380
|
return { kind: 'done' };
|
|
1434
1381
|
}
|
|
1435
1382
|
instance.currentStep += 1;
|
|
1436
1383
|
const next = instance.steps[instance.currentStep];
|
|
1437
1384
|
if (next)
|
|
1438
|
-
this.startStep(next);
|
|
1439
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
1385
|
+
this.stepGraph.startStep(next);
|
|
1386
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
1440
1387
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1441
|
-
await this.emitInstance(workspaceId, instance);
|
|
1388
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1442
1389
|
return { kind: 'continue' };
|
|
1443
1390
|
}
|
|
1444
|
-
/**
|
|
1445
|
-
* Park a step on a human decision and freeze its duration clock. Records when the
|
|
1446
|
-
* step stopped working (`pausedAt`) so elapsed time no longer accrues while it waits
|
|
1447
|
-
* for input — the symmetric counterpart of the terminal freeze on `finishedAt`.
|
|
1448
|
-
* Set-once (a Workflows replay re-parking keeps the original instant); cleared when
|
|
1449
|
-
* the step resumes ({@link startStep}) or finishes ({@link finishStep}).
|
|
1450
|
-
*/
|
|
1451
|
-
pauseStepForInput(step) {
|
|
1452
|
-
step.state = 'waiting_decision';
|
|
1453
|
-
if (step.pausedAt == null)
|
|
1454
|
-
step.pausedAt = this.clock.now();
|
|
1455
|
-
}
|
|
1456
1391
|
/**
|
|
1457
1392
|
* Infer + persist the block's `technical` label from the settled spec phase (item 5):
|
|
1458
1393
|
* combine the spec-writer's `noBusinessSpecs` determination (recorded on the producer
|
|
@@ -1496,11 +1431,11 @@ export class ExecutionService {
|
|
|
1496
1431
|
options: [...result.decision.options],
|
|
1497
1432
|
chosen: null,
|
|
1498
1433
|
};
|
|
1499
|
-
this.pauseStepForInput(step);
|
|
1434
|
+
this.stepGraph.pauseStepForInput(step);
|
|
1500
1435
|
instance.status = 'blocked';
|
|
1501
|
-
await this.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
1436
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
1502
1437
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1503
|
-
await this.emitInstance(workspaceId, instance);
|
|
1438
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1504
1439
|
return { kind: 'awaiting_decision', decisionId: step.decision.id };
|
|
1505
1440
|
}
|
|
1506
1441
|
// Completion-path interceptors short-circuit before the normal finish/advance for the
|
|
@@ -1528,7 +1463,7 @@ export class ExecutionService {
|
|
|
1528
1463
|
if (result.model)
|
|
1529
1464
|
step.model = result.model;
|
|
1530
1465
|
step.progress = 1;
|
|
1531
|
-
this.finishStep(step);
|
|
1466
|
+
this.stepGraph.finishStep(step);
|
|
1532
1467
|
// Live subtask counts only describe an in-flight run; drop them now the step
|
|
1533
1468
|
// is done so the board doesn't show a stale "3/8" against a finished step.
|
|
1534
1469
|
step.subtasks = undefined;
|
|
@@ -1615,11 +1550,11 @@ export class ExecutionService {
|
|
|
1615
1550
|
status: 'pending',
|
|
1616
1551
|
proposal: step.output,
|
|
1617
1552
|
};
|
|
1618
|
-
this.pauseStepForInput(step);
|
|
1553
|
+
this.stepGraph.pauseStepForInput(step);
|
|
1619
1554
|
instance.status = 'blocked';
|
|
1620
|
-
await this.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
1555
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
1621
1556
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1622
|
-
await this.emitInstance(workspaceId, instance);
|
|
1557
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1623
1558
|
return { kind: 'awaiting_decision', decisionId: step.approval.id };
|
|
1624
1559
|
}
|
|
1625
1560
|
// Persist the agent's reported confidence whenever a step reports it, for board
|
|
@@ -1665,102 +1600,34 @@ export class ExecutionService {
|
|
|
1665
1600
|
// POSITION-INDEPENDENTLY: confidence at the top of recordStepResult and the merger's
|
|
1666
1601
|
// real merge via the step-completion resolver registry (so a trailing
|
|
1667
1602
|
// post-release-health gate doesn't disable auto-merge). Nothing merge-specific here.
|
|
1668
|
-
await this.finalizeBlock(workspaceId, instance, result.confidence);
|
|
1603
|
+
await this.runStateMachine.finalizeBlock(workspaceId, instance, result.confidence);
|
|
1669
1604
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1670
|
-
await this.emitInstance(workspaceId, instance);
|
|
1605
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1671
1606
|
// The run is finished: reclaim its per-run container now instead of letting it
|
|
1672
1607
|
// idle out its sleepAfter window (~10 min of billed-but-useless compute). All
|
|
1673
1608
|
// pipeline steps share the one container keyed by the execution id, so this is
|
|
1674
1609
|
// only safe on the FINAL step — never between steps. Best-effort/idempotent.
|
|
1675
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
1610
|
+
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
1676
1611
|
return { kind: 'done' };
|
|
1677
1612
|
}
|
|
1678
1613
|
instance.currentStep += 1;
|
|
1679
1614
|
const next = instance.steps[instance.currentStep];
|
|
1680
1615
|
if (next)
|
|
1681
|
-
this.startStep(next);
|
|
1616
|
+
this.stepGraph.startStep(next);
|
|
1682
1617
|
// A resolver that already set the block's TERMINAL status (the merger flips it to
|
|
1683
1618
|
// `done`/`pr_ready` mid-pipeline) must not be clobbered back to `in_progress` as we
|
|
1684
1619
|
// advance to a trailing step — refresh progress only, preserving that status. (The
|
|
1685
1620
|
// final step's `finalizeBlock` then leaves a `done` block alone.)
|
|
1686
1621
|
if (resolverOwnsTerminalStatus) {
|
|
1687
|
-
await this.refreshBlockProgress(workspaceId, instance);
|
|
1622
|
+
await this.runStateMachine.refreshBlockProgress(workspaceId, instance);
|
|
1688
1623
|
}
|
|
1689
1624
|
else {
|
|
1690
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
1625
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
1691
1626
|
}
|
|
1692
1627
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1693
|
-
await this.emitInstance(workspaceId, instance);
|
|
1628
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1694
1629
|
return { kind: 'continue' };
|
|
1695
1630
|
}
|
|
1696
|
-
/**
|
|
1697
|
-
* Reset a step so the durable driver re-runs it from scratch: clear its live
|
|
1698
|
-
* container job handle (so it dispatches FRESH work rather than re-attaching to a
|
|
1699
|
-
* finished or evicted job), its timings, approval gate, live subtasks and last
|
|
1700
|
-
* output, and drop it back to `pending`. Preserves the step's identity
|
|
1701
|
-
* (`agentKind` / `requiresApproval`) and any companion budget/verdict history.
|
|
1702
|
-
*/
|
|
1703
|
-
resetStepForRerun(step) {
|
|
1704
|
-
step.state = 'pending';
|
|
1705
|
-
step.startedAt = null;
|
|
1706
|
-
step.finishedAt = null;
|
|
1707
|
-
step.pausedAt = null;
|
|
1708
|
-
step.jobId = undefined;
|
|
1709
|
-
step.approval = null;
|
|
1710
|
-
step.subtasks = undefined;
|
|
1711
|
-
step.progress = 0;
|
|
1712
|
-
step.output = undefined;
|
|
1713
|
-
// Drop the prior run's structured output too, so a re-run that produces no `custom`
|
|
1714
|
-
// doesn't leave stale JSON for the `generic-structured` result view to render.
|
|
1715
|
-
step.custom = undefined;
|
|
1716
|
-
step.rework = undefined;
|
|
1717
|
-
}
|
|
1718
|
-
/**
|
|
1719
|
-
* Loop a producer step back for rework and re-run every step from it up to and
|
|
1720
|
-
* including the companion at `companionIndex`: each one is reset (crucially clearing
|
|
1721
|
-
* stale container job handles so an intermediate container step re-dispatches fresh
|
|
1722
|
-
* work instead of re-attaching to its evicted job), the producer is handed the
|
|
1723
|
-
* `rework` feedback + started, and the instance cursor is moved back to the producer.
|
|
1724
|
-
* Shared by the automatic companion loop and the human "request changes" path.
|
|
1725
|
-
*/
|
|
1726
|
-
rerunProducerThrough(instance, producerIndex, companionIndex, rework) {
|
|
1727
|
-
for (let i = producerIndex; i <= companionIndex; i++) {
|
|
1728
|
-
this.resetStepForRerun(instance.steps[i]);
|
|
1729
|
-
}
|
|
1730
|
-
const producer = instance.steps[producerIndex];
|
|
1731
|
-
producer.rework = rework;
|
|
1732
|
-
this.startStep(producer);
|
|
1733
|
-
instance.currentStep = producerIndex;
|
|
1734
|
-
}
|
|
1735
|
-
/**
|
|
1736
|
-
* The index of the nearest preceding step a companion grades (one of its target
|
|
1737
|
-
* producer kinds), or -1 when none precedes it. The single producer-search used by the
|
|
1738
|
-
* automatic companion loop, the human "request changes" redirect, and the iteration-cap
|
|
1739
|
-
* extra-round resolution.
|
|
1740
|
-
*/
|
|
1741
|
-
companionProducerIndex(instance, companionIndex) {
|
|
1742
|
-
const targets = companionTargets(instance.steps[companionIndex].agentKind);
|
|
1743
|
-
for (let i = companionIndex - 1; i >= 0; i--) {
|
|
1744
|
-
if (targets.includes(instance.steps[i].agentKind))
|
|
1745
|
-
return i;
|
|
1746
|
-
}
|
|
1747
|
-
return -1;
|
|
1748
|
-
}
|
|
1749
|
-
/**
|
|
1750
|
-
* Loop a companion's producer back for one more automatic rework cycle: charge one
|
|
1751
|
-
* attempt against the budget, then re-run the producer (and any intermediate steps) up
|
|
1752
|
-
* to and including the companion so it re-grades. Shared by the automatic
|
|
1753
|
-
* below-threshold loop ({@link evaluateCompanion}) and the human-granted extra round
|
|
1754
|
-
* ({@link resolveCompanionExceeded}), so both consume the budget identically.
|
|
1755
|
-
*/
|
|
1756
|
-
loopCompanionProducer(instance, companionIndex, rework) {
|
|
1757
|
-
const companionStep = instance.steps[companionIndex];
|
|
1758
|
-
const producerIndex = this.companionProducerIndex(instance, companionIndex);
|
|
1759
|
-
companionStep.companion.attempts += 1;
|
|
1760
|
-
this.rerunProducerThrough(instance, producerIndex, companionIndex, rework);
|
|
1761
|
-
if (instance.status === 'blocked')
|
|
1762
|
-
instance.status = 'running';
|
|
1763
|
-
}
|
|
1764
1631
|
/**
|
|
1765
1632
|
* Deterministically provision an ephemeral environment for a deployer step.
|
|
1766
1633
|
* Produces a human-readable summary as the step output and reports no token
|
|
@@ -2381,7 +2248,7 @@ export class ExecutionService {
|
|
|
2381
2248
|
// Keep polling. Persist the head sha + phase so the board can reflect it.
|
|
2382
2249
|
step.gate.phase = 'checking';
|
|
2383
2250
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2384
|
-
await this.emitInstance(workspaceId, instance);
|
|
2251
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2385
2252
|
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
2386
2253
|
}
|
|
2387
2254
|
// probe.status === 'fail'.
|
|
@@ -2436,75 +2303,9 @@ export class ExecutionService {
|
|
|
2436
2303
|
headSha: step.gate?.headSha ?? null,
|
|
2437
2304
|
};
|
|
2438
2305
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2439
|
-
await this.emitInstance(workspaceId, instance);
|
|
2306
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2440
2307
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
2441
2308
|
}
|
|
2442
|
-
/**
|
|
2443
|
-
* Raise a `decision_required` notification when a run parks on an iteration-cap gate
|
|
2444
|
-
* after spending its automatic budget — a quality companion at its rework cap or an
|
|
2445
|
-
* iterative reviewer (requirements / clarity) at its iteration cap. Without it the
|
|
2446
|
-
* three-choice decision is reachable only by drilling into the parked step, so the run
|
|
2447
|
-
* looks silently stuck. Best-effort: a missing notification service (tests) or block is
|
|
2448
|
-
* a no-op.
|
|
2449
|
-
*/
|
|
2450
|
-
async raiseDecisionRequired(workspaceId, instance) {
|
|
2451
|
-
if (!this.notificationService)
|
|
2452
|
-
return;
|
|
2453
|
-
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
2454
|
-
if (!block)
|
|
2455
|
-
return;
|
|
2456
|
-
await this.notificationService.raise(workspaceId, {
|
|
2457
|
-
type: 'decision_required',
|
|
2458
|
-
blockId: block.id,
|
|
2459
|
-
executionId: instance.id,
|
|
2460
|
-
title: `"${block.title}" ran out of automatic iterations and needs your decision`,
|
|
2461
|
-
body: 'An automatic review loop reached its iteration cap without converging. Open the ' +
|
|
2462
|
-
'task to choose: one more round, proceed with the current result, or stop and reset.',
|
|
2463
|
-
payload: { pipelineName: instance.pipelineName },
|
|
2464
|
-
});
|
|
2465
|
-
}
|
|
2466
|
-
/**
|
|
2467
|
-
* Ensure an open notification exists for a run that has just parked waiting for a human
|
|
2468
|
-
* (an agent-raised decision, an approval gate, or an iterative review gate). Without
|
|
2469
|
-
* the old decision timeout the run waits indefinitely, so the inbox card — which the
|
|
2470
|
-
* periodic sweep escalates yellow → red — is the only signal a human is needed.
|
|
2471
|
-
*
|
|
2472
|
-
* Non-clobbering: if ANY open notification is already on the block (a more specific
|
|
2473
|
-
* `merge_review`, iteration-cap `decision_required`, etc.), it is left untouched and we
|
|
2474
|
-
* raise nothing — so the richer message wins. Best-effort: no notification service
|
|
2475
|
-
* (tests) or a missing block is a no-op.
|
|
2476
|
-
*/
|
|
2477
|
-
async ensureWaitingNotification(workspaceId, instance) {
|
|
2478
|
-
const svc = this.notificationService;
|
|
2479
|
-
if (!svc)
|
|
2480
|
-
return;
|
|
2481
|
-
const open = await svc.listOpen(workspaceId);
|
|
2482
|
-
if (open.some((n) => n.blockId === instance.blockId))
|
|
2483
|
-
return;
|
|
2484
|
-
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
2485
|
-
if (!block)
|
|
2486
|
-
return;
|
|
2487
|
-
await svc.raise(workspaceId, {
|
|
2488
|
-
type: 'decision_required',
|
|
2489
|
-
blockId: block.id,
|
|
2490
|
-
executionId: instance.id,
|
|
2491
|
-
title: `"${block.title}" is waiting for your input`,
|
|
2492
|
-
body: 'A pipeline step is parked awaiting a human decision. Open the task to respond.',
|
|
2493
|
-
payload: { pipelineName: instance.pipelineName },
|
|
2494
|
-
});
|
|
2495
|
-
}
|
|
2496
|
-
/**
|
|
2497
|
-
* Clear the auto-raised "waiting for a human decision" card once a run advances past
|
|
2498
|
-
* the decision it was parked on (so the escalation sweep can't flip a settled decision
|
|
2499
|
-
* red). Scoped to the `decision_required` type, so the human-actionable cards a stopped
|
|
2500
|
-
* run leaves behind are untouched. Best-effort: no notification service (tests) is a no-op.
|
|
2501
|
-
*/
|
|
2502
|
-
async clearWaitingNotification(workspaceId, instance) {
|
|
2503
|
-
const svc = this.notificationService;
|
|
2504
|
-
if (!svc)
|
|
2505
|
-
return;
|
|
2506
|
-
await svc.clearWaitingDecision(workspaceId, instance.blockId);
|
|
2507
|
-
}
|
|
2508
2309
|
// ---- Follow-up companion (future-looking Coder) -------------------------
|
|
2509
2310
|
// The Coder streams forward-looking items (loose ends / side-tasks / questions) which
|
|
2510
2311
|
// accrue on its `step.followUps` live (see pollAgentJob). At the Coder's completion the
|
|
@@ -2548,13 +2349,13 @@ export class ExecutionService {
|
|
|
2548
2349
|
return undefined;
|
|
2549
2350
|
if (hasPendingFollowUps(state)) {
|
|
2550
2351
|
await this.raiseFollowUpPending(workspaceId, instance, state);
|
|
2551
|
-
return this.parkStepOnDecision(workspaceId, instance, step);
|
|
2352
|
+
return this.runStateMachine.parkStepOnDecision(workspaceId, instance, step);
|
|
2552
2353
|
}
|
|
2553
2354
|
if (shouldLoopCoder(state)) {
|
|
2554
2355
|
this.loopCoderForFollowUps(instance, step);
|
|
2555
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2356
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2556
2357
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2557
|
-
await this.emitInstance(workspaceId, instance);
|
|
2358
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2558
2359
|
return { kind: 'continue' };
|
|
2559
2360
|
}
|
|
2560
2361
|
return undefined;
|
|
@@ -2576,9 +2377,9 @@ export class ExecutionService {
|
|
|
2576
2377
|
state.loops = (state.loops ?? 0) + 1;
|
|
2577
2378
|
// Reset the step for a fresh dispatch; `step.followUps` is intentionally preserved
|
|
2578
2379
|
// (resetStepForRerun doesn't touch it) so the surfaced items survive the loop.
|
|
2579
|
-
this.resetStepForRerun(step);
|
|
2380
|
+
this.stepGraph.resetStepForRerun(step);
|
|
2580
2381
|
step.rework = { previousProposal: '', feedback };
|
|
2581
|
-
this.startStep(step);
|
|
2382
|
+
this.stepGraph.startStep(step);
|
|
2582
2383
|
if (instance.status === 'blocked')
|
|
2583
2384
|
instance.status = 'running';
|
|
2584
2385
|
}
|
|
@@ -2723,19 +2524,19 @@ export class ExecutionService {
|
|
|
2723
2524
|
if (!parkedHere || hasPendingFollowUps(step.followUps)) {
|
|
2724
2525
|
// Still collecting decisions (or the run isn't parked on this gate): just record it.
|
|
2725
2526
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2726
|
-
await this.emitInstance(workspaceId, instance);
|
|
2527
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2727
2528
|
return;
|
|
2728
2529
|
}
|
|
2729
2530
|
// Every item is decided and the run is parked here: clear the waiting card and either
|
|
2730
2531
|
// loop the Coder for the send-back items or advance past the gate.
|
|
2731
|
-
await this.clearWaitingNotification(workspaceId, instance);
|
|
2532
|
+
await this.runStateMachine.clearWaitingNotification(workspaceId, instance);
|
|
2732
2533
|
if (shouldLoopCoder(step.followUps)) {
|
|
2733
2534
|
const decisionId = step.approval.id;
|
|
2734
2535
|
this.loopCoderForFollowUps(instance, step);
|
|
2735
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2536
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2736
2537
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2737
2538
|
await this.workRunner.signalDecision(workspaceId, instance.id, decisionId, 'approved');
|
|
2738
|
-
await this.emitInstance(workspaceId, instance);
|
|
2539
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2739
2540
|
return;
|
|
2740
2541
|
}
|
|
2741
2542
|
// The follow-up gate is settled and we won't loop. If this step ALSO carries a human
|
|
@@ -2750,11 +2551,11 @@ export class ExecutionService {
|
|
|
2750
2551
|
if (step.requiresApproval && !isFinalStep && step.approval?.status === 'pending') {
|
|
2751
2552
|
step.approval = { ...step.approval, proposal: step.output ?? '' };
|
|
2752
2553
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
2753
|
-
await this.ensureWaitingNotification(workspaceId, instance);
|
|
2754
|
-
await this.emitInstance(workspaceId, instance);
|
|
2554
|
+
await this.runStateMachine.ensureWaitingNotification(workspaceId, instance);
|
|
2555
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
2755
2556
|
return;
|
|
2756
2557
|
}
|
|
2757
|
-
await this.advancePastResolvedGate(workspaceId, instance, index);
|
|
2558
|
+
await this.runStateMachine.advancePastResolvedGate(workspaceId, instance, index);
|
|
2758
2559
|
}
|
|
2759
2560
|
/** Provision inputs (`{{input.*}}`) derived from the block under deployment. */
|
|
2760
2561
|
deployInputs(block) {
|
|
@@ -2864,23 +2665,6 @@ export class ExecutionService {
|
|
|
2864
2665
|
// state-machine primitives stay here — they are reused by the generic approval path and
|
|
2865
2666
|
// the companion iteration-cap gate, so they have a single home: {@link parkStepOnDecision},
|
|
2866
2667
|
// {@link advancePastResolvedGate} and {@link dispatchIterationCap}.
|
|
2867
|
-
/**
|
|
2868
|
-
* Park a step on the durable decision-wait the approval gate uses, so a human (or the
|
|
2869
|
-
* dedicated review window) can drive an iterative loop and resume the run. Shared by the
|
|
2870
|
-
* requirements gate and the companion iteration-cap gate: both reuse the SAME parking
|
|
2871
|
-
* mechanism rather than each rolling its own. `proposal` seeds the gate's stored text
|
|
2872
|
-
* (the companion's latest feedback; empty for the requirements window, which renders its
|
|
2873
|
-
* own structured surface via the universal result-view registry).
|
|
2874
|
-
*/
|
|
2875
|
-
async parkStepOnDecision(workspaceId, instance, step, proposal = '') {
|
|
2876
|
-
step.approval = { id: this.idGenerator.next('appr'), status: 'pending', proposal };
|
|
2877
|
-
this.pauseStepForInput(step);
|
|
2878
|
-
instance.status = 'blocked';
|
|
2879
|
-
await this.updateBlockProgress(workspaceId, instance, 'blocked');
|
|
2880
|
-
await this.executionRepository.upsert(workspaceId, instance);
|
|
2881
|
-
await this.emitInstance(workspaceId, instance);
|
|
2882
|
-
return { kind: 'awaiting_decision', decisionId: step.approval.id };
|
|
2883
|
-
}
|
|
2884
2668
|
/**
|
|
2885
2669
|
* Two gates park on a `step.approval` but are NOT generic prose approvals — they are
|
|
2886
2670
|
* iterative gates driven by their own dedicated surface, never the generic
|
|
@@ -3155,15 +2939,15 @@ export class ExecutionService {
|
|
|
3155
2939
|
extraRound: async () => {
|
|
3156
2940
|
step.companion.maxAttempts += 1;
|
|
3157
2941
|
step.companion.exceeded = undefined;
|
|
3158
|
-
const producer = instance.steps[this.companionProducerIndex(instance, stepIndex)];
|
|
3159
|
-
this.loopCompanionProducer(instance, stepIndex, {
|
|
2942
|
+
const producer = instance.steps[this.stepGraph.companionProducerIndex(instance, stepIndex)];
|
|
2943
|
+
this.stepGraph.loopCompanionProducer(instance, stepIndex, {
|
|
3160
2944
|
previousProposal: producer?.output ?? '',
|
|
3161
2945
|
feedback: step.companion.verdicts.at(-1)?.feedback ?? '',
|
|
3162
2946
|
});
|
|
3163
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
2947
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
3164
2948
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
3165
2949
|
await this.workRunner.signalDecision(workspaceId, instance.id, approvalId, 'extra-round');
|
|
3166
|
-
await this.emitInstance(workspaceId, instance);
|
|
2950
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3167
2951
|
},
|
|
3168
2952
|
// Proceed: accept the producer's current output and advance past the gate.
|
|
3169
2953
|
proceed: async () => {
|
|
@@ -3174,48 +2958,16 @@ export class ExecutionService {
|
|
|
3174
2958
|
// so infer the block's `technical` label here too — best-effort, human-authority
|
|
3175
2959
|
// preserved — before advancing.
|
|
3176
2960
|
if (step.agentKind === 'spec-companion') {
|
|
3177
|
-
const producer = instance.steps[this.companionProducerIndex(instance, stepIndex)];
|
|
2961
|
+
const producer = instance.steps[this.stepGraph.companionProducerIndex(instance, stepIndex)];
|
|
3178
2962
|
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
3179
2963
|
if (producer && block)
|
|
3180
2964
|
await this.inferBlockTechnical(workspaceId, block, producer, step);
|
|
3181
2965
|
}
|
|
3182
|
-
await this.advancePastResolvedGate(workspaceId, instance, stepIndex);
|
|
2966
|
+
await this.runStateMachine.advancePastResolvedGate(workspaceId, instance, stepIndex);
|
|
3183
2967
|
},
|
|
3184
2968
|
});
|
|
3185
2969
|
return instance;
|
|
3186
2970
|
}
|
|
3187
|
-
/**
|
|
3188
|
-
* Finish a gate step the human just resolved (its `approval` already marked `approved`),
|
|
3189
|
-
* then either finish the run (final step) or advance to the next step, persist, and wake
|
|
3190
|
-
* the parked durable driver. The single advance/finalize/signal path shared by every
|
|
3191
|
-
* gate-resume site — the generic approval ({@link approveStep}), the review gates (via
|
|
3192
|
-
* {@link ReviewGateController}) and the companion iteration-cap proceed
|
|
3193
|
-
* ({@link resolveCompanionExceeded}) — so the logic lives in exactly one place.
|
|
3194
|
-
*/
|
|
3195
|
-
async advancePastResolvedGate(workspaceId, instance, stepIndex) {
|
|
3196
|
-
const step = instance.steps[stepIndex];
|
|
3197
|
-
const decisionId = step.approval.id;
|
|
3198
|
-
this.finishStep(step);
|
|
3199
|
-
step.progress = 1;
|
|
3200
|
-
const isFinalStep = stepIndex === instance.steps.length - 1;
|
|
3201
|
-
if (isFinalStep) {
|
|
3202
|
-
instance.status = 'done';
|
|
3203
|
-
await this.finalizeBlock(workspaceId, instance, undefined);
|
|
3204
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
3205
|
-
}
|
|
3206
|
-
else {
|
|
3207
|
-
instance.currentStep = stepIndex + 1;
|
|
3208
|
-
const next = instance.steps[instance.currentStep];
|
|
3209
|
-
if (next)
|
|
3210
|
-
this.startStep(next);
|
|
3211
|
-
if (instance.status === 'blocked')
|
|
3212
|
-
instance.status = 'running';
|
|
3213
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
3214
|
-
}
|
|
3215
|
-
await this.executionRepository.upsert(workspaceId, instance);
|
|
3216
|
-
await this.workRunner.signalDecision(workspaceId, instance.id, decisionId, 'approved');
|
|
3217
|
-
await this.emitInstance(workspaceId, instance);
|
|
3218
|
-
}
|
|
3219
2971
|
// ---- clarity-review context helpers (bug-report triage) ------------------
|
|
3220
2972
|
// The clarity gate triages a block's bug report — optionally enriched by an upstream
|
|
3221
2973
|
// `bug-investigator` step's prose output — through the SAME {@link ReviewGateController}
|
|
@@ -3338,166 +3090,13 @@ export class ExecutionService {
|
|
|
3338
3090
|
if (instance.status === 'blocked')
|
|
3339
3091
|
instance.status = 'running';
|
|
3340
3092
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
3341
|
-
await this.emitInstance(workspaceId, instance);
|
|
3093
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3342
3094
|
// Ensure a driver is active to consume the pending fix (idempotent for a live run).
|
|
3343
3095
|
if (instance.status === 'running') {
|
|
3344
3096
|
await this.workRunner.startRun(workspaceId, instance.id);
|
|
3345
3097
|
}
|
|
3346
3098
|
return instance;
|
|
3347
3099
|
}
|
|
3348
|
-
/**
|
|
3349
|
-
* Push the run's latest state to subscribed clients, alongside its rolled-up
|
|
3350
|
-
* block so the board updates without a refetch. Best-effort: the publisher
|
|
3351
|
-
* swallows its own errors, and the persisted run remains the source of truth.
|
|
3352
|
-
*/
|
|
3353
|
-
async emitInstance(workspaceId, instance) {
|
|
3354
|
-
// Stamp each step with the run id so a lone step (in a pushed event, a log line, a
|
|
3355
|
-
// detail view) is self-describing for debugging; the value always equals the run id.
|
|
3356
|
-
for (const step of instance.steps)
|
|
3357
|
-
step.runId = instance.id;
|
|
3358
|
-
// The metrics rollup and the block fetch are independent, so run them concurrently
|
|
3359
|
-
// — the rollup adds no serial latency to the (frequent) emit path.
|
|
3360
|
-
const [, block] = await Promise.all([
|
|
3361
|
-
this.attachStepMetrics(workspaceId, instance),
|
|
3362
|
-
this.blockRepository.get(workspaceId, instance.blockId),
|
|
3363
|
-
]);
|
|
3364
|
-
await this.events.executionChanged(workspaceId, instance, block);
|
|
3365
|
-
// When a run reaches a terminal state, schedule a post-run Kaizen grading for each
|
|
3366
|
-
// completed agent step (the scheduler skips verified combos + already-graded steps).
|
|
3367
|
-
// Best-effort + idempotent: a failure here must never derail the emit, and a re-emit
|
|
3368
|
-
// of an already-scheduled run is a no-op. The actual LLM grading runs later in the
|
|
3369
|
-
// background sweep, so this only does cheap inserts.
|
|
3370
|
-
if (this.kaizenScheduler && (instance.status === 'done' || instance.status === 'failed')) {
|
|
3371
|
-
try {
|
|
3372
|
-
await this.kaizenScheduler.scheduleForRun(workspaceId, instance);
|
|
3373
|
-
}
|
|
3374
|
-
catch {
|
|
3375
|
-
// Swallow — grading is an observability concern and must never break a run.
|
|
3376
|
-
}
|
|
3377
|
-
}
|
|
3378
|
-
// When a run reaches a terminal state, delete its per-run personal-credential
|
|
3379
|
-
// activation immediately (individual-usage subscriptions) so the system-encrypted
|
|
3380
|
-
// token copy doesn't linger to its TTL. Best-effort + idempotent — a missing repo or
|
|
3381
|
-
// a re-emit of an already-cleared run is a no-op, and a failure here must never
|
|
3382
|
-
// derail the emit.
|
|
3383
|
-
if (this.subscriptionActivations &&
|
|
3384
|
-
(instance.status === 'done' || instance.status === 'failed')) {
|
|
3385
|
-
try {
|
|
3386
|
-
await this.subscriptionActivations.deleteByExecution(instance.id);
|
|
3387
|
-
}
|
|
3388
|
-
catch {
|
|
3389
|
-
// Swallow — a failure here must never derail the emit. This is not a silent
|
|
3390
|
-
// data-loss path: the TTL sweep reclaims the row as a backstop, and the sweep
|
|
3391
|
-
// (Worker cron / Node retention timer) logs its own errors, so a *systemic*
|
|
3392
|
-
// cleanup failure surfaces there rather than being lost here.
|
|
3393
|
-
}
|
|
3394
|
-
}
|
|
3395
|
-
}
|
|
3396
|
-
/**
|
|
3397
|
-
* Roll the run's recorded LLM calls into per-step `metrics` for the board, in
|
|
3398
|
-
* place on the emitted instance. The proxy keys calls by execution + agentKind
|
|
3399
|
-
* (not step index), so the aggregate is per-agent-kind within the run; steps
|
|
3400
|
-
* sharing a kind get the same rollup. Best-effort and a no-op when the sink is
|
|
3401
|
-
* not wired, so it never blocks an emit.
|
|
3402
|
-
*/
|
|
3403
|
-
async attachStepMetrics(workspaceId, instance) {
|
|
3404
|
-
if (!this.llmObservability)
|
|
3405
|
-
return;
|
|
3406
|
-
try {
|
|
3407
|
-
const summaries = await this.llmObservability.summarizeByExecution(workspaceId, instance.id);
|
|
3408
|
-
if (summaries.length === 0)
|
|
3409
|
-
return;
|
|
3410
|
-
const byKind = new Map(summaries.map((s) => [s.agentKind, s]));
|
|
3411
|
-
for (const step of instance.steps) {
|
|
3412
|
-
const s = byKind.get(step.agentKind);
|
|
3413
|
-
if (!s)
|
|
3414
|
-
continue;
|
|
3415
|
-
step.metrics = {
|
|
3416
|
-
calls: s.calls,
|
|
3417
|
-
promptTokens: s.promptTokens,
|
|
3418
|
-
cachedPromptTokens: s.cachedPromptTokens,
|
|
3419
|
-
completionTokens: s.completionTokens,
|
|
3420
|
-
peakCompletionTokens: s.peakCompletionTokens,
|
|
3421
|
-
maxOutputTokens: s.maxOutputTokens,
|
|
3422
|
-
truncatedCalls: s.truncatedCalls,
|
|
3423
|
-
upstreamMs: s.upstreamMs,
|
|
3424
|
-
overheadMs: s.overheadMs,
|
|
3425
|
-
errors: s.errors,
|
|
3426
|
-
warnings: s.warnings,
|
|
3427
|
-
};
|
|
3428
|
-
}
|
|
3429
|
-
}
|
|
3430
|
-
catch (error) {
|
|
3431
|
-
// Observability is best-effort; never block an emit on a metrics read.
|
|
3432
|
-
void error;
|
|
3433
|
-
}
|
|
3434
|
-
}
|
|
3435
|
-
/** Set the block's in-progress/blocked status and step-completion progress. */
|
|
3436
|
-
async updateBlockProgress(workspaceId, instance, status) {
|
|
3437
|
-
const total = instance.steps.length || 1;
|
|
3438
|
-
const done = instance.steps.filter((s) => s.state === 'done').length;
|
|
3439
|
-
await this.blockRepository.update(workspaceId, instance.blockId, {
|
|
3440
|
-
status,
|
|
3441
|
-
progress: Math.min(1, done / total),
|
|
3442
|
-
});
|
|
3443
|
-
}
|
|
3444
|
-
/**
|
|
3445
|
-
* Advance the block's step PROGRESS without touching its status — used when a step
|
|
3446
|
-
* resolver already owns the block's terminal status (the merger set `done`/`pr_ready`)
|
|
3447
|
-
* and a trailing step still follows, so the bar moves on without downgrading that status.
|
|
3448
|
-
*/
|
|
3449
|
-
async refreshBlockProgress(workspaceId, instance) {
|
|
3450
|
-
const total = instance.steps.length || 1;
|
|
3451
|
-
const done = instance.steps.filter((s) => s.state === 'done').length;
|
|
3452
|
-
await this.blockRepository.update(workspaceId, instance.blockId, {
|
|
3453
|
-
progress: Math.min(1, done / total),
|
|
3454
|
-
});
|
|
3455
|
-
}
|
|
3456
|
-
/**
|
|
3457
|
-
* A pipeline finished. A frame becomes `done` (a mapping-only run leaves it
|
|
3458
|
-
* `ready`). A *task* never auto-`done`s from a confidence score any more — that
|
|
3459
|
-
* looked merged when the PR was still open with red CI. Instead:
|
|
3460
|
-
* - if the pipeline has a `merger` step, it already owned the merge/notify
|
|
3461
|
-
* decision (see {@link resolveMergerStep}); we only backstop a missing one;
|
|
3462
|
-
* - otherwise the work is complete but unmerged: leave the PR open (`pr_ready`)
|
|
3463
|
-
* and raise a `pipeline_complete` notification for a human to confirm + merge.
|
|
3464
|
-
* `done` now strictly means the PR was merged (see {@link finalizeMerge}).
|
|
3465
|
-
*/
|
|
3466
|
-
async finalizeBlock(workspaceId, instance, confidence) {
|
|
3467
|
-
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
3468
|
-
if (!block || block.status === 'done')
|
|
3469
|
-
return;
|
|
3470
|
-
if ((block.level ?? 'frame') !== 'task') {
|
|
3471
|
-
// A mapping-only run (just the `blueprints` step, e.g. kicked off after a
|
|
3472
|
-
// bootstrap) leaves the service frame `ready` and droppable rather than
|
|
3473
|
-
// marking the whole service "done".
|
|
3474
|
-
const mappingOnly = instance.steps.every((s) => s.agentKind === 'blueprints');
|
|
3475
|
-
await this.blockRepository.update(workspaceId, block.id, {
|
|
3476
|
-
status: mappingOnly ? 'ready' : 'done',
|
|
3477
|
-
progress: 1,
|
|
3478
|
-
});
|
|
3479
|
-
return;
|
|
3480
|
-
}
|
|
3481
|
-
// Confidence is recorded by the caller (recordStepResult) before any merge, so
|
|
3482
|
-
// it persists on both the merge and review paths; `confidence` is unused here.
|
|
3483
|
-
void confidence;
|
|
3484
|
-
const hasMerger = instance.steps.some((s) => s.agentKind === MERGER_AGENT_KIND);
|
|
3485
|
-
if (hasMerger) {
|
|
3486
|
-
// The `merger` step already merged (→ `done`) or raised a review (→ `pr_ready`).
|
|
3487
|
-
// Only backstop the case where it produced no decision at all.
|
|
3488
|
-
const fresh = await this.blockRepository.get(workspaceId, block.id);
|
|
3489
|
-
if (fresh && fresh.status !== 'done' && fresh.status !== 'pr_ready') {
|
|
3490
|
-
await this.blockRepository.update(workspaceId, block.id, {
|
|
3491
|
-
status: 'pr_ready',
|
|
3492
|
-
progress: 1,
|
|
3493
|
-
});
|
|
3494
|
-
}
|
|
3495
|
-
return;
|
|
3496
|
-
}
|
|
3497
|
-
// No merger in this pipeline: complete but unmerged — ask a human to confirm.
|
|
3498
|
-
await this.blockRepository.update(workspaceId, block.id, { status: 'pr_ready', progress: 1 });
|
|
3499
|
-
await this.raisePipelineComplete(workspaceId, instance, block);
|
|
3500
|
-
}
|
|
3501
3100
|
/**
|
|
3502
3101
|
* Merge a block's PR for real, then mark it `done`. The remote merge happens
|
|
3503
3102
|
* FIRST (via the {@link PullRequestMerger} port) and only on its success does the
|
|
@@ -3600,23 +3199,6 @@ export class ExecutionService {
|
|
|
3600
3199
|
}
|
|
3601
3200
|
return DEFAULT_MERGE_PRESET;
|
|
3602
3201
|
}
|
|
3603
|
-
/** Raise a `pipeline_complete` notification for a no-merger run awaiting confirmation. */
|
|
3604
|
-
async raisePipelineComplete(workspaceId, instance, block) {
|
|
3605
|
-
if (!this.notificationService)
|
|
3606
|
-
return;
|
|
3607
|
-
await this.notificationService.raise(workspaceId, {
|
|
3608
|
-
type: 'pipeline_complete',
|
|
3609
|
-
blockId: block.id,
|
|
3610
|
-
executionId: instance.id,
|
|
3611
|
-
title: `Confirm "${block.title}" is complete`,
|
|
3612
|
-
body: `The "${instance.pipelineName}" pipeline finished and opened a PR, but it has no ` +
|
|
3613
|
-
`merger step. Review the work and confirm it as complete (this merges the PR).`,
|
|
3614
|
-
payload: {
|
|
3615
|
-
...(block.pullRequest?.url ? { prUrl: block.pullRequest.url } : {}),
|
|
3616
|
-
pipelineName: instance.pipelineName,
|
|
3617
|
-
},
|
|
3618
|
-
});
|
|
3619
|
-
}
|
|
3620
3202
|
/**
|
|
3621
3203
|
* Implementing a task assigned to a module materialises that module: create it
|
|
3622
3204
|
* in the service if missing, then move the task inside it.
|
|
@@ -3655,16 +3237,16 @@ export class ExecutionService {
|
|
|
3655
3237
|
if (!step || !step.decision)
|
|
3656
3238
|
throw new NotFoundError('Decision', decisionId);
|
|
3657
3239
|
step.decision.chosen = choice;
|
|
3658
|
-
this.startStep(step);
|
|
3240
|
+
this.stepGraph.startStep(step);
|
|
3659
3241
|
if (instance.status === 'blocked')
|
|
3660
3242
|
instance.status = 'running';
|
|
3661
|
-
await this.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
3243
|
+
await this.runStateMachine.updateBlockProgress(workspaceId, instance, 'in_progress');
|
|
3662
3244
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
3663
3245
|
// Wake the parked durable run, if any. The DB write above remains the source
|
|
3664
3246
|
// of truth (so the backstop sweeper can still re-drive it); the signal is an
|
|
3665
3247
|
// optimisation that lets the workflow continue immediately.
|
|
3666
3248
|
await this.workRunner.signalDecision(workspaceId, instance.id, decisionId, choice);
|
|
3667
|
-
await this.emitInstance(workspaceId, instance);
|
|
3249
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3668
3250
|
return instance;
|
|
3669
3251
|
}
|
|
3670
3252
|
/**
|
|
@@ -3692,7 +3274,7 @@ export class ExecutionService {
|
|
|
3692
3274
|
}
|
|
3693
3275
|
step.approval.status = 'approved';
|
|
3694
3276
|
// A gate is never raised on the final step, but the shared advance stays defensive.
|
|
3695
|
-
await this.advancePastResolvedGate(workspaceId, instance, stepIndex);
|
|
3277
|
+
await this.runStateMachine.advancePastResolvedGate(workspaceId, instance, stepIndex);
|
|
3696
3278
|
return instance;
|
|
3697
3279
|
}
|
|
3698
3280
|
/**
|
|
@@ -3744,7 +3326,7 @@ export class ExecutionService {
|
|
|
3744
3326
|
// including the companion, then the companion re-grades. Does NOT touch the
|
|
3745
3327
|
// companion's automatic-rework budget — a human-driven iteration is unbounded.
|
|
3746
3328
|
const previousProposal = producer.output ?? step.approval.proposal;
|
|
3747
|
-
this.rerunProducerThrough(instance, producerIndex, stepIndex, {
|
|
3329
|
+
this.stepGraph.rerunProducerThrough(instance, producerIndex, stepIndex, {
|
|
3748
3330
|
previousProposal,
|
|
3749
3331
|
feedback: review.feedback ?? '',
|
|
3750
3332
|
...(review.comments?.length ? { comments: review.comments } : {}),
|
|
@@ -3753,7 +3335,7 @@ export class ExecutionService {
|
|
|
3753
3335
|
instance.status = 'running';
|
|
3754
3336
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
3755
3337
|
await this.workRunner.signalDecision(workspaceId, instance.id, approvalId, 'changes_requested');
|
|
3756
|
-
await this.emitInstance(workspaceId, instance);
|
|
3338
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3757
3339
|
return instance;
|
|
3758
3340
|
}
|
|
3759
3341
|
}
|
|
@@ -3764,12 +3346,12 @@ export class ExecutionService {
|
|
|
3764
3346
|
// start/finish times this attempt rather than spanning the human gate wait.
|
|
3765
3347
|
step.startedAt = null;
|
|
3766
3348
|
step.finishedAt = null;
|
|
3767
|
-
this.startStep(step);
|
|
3349
|
+
this.stepGraph.startStep(step);
|
|
3768
3350
|
if (instance.status === 'blocked')
|
|
3769
3351
|
instance.status = 'running';
|
|
3770
3352
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
3771
3353
|
await this.workRunner.signalDecision(workspaceId, instance.id, approvalId, 'changes_requested');
|
|
3772
|
-
await this.emitInstance(workspaceId, instance);
|
|
3354
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3773
3355
|
return instance;
|
|
3774
3356
|
}
|
|
3775
3357
|
/**
|
|
@@ -3830,42 +3412,8 @@ export class ExecutionService {
|
|
|
3830
3412
|
* durable driver once a step has exhausted its retries (or a job/decision
|
|
3831
3413
|
* faulted); `kind` classifies the cause so the right hint is shown.
|
|
3832
3414
|
*/
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
if (!instance)
|
|
3836
|
-
return;
|
|
3837
|
-
// Reclaim the per-run container on the failure path too: a failed run otherwise
|
|
3838
|
-
// leaves its container to idle out sleepAfter. This is the single funnel for
|
|
3839
|
-
// every failure kind (job_failed from the driver, the spend/decision timeouts,
|
|
3840
|
-
// and the user-facing stopRun, which already reclaimed — the call is idempotent).
|
|
3841
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
3842
|
-
// The FIRST recorded failure wins: a run already in a terminal `failed` state keeps
|
|
3843
|
-
// its existing (richest) failure rather than being overwritten. An inline gate that
|
|
3844
|
-
// knows the precise kind/detail returns a `job_failed` result the driver funnels here,
|
|
3845
|
-
// so there should only ever be one write — but this guards against a future path that
|
|
3846
|
-
// both records a failure and returns `job_failed`, which would otherwise clobber the
|
|
3847
|
-
// good record with a generic one (the companion-rejected regression).
|
|
3848
|
-
if (instance.status === 'failed')
|
|
3849
|
-
return;
|
|
3850
|
-
const failure = {
|
|
3851
|
-
kind,
|
|
3852
|
-
message,
|
|
3853
|
-
detail,
|
|
3854
|
-
hint: EXECUTION_FAILURE_HINTS[kind] ?? null,
|
|
3855
|
-
occurredAt: this.clock.now(),
|
|
3856
|
-
lastSubtasks: instance.steps[instance.currentStep]?.subtasks ?? null,
|
|
3857
|
-
};
|
|
3858
|
-
await this.executionRepository.markFailed(workspaceId, executionId, failure);
|
|
3859
|
-
// Progress reflects how far the pipeline got before failing.
|
|
3860
|
-
const done = instance.steps.filter((s) => s.state === 'done').length;
|
|
3861
|
-
const progress = instance.steps.length > 0 ? done / instance.steps.length : 0;
|
|
3862
|
-
await this.blockRepository.update(workspaceId, instance.blockId, {
|
|
3863
|
-
status: 'blocked',
|
|
3864
|
-
progress,
|
|
3865
|
-
});
|
|
3866
|
-
const failed = await this.executionRepository.get(workspaceId, executionId);
|
|
3867
|
-
if (failed)
|
|
3868
|
-
await this.emitInstance(workspaceId, failed);
|
|
3415
|
+
failRun(workspaceId, executionId, message, kind = 'agent', detail = null) {
|
|
3416
|
+
return this.runStateMachine.failRun(workspaceId, executionId, message, kind, detail);
|
|
3869
3417
|
}
|
|
3870
3418
|
/**
|
|
3871
3419
|
* Retry a failed run: re-drive the same pipeline on the same block, **resuming
|
|
@@ -3926,7 +3474,7 @@ export class ExecutionService {
|
|
|
3926
3474
|
executionId: instance.id,
|
|
3927
3475
|
});
|
|
3928
3476
|
await this.workRunner.startRun(workspaceId, instance.id);
|
|
3929
|
-
await this.emitInstance(workspaceId, instance);
|
|
3477
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
3930
3478
|
return instance;
|
|
3931
3479
|
}
|
|
3932
3480
|
/**
|
|
@@ -3973,7 +3521,7 @@ export class ExecutionService {
|
|
|
3973
3521
|
// container AND its durable driver — before minting the restart. A `done`/`failed`
|
|
3974
3522
|
// run is already terminal (a no-op teardown), but a still-`running` run would
|
|
3975
3523
|
// otherwise leak a container and a live Workflows/pg-boss driver.
|
|
3976
|
-
await this.stopRunContainer(workspaceId, previous);
|
|
3524
|
+
await this.runStateMachine.stopRunContainer(workspaceId, previous);
|
|
3977
3525
|
await this.workRunner.cancelRun(workspaceId, executionId);
|
|
3978
3526
|
const { steps, currentStep } = planRestartFromStep(previous, fromStepIndex);
|
|
3979
3527
|
// Mint the activation before replacing the prior run, so a bad password aborts the
|
|
@@ -3999,7 +3547,7 @@ export class ExecutionService {
|
|
|
3999
3547
|
executionId: instance.id,
|
|
4000
3548
|
});
|
|
4001
3549
|
await this.workRunner.startRun(workspaceId, instance.id);
|
|
4002
|
-
await this.emitInstance(workspaceId, instance);
|
|
3550
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
4003
3551
|
return instance;
|
|
4004
3552
|
}
|
|
4005
3553
|
/**
|
|
@@ -4015,7 +3563,7 @@ export class ExecutionService {
|
|
|
4015
3563
|
instance.status = 'running';
|
|
4016
3564
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
4017
3565
|
await this.workRunner.startRun(workspaceId, instance.id);
|
|
4018
|
-
await this.emitInstance(workspaceId, instance);
|
|
3566
|
+
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
4019
3567
|
}
|
|
4020
3568
|
return this.executionRepository.listByWorkspace(workspaceId);
|
|
4021
3569
|
}
|
|
@@ -4027,7 +3575,7 @@ export class ExecutionService {
|
|
|
4027
3575
|
// the record, so a cancel never leaves a container running until its watchdog.
|
|
4028
3576
|
const existing = await this.executionRepository.getByBlock(workspaceId, blockId);
|
|
4029
3577
|
if (existing) {
|
|
4030
|
-
await this.stopRunContainer(workspaceId, existing);
|
|
3578
|
+
await this.runStateMachine.stopRunContainer(workspaceId, existing);
|
|
4031
3579
|
await this.workRunner.cancelRun(workspaceId, existing.id);
|
|
4032
3580
|
}
|
|
4033
3581
|
await this.executionRepository.deleteByBlock(workspaceId, blockId);
|
|
@@ -4055,7 +3603,7 @@ export class ExecutionService {
|
|
|
4055
3603
|
const instance = assertFound(await this.executionRepository.get(workspaceId, executionId), 'Execution', executionId);
|
|
4056
3604
|
if (instance.status === 'failed' || instance.status === 'done')
|
|
4057
3605
|
return instance;
|
|
4058
|
-
await this.stopRunContainer(workspaceId, instance);
|
|
3606
|
+
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
4059
3607
|
await this.workRunner.cancelRun(workspaceId, executionId);
|
|
4060
3608
|
await this.failRun(workspaceId, executionId, opts.reason ?? 'Stopped by the user.', opts.kind ?? 'cancelled');
|
|
4061
3609
|
return assertFound(await this.executionRepository.get(workspaceId, executionId), 'Execution', executionId);
|
|
@@ -4072,33 +3620,10 @@ export class ExecutionService {
|
|
|
4072
3620
|
const run = await this.executionRepository.getByBlock(workspaceId, blockId);
|
|
4073
3621
|
if (!run)
|
|
4074
3622
|
continue;
|
|
4075
|
-
await this.stopRunContainer(workspaceId, run);
|
|
3623
|
+
await this.runStateMachine.stopRunContainer(workspaceId, run);
|
|
4076
3624
|
await this.workRunner.cancelRun(workspaceId, run.id);
|
|
4077
3625
|
await this.executionRepository.deleteByBlock(workspaceId, blockId);
|
|
4078
3626
|
}
|
|
4079
3627
|
}
|
|
4080
|
-
/**
|
|
4081
|
-
* Best-effort: reclaim the per-run container backing an execution. The container is
|
|
4082
|
-
* addressed by the run (execution) id, so a backend that shares one across the run
|
|
4083
|
-
* (Cloudflare, local Docker) tears the whole thing down. A per-job backend (a
|
|
4084
|
-
* self-hosted pool) has no run container, so it cancels the run's IN-FLIGHT step job
|
|
4085
|
-
* instead — hence we pass the current step's job id alongside the run id. A no-op for
|
|
4086
|
-
* inline executors (no `stopJob`) and for an already-gone container/job; never
|
|
4087
|
-
* throws, so it can't derail the teardown that calls it.
|
|
4088
|
-
*/
|
|
4089
|
-
async stopRunContainer(workspaceId, instance) {
|
|
4090
|
-
const executor = this.agentExecutor;
|
|
4091
|
-
if (!isAsyncAgentExecutor(executor) || !executor.stopJob)
|
|
4092
|
-
return;
|
|
4093
|
-
// The in-flight step's job id (when a job is parked), so a per-job backend can
|
|
4094
|
-
// cancel exactly it; the run-container backends ignore it and use the run id.
|
|
4095
|
-
const jobId = instance.steps[instance.currentStep]?.jobId ?? instance.id;
|
|
4096
|
-
try {
|
|
4097
|
-
await executor.stopJob({ jobId, runId: instance.id, workspaceId });
|
|
4098
|
-
}
|
|
4099
|
-
catch {
|
|
4100
|
-
// The container may already be gone (eviction/completion) — nothing to reclaim.
|
|
4101
|
-
}
|
|
4102
|
-
}
|
|
4103
3628
|
}
|
|
4104
3629
|
//# sourceMappingURL=ExecutionService.js.map
|