@oisincoveney/pipeline 2.8.0 → 2.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/argo-submit.d.ts +2 -4
- package/dist/argo-submit.js +80 -80
- package/dist/cluster-doctor.js +89 -101
- package/dist/config/defaults.js +9 -19
- package/dist/config/load.js +32 -39
- package/dist/config/schemas.d.ts +1 -1
- package/dist/gates.js +6 -225
- package/dist/mcp/gateway-error.js +15 -0
- package/dist/mcp/gateway.js +119 -220
- package/dist/moka-global-config.js +20 -20
- package/dist/moka-submit.d.ts +1 -1
- package/dist/pipeline-runtime.js +580 -371
- package/dist/run-state/git-refs.js +124 -94
- package/dist/runner-command-contract.d.ts +2 -2
- package/dist/runner-event-sink.js +37 -69
- package/dist/runtime/agent-node/agent-node.js +214 -173
- package/dist/runtime/builtins/builtins.js +344 -57
- package/dist/runtime/changed-files/changed-files.js +15 -27
- package/dist/runtime/changed-files/index.js +2 -0
- package/dist/runtime/drain-merge/drain-merge.js +124 -82
- package/dist/runtime/gates/gates.js +46 -28
- package/dist/runtime/hooks/hooks.js +74 -29
- package/dist/runtime/opencode-server.js +27 -21
- package/dist/runtime/opencode-session-executor.js +101 -44
- package/dist/runtime/parallel-node/parallel-node.js +24 -5
- package/dist/runtime/select-candidate/select-candidate.js +45 -29
- package/dist/runtime/services/agent-node-runtime-service.js +15 -0
- package/dist/runtime/services/command-executor-service.js +8 -0
- package/dist/runtime/services/config-io-service.js +42 -0
- package/dist/runtime/services/drain-merge-git-service.js +10 -0
- package/dist/runtime/services/git-porcelain-service.js +38 -0
- package/dist/runtime/services/kubernetes-argo-service.d.ts +13 -0
- package/dist/runtime/services/kubernetes-argo-service.js +81 -0
- package/dist/runtime/services/mcp-gateway-service.js +184 -0
- package/dist/runtime/services/opencode-sdk-service.js +27 -0
- package/dist/runtime/services/runner-event-sink-http-service.js +80 -0
- package/dist/runtime/services/select-candidate-service.js +13 -0
- package/package.json +1 -1
package/dist/pipeline-runtime.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { loadPipelineConfig } from "./config/load.js";
|
|
2
2
|
import "./config.js";
|
|
3
|
-
import { executeCommand } from "./runtime/command-executor/command-executor.js";
|
|
4
|
-
import "./runtime/command-executor/index.js";
|
|
5
3
|
import { parseJsonObject } from "./runtime/json-validation/json-validation.js";
|
|
6
4
|
import "./runtime/json-validation/index.js";
|
|
7
5
|
import { emitNodeFinish, emitNodeOutputRecorded, emitNodeStart, emitWorkflowFinish, emitWorkflowPlanned, emitWorkflowStarted, runtimeNodeActorDescriptor } from "./runtime/events/events.js";
|
|
8
6
|
import "./runtime/events/index.js";
|
|
7
|
+
import { executeCommand } from "./runtime/command-executor/command-executor.js";
|
|
8
|
+
import "./runtime/command-executor/index.js";
|
|
9
9
|
import { dispatchHooks } from "./runtime/hooks/hooks.js";
|
|
10
10
|
import "./runtime/hooks/index.js";
|
|
11
11
|
import { findPlannedNode } from "./planned-node.js";
|
|
@@ -16,6 +16,7 @@ import "./runtime/context/index.js";
|
|
|
16
16
|
import { executeBuiltin } from "./runtime/builtins/builtins.js";
|
|
17
17
|
import "./runtime/builtins/index.js";
|
|
18
18
|
import { diffChangedFiles, snapshotChangedFiles } from "./runtime/changed-files/changed-files.js";
|
|
19
|
+
import "./runtime/changed-files/index.js";
|
|
19
20
|
import { evaluateNodeGates } from "./runtime/gates/gates.js";
|
|
20
21
|
import "./runtime/gates/index.js";
|
|
21
22
|
import { LocalScheduler } from "./runtime/local-scheduler.js";
|
|
@@ -25,14 +26,15 @@ import { executeParallelNode } from "./runtime/parallel-node/parallel-node.js";
|
|
|
25
26
|
import "./runtime/parallel-node/index.js";
|
|
26
27
|
import { decideNodeRetry, nodeRetryPolicy } from "./runtime/retry.js";
|
|
27
28
|
import { fileRunJournal } from "./runtime/run-journal.js";
|
|
29
|
+
import { Effect } from "effect";
|
|
28
30
|
import { join } from "node:path";
|
|
29
31
|
//#region src/pipeline-runtime.ts
|
|
30
32
|
function runPipelineFromConfig(options) {
|
|
31
|
-
return withOpencodeRuntime(options, (resolved) => runPipelineWithContext(createRuntimeContext(resolved)));
|
|
33
|
+
return Effect.runPromise(withOpencodeRuntime(options, (resolved) => runPipelineWithContext(createRuntimeContext(resolved))));
|
|
32
34
|
}
|
|
33
35
|
function runScheduledWorkflowTask(options) {
|
|
34
36
|
const { dependencyOutputs, nodeId, ...runtimeOptions } = options;
|
|
35
|
-
return withOpencodeRuntime(runtimeOptions, (resolved) => {
|
|
37
|
+
return Effect.runPromise(withOpencodeRuntime(runtimeOptions, (resolved) => Effect.gen(function* () {
|
|
36
38
|
const context = createRuntimeContext(resolved);
|
|
37
39
|
hydrateScheduledDependencyStates(context, nodeId);
|
|
38
40
|
hydrateDependencyOutputs(context, dependencyOutputs);
|
|
@@ -40,8 +42,8 @@ function runScheduledWorkflowTask(options) {
|
|
|
40
42
|
at: now(),
|
|
41
43
|
type: "READY"
|
|
42
44
|
});
|
|
43
|
-
return executePlannedNode(nodeId, context);
|
|
44
|
-
});
|
|
45
|
+
return yield* executePlannedNode(nodeId, context);
|
|
46
|
+
})));
|
|
45
47
|
}
|
|
46
48
|
/**
|
|
47
49
|
* When the config uses opencode and the caller did not inject an executor,
|
|
@@ -49,12 +51,15 @@ function runScheduledWorkflowTask(options) {
|
|
|
49
51
|
* and tear the server down afterward. Command-only configs and callers that
|
|
50
52
|
* supply their own executor (tests, embedders) are passed through untouched.
|
|
51
53
|
*/
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
54
|
+
function withOpencodeRuntime(options, run) {
|
|
55
|
+
return Effect.gen(function* () {
|
|
56
|
+
if (options.executor) return yield* run(options);
|
|
57
|
+
const { config, worktreePath } = resolveConfigForRun(options);
|
|
58
|
+
if (configUsesOpencode(config)) return yield* runWithLeasedOpencode(options, config, worktreePath, run);
|
|
59
|
+
return yield* run({
|
|
60
|
+
...options,
|
|
61
|
+
config
|
|
62
|
+
});
|
|
58
63
|
});
|
|
59
64
|
}
|
|
60
65
|
function resolveConfigForRun(options) {
|
|
@@ -64,21 +69,19 @@ function resolveConfigForRun(options) {
|
|
|
64
69
|
worktreePath
|
|
65
70
|
};
|
|
66
71
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
return
|
|
72
|
+
function runWithLeasedOpencode(options, config, worktreePath, run) {
|
|
73
|
+
return Effect.scoped(Effect.gen(function* () {
|
|
74
|
+
const lease = yield* Effect.acquireRelease(Effect.tryPromise(() => leaseOpencodeRuntime({
|
|
75
|
+
config,
|
|
76
|
+
...options.signal ? { signal: options.signal } : {},
|
|
77
|
+
worktreePath
|
|
78
|
+
})), (lease) => Effect.promise(() => lease.release()));
|
|
79
|
+
return yield* run({
|
|
75
80
|
...options,
|
|
76
81
|
config,
|
|
77
82
|
executor: lease.executor
|
|
78
83
|
});
|
|
79
|
-
}
|
|
80
|
-
await lease.release();
|
|
81
|
-
}
|
|
84
|
+
}));
|
|
82
85
|
}
|
|
83
86
|
function runJournalPath(context, dir) {
|
|
84
87
|
return join(context.worktreePath ?? process.cwd(), dir, `${context.runId}.jsonl`);
|
|
@@ -88,41 +91,60 @@ function resolveRunJournal(context) {
|
|
|
88
91
|
if (!(durability?.enabled && context.runId)) return;
|
|
89
92
|
return fileRunJournal(runJournalPath(context, durability.dir));
|
|
90
93
|
}
|
|
91
|
-
|
|
92
|
-
|
|
94
|
+
function runPipelineWithContext(context) {
|
|
95
|
+
const scheduler = new LocalScheduler({
|
|
93
96
|
buildResult: (outcome, nodes, failure) => workflowRuntimeResult(context, outcome, nodes, failure),
|
|
94
97
|
emitWorkflowPlanned: (nextContext) => emitWorkflowPlanned(nextContext),
|
|
95
98
|
emitWorkflowStarted: (nextContext) => emitWorkflowStarted(nextContext),
|
|
96
|
-
executeNode: (nodeId, nextContext) => executePlannedNode(nodeId, nextContext),
|
|
99
|
+
executeNode: (nodeId, nextContext) => Effect.runPromise(executePlannedNode(nodeId, nextContext)),
|
|
97
100
|
isCancelled: (nextContext) => isCancelled(nextContext),
|
|
98
101
|
markNodeReady: (nodeId, nextContext) => recordNodeEvent(nextContext, nodeId, {
|
|
99
102
|
at: now(),
|
|
100
103
|
type: "READY"
|
|
101
104
|
}),
|
|
102
105
|
resolveJournal: (nextContext) => resolveRunJournal(nextContext),
|
|
103
|
-
runWorkflowHook: (event, failure, nextContext) =>
|
|
106
|
+
runWorkflowHook: (event, failure, nextContext) => Effect.runPromise(dispatchHooksEffect(nextContext, event, failure)),
|
|
104
107
|
shouldContinueAfterNodeResult: (result, nextContext) => shouldContinueAfterNodeResult(result, nextContext),
|
|
105
108
|
skipNode: (nodeId, reason, nextContext) => recordSkippedNodeState(nextContext, nodeId, reason, now())
|
|
106
|
-
})
|
|
109
|
+
});
|
|
110
|
+
return Effect.gen(function* () {
|
|
111
|
+
return finishRuntime(context, yield* Effect.tryPromise(() => scheduler.runWorkflow(context.plan, context)));
|
|
112
|
+
});
|
|
107
113
|
}
|
|
108
114
|
function shouldContinueAfterNodeResult(result, context) {
|
|
109
115
|
if (result.status !== "failed") return true;
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
116
|
+
return isRecoverableParallelFailure(context.plan.graph.node(result.nodeId), result.output, context);
|
|
117
|
+
}
|
|
118
|
+
function isRecoverableParallelFailure(node, output, context) {
|
|
119
|
+
if (!isParallelWithChildren(node, output)) return false;
|
|
120
|
+
return hasOnlyDrainMergeDependents(node, context);
|
|
121
|
+
}
|
|
122
|
+
function isParallelWithChildren(node, output) {
|
|
123
|
+
if (!node) return false;
|
|
124
|
+
return node.kind === "parallel" ? parallelOutputHasChildren(output) : false;
|
|
125
|
+
}
|
|
126
|
+
function hasOnlyDrainMergeDependents(node, context) {
|
|
127
|
+
if (node.dependents.length === 0) return false;
|
|
128
|
+
return node.dependents.every((dependentId) => isDrainMergeNode(context.plan.graph.node(dependentId)));
|
|
113
129
|
}
|
|
114
130
|
function parallelOutputHasChildren(output) {
|
|
115
131
|
return Object.keys(parseJsonObject(parseJsonObject(output).children)).length > 0;
|
|
116
132
|
}
|
|
117
133
|
function isDrainMergeNode(node) {
|
|
118
|
-
|
|
134
|
+
if (!node) return false;
|
|
135
|
+
return node.kind === "builtin" ? node.builtin === "drain-merge" : false;
|
|
136
|
+
}
|
|
137
|
+
function executePlannedNode(nodeId, context) {
|
|
138
|
+
return Effect.gen(function* () {
|
|
139
|
+
const node = plannedNodeById(context, nodeId);
|
|
140
|
+
if (!node) return yield* Effect.fail(/* @__PURE__ */ new Error(`workflow scheduler referenced unknown node '${nodeId}'`));
|
|
141
|
+
const result = yield* executeNode(node, context);
|
|
142
|
+
yield* dispatchHooksEffect(context, "node.finish", result.status === "failed" ? nodeRuntimeFailure(result) : void 0, node);
|
|
143
|
+
return result;
|
|
144
|
+
});
|
|
119
145
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
if (!node) throw new Error(`workflow scheduler referenced unknown node '${nodeId}'`);
|
|
123
|
-
const result = await executeNode(node, context);
|
|
124
|
-
await dispatchHooks(context, "node.finish", result.status === "failed" ? nodeRuntimeFailure(result) : void 0, node);
|
|
125
|
-
return result;
|
|
146
|
+
function dispatchHooksEffect(...args) {
|
|
147
|
+
return Effect.tryPromise(() => dispatchHooks(...args));
|
|
126
148
|
}
|
|
127
149
|
function plannedNodeById(context, nodeId) {
|
|
128
150
|
return context.plan.graph.node(nodeId) ?? findPlannedNode(context.plan.topologicalOrder, nodeId);
|
|
@@ -197,24 +219,46 @@ function runtimeStructuredOutputs(context) {
|
|
|
197
219
|
return context.nodeStateStore.structuredOutputList();
|
|
198
220
|
}
|
|
199
221
|
function hydrateDependencyOutputs(context, dependencyOutputs) {
|
|
200
|
-
const outputs =
|
|
222
|
+
const outputs = dependencyOutputMap(dependencyOutputs);
|
|
201
223
|
const finishedAt = now();
|
|
202
224
|
for (const [nodeId, output] of outputs) {
|
|
203
|
-
const existing = context.nodeStateStore.getNodeState(nodeId);
|
|
204
225
|
context.nodeStateStore.recordOutput(nodeId, output);
|
|
205
226
|
context.nodeStateStore.markInheritedOutput(nodeId);
|
|
206
|
-
context.nodeStateStore.setNodeState(nodeId,
|
|
207
|
-
attempts: existing?.attempts ?? 1,
|
|
208
|
-
evidence: [...existing?.evidence ?? [], "dependency output inherited from Argo artifact"],
|
|
209
|
-
exitCode: existing?.exitCode ?? 0,
|
|
210
|
-
finishedAt: existing?.finishedAt ?? finishedAt,
|
|
211
|
-
gates: existing?.gates ?? [],
|
|
212
|
-
id: nodeId,
|
|
213
|
-
output,
|
|
214
|
-
status: "passed"
|
|
215
|
-
});
|
|
227
|
+
context.nodeStateStore.setNodeState(nodeId, inheritedDependencyOutputState(context, nodeId, output, finishedAt));
|
|
216
228
|
}
|
|
217
229
|
}
|
|
230
|
+
function dependencyOutputMap(dependencyOutputs) {
|
|
231
|
+
if (dependencyOutputs instanceof Map) return dependencyOutputs;
|
|
232
|
+
return new Map(Object.entries(dependencyOutputs ?? {}));
|
|
233
|
+
}
|
|
234
|
+
function inheritedDependencyOutputState(context, nodeId, output, finishedAt) {
|
|
235
|
+
const existing = context.nodeStateStore.getNodeState(nodeId);
|
|
236
|
+
return {
|
|
237
|
+
attempts: existingAttempts(existing),
|
|
238
|
+
evidence: inheritedOutputEvidence(existing),
|
|
239
|
+
exitCode: existingExitCode(existing),
|
|
240
|
+
finishedAt: existingFinishedAt(existing, finishedAt),
|
|
241
|
+
gates: existingGates(existing),
|
|
242
|
+
id: nodeId,
|
|
243
|
+
output,
|
|
244
|
+
status: "passed"
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
function existingAttempts(existing) {
|
|
248
|
+
return existing ? existing.attempts : 1;
|
|
249
|
+
}
|
|
250
|
+
function inheritedOutputEvidence(existing) {
|
|
251
|
+
return [...existing ? existing.evidence : [], "dependency output inherited from Argo artifact"];
|
|
252
|
+
}
|
|
253
|
+
function existingExitCode(existing) {
|
|
254
|
+
return existing ? existing.exitCode ?? 0 : 0;
|
|
255
|
+
}
|
|
256
|
+
function existingFinishedAt(existing, fallback) {
|
|
257
|
+
return existing ? existing.finishedAt ?? fallback : fallback;
|
|
258
|
+
}
|
|
259
|
+
function existingGates(existing) {
|
|
260
|
+
return existing ? existing.gates : [];
|
|
261
|
+
}
|
|
218
262
|
function hydrateScheduledDependencyStates(context, nodeId) {
|
|
219
263
|
const finishedAt = now();
|
|
220
264
|
for (const dependencyId of scheduledDependencyNodeIds(context, nodeId)) {
|
|
@@ -290,38 +334,89 @@ function now() {
|
|
|
290
334
|
function isCancelled(context) {
|
|
291
335
|
return context.signal?.aborted === true;
|
|
292
336
|
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
337
|
+
function executeNode(node, context) {
|
|
338
|
+
return Effect.gen(function* () {
|
|
339
|
+
const retryPolicy = nodeRetryPolicy(node);
|
|
340
|
+
const state = initialAttemptLoopState();
|
|
341
|
+
const result = yield* runNodeAttempts(node, context, retryPolicy, state);
|
|
342
|
+
if (result) return result;
|
|
343
|
+
const finalRetry = state.retry ?? exhaustedRetry(node, retryPolicy.maxAttempts, state.last);
|
|
344
|
+
return yield* finishFailedNode(node, context, state.last, finalRetry);
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
function initialAttemptLoopState() {
|
|
348
|
+
return { last: {
|
|
296
349
|
evidence: [],
|
|
297
350
|
exitCode: 1,
|
|
298
351
|
output: ""
|
|
299
|
-
};
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
return
|
|
352
|
+
} };
|
|
353
|
+
}
|
|
354
|
+
function runNodeAttempts(node, context, retryPolicy, state) {
|
|
355
|
+
return Effect.gen(function* () {
|
|
356
|
+
for (let attempt = 1;; attempt += 1) {
|
|
357
|
+
const step = yield* runSingleNodeAttempt(node, context, retryPolicy, state, attempt);
|
|
358
|
+
if (step === "retry") continue;
|
|
359
|
+
return step === "failed" ? null : step;
|
|
360
|
+
}
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
function runSingleNodeAttempt(node, context, retryPolicy, state, attempt) {
|
|
364
|
+
return Effect.gen(function* () {
|
|
365
|
+
const outcome = yield* nodeAttemptCycleOrError(node, context, attempt, state.last);
|
|
366
|
+
if ("error" in outcome) {
|
|
367
|
+
state.retry = retryFromAttemptError(node, context, attempt, state.last, outcome.error);
|
|
368
|
+
return "failed";
|
|
307
369
|
}
|
|
308
|
-
|
|
309
|
-
|
|
370
|
+
state.last = outcome.last;
|
|
371
|
+
return yield* continueAfterAttemptCycle(node, context, retryPolicy, state, attempt, outcome);
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
function nodeAttemptCycleOrError(node, context, attempt, last) {
|
|
375
|
+
return Effect.catchAll(executeNodeAttemptCycle(node, context, attempt, last), (error) => Effect.succeed({ error }));
|
|
376
|
+
}
|
|
377
|
+
function continueAfterAttemptCycle(node, context, retryPolicy, state, attempt, cycle) {
|
|
378
|
+
if (cycle.result) {
|
|
379
|
+
emitNodeFinish(context, cycle.result);
|
|
380
|
+
return Effect.succeed(cycle.result);
|
|
381
|
+
}
|
|
382
|
+
state.retry = retryCandidateForCycle(node, cycle, state.last, attempt);
|
|
383
|
+
return continueAfterRetryCandidate(node, context, retryPolicy, state.retry, attempt);
|
|
384
|
+
}
|
|
385
|
+
function continueAfterRetryCandidate(node, context, retryPolicy, retry, attempt) {
|
|
386
|
+
return Effect.gen(function* () {
|
|
387
|
+
const remediation = yield* remediateFailedNode({
|
|
310
388
|
attempt,
|
|
311
389
|
context,
|
|
312
390
|
node,
|
|
313
391
|
retry
|
|
314
392
|
});
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
type: "PASSED"
|
|
320
|
-
});
|
|
321
|
-
emitNodeFinish(context, remediation.result);
|
|
322
|
-
return remediation.result;
|
|
393
|
+
const passed = remediationPassedResult(remediation);
|
|
394
|
+
if (passed) {
|
|
395
|
+
emitRemediationPass(context, node.id, passed);
|
|
396
|
+
return passed;
|
|
323
397
|
}
|
|
324
|
-
if (remediation
|
|
398
|
+
if (remediationRequestsRetry(remediation)) return "retry";
|
|
399
|
+
return yield* scheduleNodeRetry(node, context, retryPolicy, retry, attempt);
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
function remediationPassedResult(remediation) {
|
|
403
|
+
if (!remediation) return null;
|
|
404
|
+
return remediation.result ?? null;
|
|
405
|
+
}
|
|
406
|
+
function remediationRequestsRetry(remediation) {
|
|
407
|
+
if (!remediation) return false;
|
|
408
|
+
return remediation.retryNode === true;
|
|
409
|
+
}
|
|
410
|
+
function emitRemediationPass(context, nodeId, result) {
|
|
411
|
+
recordNodeEvent(context, nodeId, {
|
|
412
|
+
at: now(),
|
|
413
|
+
result,
|
|
414
|
+
type: "PASSED"
|
|
415
|
+
});
|
|
416
|
+
emitNodeFinish(context, result);
|
|
417
|
+
}
|
|
418
|
+
function scheduleNodeRetry(node, context, retryPolicy, retry, attempt) {
|
|
419
|
+
return Effect.gen(function* () {
|
|
325
420
|
const retryDecision = decideNodeRetry({
|
|
326
421
|
attempt,
|
|
327
422
|
evidence: retry.evidence,
|
|
@@ -330,61 +425,77 @@ async function executeNode(node, context) {
|
|
|
330
425
|
reason: retry.reason,
|
|
331
426
|
retryReason: retry.retryReason
|
|
332
427
|
});
|
|
333
|
-
|
|
334
|
-
at: now(),
|
|
335
|
-
attempt,
|
|
336
|
-
evidence: retry.evidence,
|
|
337
|
-
gate: retry.gate,
|
|
338
|
-
reason: retry.reason,
|
|
339
|
-
retry: retryDecision,
|
|
340
|
-
retryReason: retry.retryReason,
|
|
341
|
-
type: "RETRYING"
|
|
342
|
-
});
|
|
428
|
+
recordRetryingNodeEvent(context, node.id, attempt, retry, retryDecision);
|
|
343
429
|
emitRuntimeRetry(context, node.id, retryDecision, retry.retryReason);
|
|
344
|
-
if (!retryDecision?.scheduled)
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
retry
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
430
|
+
if (!retryDecision?.scheduled) return "failed";
|
|
431
|
+
yield* waitForRetryDelay(retryDecision.delayMs, context.signal);
|
|
432
|
+
return "retry";
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
function recordRetryingNodeEvent(context, nodeId, attempt, retry, retryDecision) {
|
|
436
|
+
recordNodeEvent(context, nodeId, {
|
|
437
|
+
at: now(),
|
|
438
|
+
attempt,
|
|
439
|
+
evidence: retry.evidence,
|
|
440
|
+
gate: retry.gate,
|
|
441
|
+
reason: retry.reason,
|
|
442
|
+
retry: retryDecision,
|
|
443
|
+
retryReason: retry.retryReason,
|
|
444
|
+
type: "RETRYING"
|
|
445
|
+
});
|
|
446
|
+
}
|
|
447
|
+
function retryFromAttemptError(node, context, attempt, last, err) {
|
|
448
|
+
return isCancelled(context) ? cancelledRetry(node.id, attempt, last) : failedAttemptRetry(node.id, attempt, last, err);
|
|
449
|
+
}
|
|
450
|
+
function cancelledRetry(nodeId, attempt, last) {
|
|
451
|
+
return {
|
|
452
|
+
attempt,
|
|
453
|
+
evidence: [...last.evidence, ...cancelledFailure().evidence],
|
|
454
|
+
gate: nodeId,
|
|
455
|
+
reason: "pipeline cancelled",
|
|
456
|
+
retryReason: "timeout"
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
function failedAttemptRetry(nodeId, attempt, last, err) {
|
|
460
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
461
|
+
return {
|
|
462
|
+
attempt,
|
|
463
|
+
evidence: [...last.evidence, message],
|
|
464
|
+
gate: nodeId,
|
|
465
|
+
reason: err instanceof Error ? err.message : "node retry failed",
|
|
466
|
+
retryReason: nodeRetryReason(last)
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
function exhaustedRetry(node, maxAttempts, last) {
|
|
470
|
+
return {
|
|
471
|
+
attempt: Math.max(1, maxAttempts),
|
|
368
472
|
evidence: last.evidence,
|
|
369
473
|
gate: node.id,
|
|
370
474
|
reason: `node exited with code ${last.exitCode}`,
|
|
371
475
|
retryReason: nodeRetryReason(last)
|
|
372
476
|
};
|
|
373
|
-
|
|
477
|
+
}
|
|
478
|
+
function finishFailedNode(node, context, last, retry) {
|
|
479
|
+
return Effect.gen(function* () {
|
|
480
|
+
yield* dispatchHooksEffect(context, "node.error", nodeRetryFailure(node, retry), node);
|
|
481
|
+
const result = nodeFailure(node.id, retry.attempt, retry.evidence, last.output);
|
|
482
|
+
recordNodeEvent(context, node.id, {
|
|
483
|
+
at: now(),
|
|
484
|
+
failure: nodeRuntimeFailure(result),
|
|
485
|
+
result,
|
|
486
|
+
type: "FAILED"
|
|
487
|
+
});
|
|
488
|
+
emitNodeFinish(context, result);
|
|
489
|
+
return result;
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
function nodeRetryFailure(node, retry) {
|
|
493
|
+
return {
|
|
374
494
|
evidence: retry.evidence,
|
|
375
495
|
gate: retry.gate,
|
|
376
496
|
nodeId: node.id,
|
|
377
497
|
reason: retry.reason
|
|
378
|
-
}
|
|
379
|
-
const result = nodeFailure(node.id, retry.attempt, retry.evidence, last.output);
|
|
380
|
-
recordNodeEvent(context, node.id, {
|
|
381
|
-
at: now(),
|
|
382
|
-
failure: nodeRuntimeFailure(result),
|
|
383
|
-
result,
|
|
384
|
-
type: "FAILED"
|
|
385
|
-
});
|
|
386
|
-
emitNodeFinish(context, result);
|
|
387
|
-
return result;
|
|
498
|
+
};
|
|
388
499
|
}
|
|
389
500
|
function emitRuntimeRetry(context, nodeId, retry, reason) {
|
|
390
501
|
context.observability?.({
|
|
@@ -396,113 +507,132 @@ function emitRuntimeRetry(context, nodeId, retry, reason) {
|
|
|
396
507
|
type: retry.scheduled ? "runtime.retry.scheduled" : "runtime.retry.exhausted"
|
|
397
508
|
});
|
|
398
509
|
}
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
if (input.retry.retryReason !== "gate_failure" || isRemediationNode(input.node) || !nodeCanWrite(input.context, input.node)) return null;
|
|
408
|
-
const beforeSnapshot = await snapshotChangedFiles(input.context.worktreePath);
|
|
409
|
-
const beforeOutput = input.context.nodeStateStore.getOutput(input.node.id);
|
|
410
|
-
const result = await executeSelfRemediation(input);
|
|
411
|
-
if (result.status !== "passed") return null;
|
|
412
|
-
const changed = diffChangedFiles(beforeSnapshot, await snapshotChangedFiles(input.context.worktreePath), input.context.worktreePath);
|
|
413
|
-
if (changed.files.size === 0 && result.output === beforeOutput) return null;
|
|
414
|
-
input.context.nodeStateStore.setSnapshot(input.node.id, changed);
|
|
415
|
-
input.context.nodeStateStore.recordOutput(input.node.id, result.output);
|
|
416
|
-
return {
|
|
417
|
-
attempts: input.attempt + 1,
|
|
418
|
-
evidence: result.evidence,
|
|
419
|
-
exitCode: result.exitCode,
|
|
420
|
-
nodeId: input.node.id,
|
|
421
|
-
output: result.output,
|
|
422
|
-
status: "passed"
|
|
423
|
-
};
|
|
510
|
+
function remediateFailedNode(input) {
|
|
511
|
+
return Effect.gen(function* () {
|
|
512
|
+
const selfRemediation = yield* remediateWritableNodeFailure(input);
|
|
513
|
+
if (selfRemediation) return { result: selfRemediation };
|
|
514
|
+
if (yield* remediateCoverageFailure(input)) return { retryNode: true };
|
|
515
|
+
if (yield* remediateUpstreamImplementationFailure(input)) return { retryNode: true };
|
|
516
|
+
return null;
|
|
517
|
+
});
|
|
424
518
|
}
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
}
|
|
519
|
+
function remediateWritableNodeFailure(input) {
|
|
520
|
+
return Effect.gen(function* () {
|
|
521
|
+
if (!canSelfRemediateWritableNode(input)) return null;
|
|
522
|
+
const beforeSnapshot = yield* snapshotChangedFilesEffect(input.context.worktreePath);
|
|
523
|
+
const beforeOutput = input.context.nodeStateStore.getOutput(input.node.id);
|
|
524
|
+
const result = yield* executeSelfRemediation(input);
|
|
525
|
+
if (result.status !== "passed") return null;
|
|
526
|
+
const changed = diffChangedFiles(beforeSnapshot, yield* snapshotChangedFilesEffect(input.context.worktreePath), input.context.worktreePath);
|
|
527
|
+
if (remediationChangedNothing(changed.files.size, result, beforeOutput)) return null;
|
|
528
|
+
input.context.nodeStateStore.setSnapshot(input.node.id, changed);
|
|
529
|
+
input.context.nodeStateStore.recordOutput(input.node.id, result.output);
|
|
530
|
+
return {
|
|
531
|
+
attempts: input.attempt + 1,
|
|
532
|
+
evidence: result.evidence,
|
|
533
|
+
exitCode: result.exitCode,
|
|
534
|
+
nodeId: input.node.id,
|
|
535
|
+
output: result.output,
|
|
536
|
+
status: "passed"
|
|
537
|
+
};
|
|
538
|
+
});
|
|
445
539
|
}
|
|
446
|
-
|
|
447
|
-
if (input.retry.retryReason !== "gate_failure"
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
beforeSnapshot,
|
|
475
|
-
context: input.context,
|
|
476
|
-
implementationNode,
|
|
477
|
-
result
|
|
540
|
+
function canSelfRemediateWritableNode(input) {
|
|
541
|
+
if (input.retry.retryReason !== "gate_failure") return false;
|
|
542
|
+
if (isRemediationNode(input.node)) return false;
|
|
543
|
+
return nodeCanWrite(input.context, input.node);
|
|
544
|
+
}
|
|
545
|
+
function remediationChangedNothing(changedFileCount, result, beforeOutput) {
|
|
546
|
+
if (changedFileCount !== 0) return false;
|
|
547
|
+
return result.output === beforeOutput;
|
|
548
|
+
}
|
|
549
|
+
function executeSelfRemediation(input) {
|
|
550
|
+
return Effect.gen(function* () {
|
|
551
|
+
const node = {
|
|
552
|
+
...input.node,
|
|
553
|
+
artifacts: void 0,
|
|
554
|
+
dependents: [],
|
|
555
|
+
id: `${input.node.id}:remediate:${input.retry.gate}:${input.attempt}`,
|
|
556
|
+
needs: [],
|
|
557
|
+
retries: void 0
|
|
558
|
+
};
|
|
559
|
+
const originalTask = input.context.task;
|
|
560
|
+
input.context.task = nodeRemediationTask({
|
|
561
|
+
node: input.node,
|
|
562
|
+
originalTask,
|
|
563
|
+
retry: input.retry
|
|
564
|
+
});
|
|
565
|
+
return yield* Effect.ensuring(executeNode(node, input.context), Effect.sync(() => {
|
|
566
|
+
input.context.task = originalTask;
|
|
567
|
+
}));
|
|
478
568
|
});
|
|
479
569
|
}
|
|
480
|
-
|
|
481
|
-
if (
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
};
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
570
|
+
function remediateCoverageFailure(input) {
|
|
571
|
+
if (input.retry.retryReason !== "gate_failure" || !hasSchedulingRole(input.context, input.node, "coverage")) return Effect.succeed(false);
|
|
572
|
+
return remediatePassedImplementationAncestors(input);
|
|
573
|
+
}
|
|
574
|
+
function remediateUpstreamImplementationFailure(input) {
|
|
575
|
+
if (isRemediationNode(input.node) || nodeCanWrite(input.context, input.node) || hasSchedulingRole(input.context, input.node, "coverage")) return Effect.succeed(false);
|
|
576
|
+
return remediatePassedImplementationAncestors(input);
|
|
577
|
+
}
|
|
578
|
+
function remediatePassedImplementationAncestors(input) {
|
|
579
|
+
return Effect.gen(function* () {
|
|
580
|
+
const implementationNodes = upstreamImplementationNodes(input.context, input.node).filter((candidate) => input.context.nodeStateStore.getNodeState(candidate.id)?.status === "passed");
|
|
581
|
+
if (implementationNodes.length === 0) return false;
|
|
582
|
+
for (const implementationNode of implementationNodes) if (!(yield* remediateImplementationAncestor(input, implementationNode))) return false;
|
|
583
|
+
return true;
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
function remediateImplementationAncestor(input, implementationNode) {
|
|
587
|
+
return Effect.gen(function* () {
|
|
588
|
+
if (isCancelled(input.context)) return false;
|
|
589
|
+
const beforeSnapshot = yield* snapshotChangedFilesEffect(input.context.worktreePath);
|
|
590
|
+
const beforeOutput = input.context.nodeStateStore.getOutput(implementationNode.id);
|
|
591
|
+
const result = yield* executeImplementationRemediation({
|
|
592
|
+
attempt: input.attempt,
|
|
593
|
+
context: input.context,
|
|
594
|
+
coverageNode: input.node,
|
|
595
|
+
implementationNode,
|
|
596
|
+
retry: input.retry
|
|
597
|
+
});
|
|
598
|
+
if (result.status !== "passed") return false;
|
|
599
|
+
return yield* recordImplementationRemediationEffect({
|
|
600
|
+
beforeOutput,
|
|
601
|
+
beforeSnapshot,
|
|
602
|
+
context: input.context,
|
|
603
|
+
implementationNode,
|
|
604
|
+
result
|
|
605
|
+
});
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
function recordImplementationRemediationEffect(input) {
|
|
609
|
+
return Effect.gen(function* () {
|
|
610
|
+
if (diffChangedFiles(input.beforeSnapshot, yield* snapshotChangedFilesEffect(input.context.worktreePath), input.context.worktreePath).files.size === 0 && input.result.output === input.beforeOutput) return false;
|
|
611
|
+
input.context.nodeStateStore.recordOutput(input.implementationNode.id, input.result.output);
|
|
612
|
+
return true;
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
function executeImplementationRemediation(input) {
|
|
616
|
+
return Effect.gen(function* () {
|
|
617
|
+
const node = {
|
|
618
|
+
...input.implementationNode,
|
|
619
|
+
artifacts: void 0,
|
|
620
|
+
dependents: [],
|
|
621
|
+
gates: void 0,
|
|
622
|
+
id: `${input.implementationNode.id}:remediate:${input.coverageNode.id}:${input.attempt}`,
|
|
623
|
+
needs: [],
|
|
624
|
+
retries: void 0
|
|
625
|
+
};
|
|
626
|
+
const originalTask = input.context.task;
|
|
627
|
+
input.context.task = remediationTask({
|
|
628
|
+
coverageNode: input.coverageNode,
|
|
629
|
+
originalTask,
|
|
630
|
+
retry: input.retry
|
|
631
|
+
});
|
|
632
|
+
return yield* Effect.ensuring(executeNode(node, input.context), Effect.sync(() => {
|
|
633
|
+
input.context.task = originalTask;
|
|
634
|
+
}));
|
|
635
|
+
});
|
|
506
636
|
}
|
|
507
637
|
function remediationTask(input) {
|
|
508
638
|
return [
|
|
@@ -527,9 +657,22 @@ function remediationTask(input) {
|
|
|
527
657
|
].join("\n");
|
|
528
658
|
}
|
|
529
659
|
function nodeCanWrite(context, node) {
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
return
|
|
660
|
+
const profileId = node.profile;
|
|
661
|
+
if (!profileId) return false;
|
|
662
|
+
return profileCanWrite(context.config.profiles[profileId]);
|
|
663
|
+
}
|
|
664
|
+
function profileCanWrite(profile) {
|
|
665
|
+
if (!profile) return false;
|
|
666
|
+
return hasWorkspaceWriteMode(profile) ? true : hasWriteTool(profile.tools ?? []);
|
|
667
|
+
}
|
|
668
|
+
function hasWorkspaceWriteMode(profile) {
|
|
669
|
+
return profile.filesystem?.mode === "workspace-write";
|
|
670
|
+
}
|
|
671
|
+
function hasWriteTool(tools) {
|
|
672
|
+
return tools.some(isWriteTool);
|
|
673
|
+
}
|
|
674
|
+
function isWriteTool(tool) {
|
|
675
|
+
return tool === "edit" ? true : tool === "write";
|
|
533
676
|
}
|
|
534
677
|
function isRemediationNode(node) {
|
|
535
678
|
return node.id.includes(":remediate:");
|
|
@@ -559,168 +702,222 @@ function nodeRemediationTask(input) {
|
|
|
559
702
|
function upstreamImplementationNodes(context, node) {
|
|
560
703
|
const visited = /* @__PURE__ */ new Set();
|
|
561
704
|
const ordered = [];
|
|
562
|
-
const visit = (
|
|
563
|
-
if (visited.has(nodeId)) return;
|
|
564
|
-
visited.add(nodeId);
|
|
565
|
-
const candidate = context.plan.graph.node(nodeId);
|
|
566
|
-
if (!candidate) return;
|
|
567
|
-
for (const need of candidate.needs) visit(need);
|
|
568
|
-
if (hasSchedulingRole(context, candidate, "implementation")) ordered.push(candidate);
|
|
569
|
-
};
|
|
705
|
+
const visit = (candidateId) => visitImplementationNode(context, visited, ordered, candidateId, visit);
|
|
570
706
|
for (const need of node.needs) visit(need);
|
|
571
707
|
return ordered;
|
|
572
708
|
}
|
|
709
|
+
function visitImplementationNode(context, visited, ordered, nodeId, visit) {
|
|
710
|
+
if (visited.has(nodeId)) return;
|
|
711
|
+
visited.add(nodeId);
|
|
712
|
+
const candidate = context.plan.graph.node(nodeId);
|
|
713
|
+
if (!candidate) return;
|
|
714
|
+
visitImplementationDependencies(candidate, visit);
|
|
715
|
+
appendImplementationNode(context, ordered, candidate);
|
|
716
|
+
}
|
|
717
|
+
function visitImplementationDependencies(candidate, visit) {
|
|
718
|
+
for (const need of candidate.needs) visit(need);
|
|
719
|
+
}
|
|
720
|
+
function appendImplementationNode(context, ordered, candidate) {
|
|
721
|
+
if (hasSchedulingRole(context, candidate, "implementation")) ordered.push(candidate);
|
|
722
|
+
}
|
|
573
723
|
function hasSchedulingRole(context, node, role) {
|
|
574
724
|
return node.profile ? context.config.profiles[node.profile]?.scheduling_roles?.includes(role) ?? false : false;
|
|
575
725
|
}
|
|
576
|
-
|
|
577
|
-
if (delayMs <= 0 || signal?.aborted) return;
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
726
|
+
function waitForRetryDelay(delayMs, signal) {
|
|
727
|
+
if (delayMs <= 0 || signal?.aborted) return Effect.void;
|
|
728
|
+
return Effect.race(Effect.sleep(delayMs), waitForAbort(signal));
|
|
729
|
+
}
|
|
730
|
+
function waitForAbort(signal) {
|
|
731
|
+
if (!signal) return Effect.never;
|
|
732
|
+
return Effect.async((resume) => {
|
|
733
|
+
const onAbort = () => resume(Effect.void);
|
|
734
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
735
|
+
return Effect.sync(() => signal.removeEventListener("abort", onAbort));
|
|
585
736
|
});
|
|
586
737
|
}
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
738
|
+
function executeNodeAttemptCycle(node, context, attempt, previous) {
|
|
739
|
+
return Effect.gen(function* () {
|
|
740
|
+
const startResult = yield* beginNodeAttempt(node, context, attempt, previous);
|
|
741
|
+
if (startResult) return startResult;
|
|
742
|
+
const last = yield* runNodeAttemptBody(node, context, attempt);
|
|
743
|
+
const cancelledAfterAttempt = cancelledNodeResult(context, node.id, attempt, last);
|
|
744
|
+
if (cancelledAfterAttempt) return {
|
|
745
|
+
last,
|
|
746
|
+
result: cancelledAfterAttempt
|
|
747
|
+
};
|
|
748
|
+
return yield* finishNodeAttemptAfterGates(node, context, attempt, last);
|
|
597
749
|
});
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
750
|
+
}
|
|
751
|
+
function beginNodeAttempt(node, context, attempt, previous) {
|
|
752
|
+
return Effect.gen(function* () {
|
|
753
|
+
if (isCancelled(context)) return cancelledCycle(node.id, attempt, previous);
|
|
754
|
+
emitNodeStart(context, node, attempt);
|
|
601
755
|
recordNodeEvent(context, node.id, {
|
|
602
756
|
at: now(),
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
type: "FAILED"
|
|
757
|
+
attempt,
|
|
758
|
+
type: "STARTED"
|
|
606
759
|
});
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
760
|
+
const startHook = yield* dispatchHooksEffect(context, "node.start", void 0, node);
|
|
761
|
+
if (startHook) return failedHookCycle(node.id, attempt, previous, startHook.evidence, context);
|
|
762
|
+
return isCancelled(context) ? cancelledCycle(node.id, attempt, previous) : null;
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
function cancelledCycle(nodeId, attempt, previous) {
|
|
766
|
+
return {
|
|
613
767
|
last: previous,
|
|
614
|
-
result: nodeFailure(
|
|
768
|
+
result: nodeFailure(nodeId, attempt, cancelledFailure().evidence, previous.output)
|
|
615
769
|
};
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
context.nodeStateStore.setSnapshot(node.id, await snapshotChangedFiles(context.worktreePath));
|
|
621
|
-
recordNodeEvent(context, node.id, {
|
|
770
|
+
}
|
|
771
|
+
function failedHookCycle(nodeId, attempt, previous, evidence, context) {
|
|
772
|
+
const result = nodeFailure(nodeId, attempt, evidence, previous.output);
|
|
773
|
+
recordNodeEvent(context, nodeId, {
|
|
622
774
|
at: now(),
|
|
623
|
-
|
|
775
|
+
failure: nodeRuntimeFailure(result),
|
|
776
|
+
result,
|
|
777
|
+
type: "FAILED"
|
|
624
778
|
});
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
779
|
+
return {
|
|
780
|
+
last: previous,
|
|
781
|
+
result
|
|
782
|
+
};
|
|
783
|
+
}
|
|
784
|
+
function runNodeAttemptBody(node, context, attempt) {
|
|
785
|
+
return Effect.gen(function* () {
|
|
786
|
+
recordNodeEvent(context, node.id, {
|
|
787
|
+
at: now(),
|
|
788
|
+
type: "START_HOOKS_FINISHED"
|
|
789
|
+
});
|
|
790
|
+
context.nodeStateStore.setSnapshot(node.id, yield* snapshotChangedFilesEffect(context.worktreePath));
|
|
791
|
+
recordNodeEvent(context, node.id, {
|
|
792
|
+
at: now(),
|
|
793
|
+
type: "SNAPSHOT_BEFORE_FINISHED"
|
|
794
|
+
});
|
|
795
|
+
recordNodeEvent(context, node.id, {
|
|
796
|
+
at: now(),
|
|
797
|
+
type: "RUNNER_STARTED"
|
|
798
|
+
});
|
|
799
|
+
const last = yield* executeNodeAttempt(node, context, attempt);
|
|
800
|
+
recordNodeEvent(context, node.id, runnerFinishedEvent(last));
|
|
801
|
+
yield* recordAttemptOutput(node, context, attempt, last);
|
|
802
|
+
return last;
|
|
628
803
|
});
|
|
629
|
-
|
|
630
|
-
|
|
804
|
+
}
|
|
805
|
+
function runnerFinishedEvent(last) {
|
|
806
|
+
return {
|
|
631
807
|
at: now(),
|
|
632
808
|
evidence: last.evidence,
|
|
633
809
|
exitCode: last.exitCode,
|
|
634
810
|
output: last.output,
|
|
635
811
|
timedOut: last.timedOut,
|
|
636
812
|
type: "RUNNER_FINISHED"
|
|
637
|
-
});
|
|
638
|
-
const afterSnapshot = await snapshotChangedFiles(context.worktreePath);
|
|
639
|
-
const beforeSnapshot = context.nodeStateStore.getSnapshot(node.id);
|
|
640
|
-
if (beforeSnapshot) context.nodeStateStore.setSnapshot(node.id, diffChangedFiles(beforeSnapshot, afterSnapshot, context.worktreePath));
|
|
641
|
-
context.nodeStateStore.recordOutput(node.id, last.output);
|
|
642
|
-
context.nodeStateStore.recordHandoff(node.id, last.handoff);
|
|
643
|
-
emitNodeOutputRecorded(context, node, attempt, last.output);
|
|
644
|
-
recordNodeEvent(context, node.id, {
|
|
645
|
-
at: now(),
|
|
646
|
-
type: "OUTPUT_RECORDED"
|
|
647
|
-
});
|
|
648
|
-
recordNodeEvent(context, node.id, {
|
|
649
|
-
at: now(),
|
|
650
|
-
type: "SNAPSHOT_AFTER_FINISHED"
|
|
651
|
-
});
|
|
652
|
-
const cancelledAfterAttempt = cancelledNodeResult(context, node.id, attempt, last);
|
|
653
|
-
if (cancelledAfterAttempt) return {
|
|
654
|
-
last,
|
|
655
|
-
result: cancelledAfterAttempt
|
|
656
813
|
};
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
814
|
+
}
|
|
815
|
+
function recordAttemptOutput(node, context, attempt, last) {
|
|
816
|
+
return Effect.gen(function* () {
|
|
817
|
+
const afterSnapshot = yield* snapshotChangedFilesEffect(context.worktreePath);
|
|
818
|
+
const beforeSnapshot = context.nodeStateStore.getSnapshot(node.id);
|
|
819
|
+
if (beforeSnapshot) context.nodeStateStore.setSnapshot(node.id, diffChangedFiles(beforeSnapshot, afterSnapshot, context.worktreePath));
|
|
820
|
+
context.nodeStateStore.recordOutput(node.id, last.output);
|
|
821
|
+
context.nodeStateStore.recordHandoff(node.id, last.handoff);
|
|
822
|
+
emitNodeOutputRecorded(context, node, attempt, last.output);
|
|
823
|
+
recordNodeEvent(context, node.id, {
|
|
824
|
+
at: now(),
|
|
825
|
+
type: "OUTPUT_RECORDED"
|
|
826
|
+
});
|
|
827
|
+
recordNodeEvent(context, node.id, {
|
|
828
|
+
at: now(),
|
|
829
|
+
type: "SNAPSHOT_AFTER_FINISHED"
|
|
830
|
+
});
|
|
666
831
|
});
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
if (!failedGate && last.exitCode === 0) {
|
|
674
|
-
const successHook = await dispatchHooks(context, "node.success", void 0, node);
|
|
675
|
-
if (successHook) {
|
|
676
|
-
const result = nodeFailure(node.id, attempt, successHook.evidence, last.output);
|
|
677
|
-
recordNodeEvent(context, node.id, {
|
|
678
|
-
at: now(),
|
|
679
|
-
failure: nodeRuntimeFailure(result),
|
|
680
|
-
result,
|
|
681
|
-
type: "FAILED"
|
|
682
|
-
});
|
|
683
|
-
return {
|
|
684
|
-
last,
|
|
685
|
-
result
|
|
686
|
-
};
|
|
687
|
-
}
|
|
688
|
-
const cancelledAfterHook = cancelledNodeResult(context, node.id, attempt, last);
|
|
689
|
-
if (cancelledAfterHook) return {
|
|
832
|
+
}
|
|
833
|
+
function finishNodeAttemptAfterGates(node, context, attempt, last) {
|
|
834
|
+
return Effect.gen(function* () {
|
|
835
|
+
const gateResults = yield* evaluateGatesForAttempt(node, context, last);
|
|
836
|
+
const cancelledAfterGates = cancelledNodeResult(context, node.id, attempt, last);
|
|
837
|
+
if (cancelledAfterGates) return {
|
|
690
838
|
last,
|
|
691
|
-
result:
|
|
692
|
-
};
|
|
693
|
-
const result = {
|
|
694
|
-
attempts: attempt,
|
|
695
|
-
evidence: last.evidence,
|
|
696
|
-
exitCode: 0,
|
|
697
|
-
nodeId: node.id,
|
|
698
|
-
output: last.output,
|
|
699
|
-
status: "passed"
|
|
839
|
+
result: cancelledAfterGates
|
|
700
840
|
};
|
|
841
|
+
return yield* finishNodeAttemptWithGate(node, context, attempt, last, gateResults.find((gate) => !gate.passed));
|
|
842
|
+
});
|
|
843
|
+
}
|
|
844
|
+
function evaluateGatesForAttempt(node, context, last) {
|
|
845
|
+
return Effect.gen(function* () {
|
|
701
846
|
recordNodeEvent(context, node.id, {
|
|
702
847
|
at: now(),
|
|
703
|
-
|
|
704
|
-
type: "PASSED"
|
|
848
|
+
type: "GATES_STARTED"
|
|
705
849
|
});
|
|
706
|
-
|
|
850
|
+
const gateResults = yield* Effect.tryPromise(() => evaluateNodeGates(node, context, last, (failedNode, result) => Effect.runPromise(dispatchGateFailureHook(context, failedNode, result))));
|
|
851
|
+
recordNodeEvent(context, node.id, {
|
|
852
|
+
at: now(),
|
|
853
|
+
gates: gateResults,
|
|
854
|
+
type: "GATES_FINISHED"
|
|
855
|
+
});
|
|
856
|
+
return gateResults;
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
function finishNodeAttemptWithGate(node, context, attempt, last, failedGate) {
|
|
860
|
+
if (failedGate || last.exitCode !== 0) return Effect.succeed(retryCycle(node, attempt, last, failedGate));
|
|
861
|
+
return successfulAttemptCycle(node, context, attempt, last);
|
|
862
|
+
}
|
|
863
|
+
function successfulAttemptCycle(node, context, attempt, last) {
|
|
864
|
+
return Effect.gen(function* () {
|
|
865
|
+
const successHook = yield* dispatchHooksEffect(context, "node.success", void 0, node);
|
|
866
|
+
if (successHook) return failedHookCycle(node.id, attempt, last, successHook.evidence, context);
|
|
867
|
+
const cancelledAfterHook = cancelledNodeResult(context, node.id, attempt, last);
|
|
868
|
+
return cancelledAfterHook ? {
|
|
707
869
|
last,
|
|
708
|
-
result
|
|
709
|
-
};
|
|
710
|
-
}
|
|
711
|
-
|
|
712
|
-
|
|
870
|
+
result: cancelledAfterHook
|
|
871
|
+
} : passedCycle(node.id, attempt, last, context);
|
|
872
|
+
});
|
|
873
|
+
}
|
|
874
|
+
function passedCycle(nodeId, attempt, last, context) {
|
|
875
|
+
const result = passedNodeResult(nodeId, attempt, last);
|
|
876
|
+
recordNodeEvent(context, nodeId, {
|
|
877
|
+
at: now(),
|
|
878
|
+
result,
|
|
879
|
+
type: "PASSED"
|
|
880
|
+
});
|
|
881
|
+
return {
|
|
882
|
+
last,
|
|
883
|
+
result
|
|
884
|
+
};
|
|
885
|
+
}
|
|
886
|
+
function passedNodeResult(nodeId, attempt, last) {
|
|
887
|
+
return {
|
|
888
|
+
attempts: attempt,
|
|
889
|
+
evidence: last.evidence,
|
|
890
|
+
exitCode: 0,
|
|
891
|
+
nodeId,
|
|
892
|
+
output: last.output,
|
|
893
|
+
status: "passed"
|
|
894
|
+
};
|
|
895
|
+
}
|
|
896
|
+
function retryCycle(node, attempt, last, failedGate) {
|
|
713
897
|
return {
|
|
714
898
|
last,
|
|
715
899
|
retry: {
|
|
716
900
|
attempt,
|
|
717
|
-
evidence,
|
|
718
|
-
gate:
|
|
719
|
-
reason:
|
|
720
|
-
retryReason
|
|
901
|
+
evidence: retryEvidence(last, failedGate),
|
|
902
|
+
gate: retryGateId(node.id, failedGate),
|
|
903
|
+
reason: retryReasonText(last.exitCode, failedGate),
|
|
904
|
+
retryReason: nodeRetryReason(last, failedGate)
|
|
721
905
|
}
|
|
722
906
|
};
|
|
723
907
|
}
|
|
908
|
+
function retryGateId(nodeId, failedGate) {
|
|
909
|
+
return failedGate ? failedGate.gateId : nodeId;
|
|
910
|
+
}
|
|
911
|
+
function retryReasonText(exitCode, failedGate) {
|
|
912
|
+
if (!failedGate) return `node exited with code ${exitCode}`;
|
|
913
|
+
return failedGate.reason ?? `node exited with code ${exitCode}`;
|
|
914
|
+
}
|
|
915
|
+
function retryEvidence(last, failedGate) {
|
|
916
|
+
return failedGate ? [...last.evidence, ...failedGate.evidence] : last.evidence.concat(`node exited with code ${last.exitCode}`);
|
|
917
|
+
}
|
|
918
|
+
function snapshotChangedFilesEffect(worktreePath) {
|
|
919
|
+
return Effect.sync(() => snapshotChangedFiles(worktreePath));
|
|
920
|
+
}
|
|
724
921
|
function nodeRetryReason(attempt, failedGate) {
|
|
725
922
|
if (attempt.timedOut) return "timeout";
|
|
726
923
|
if (failedGate) return "gate_failure";
|
|
@@ -763,35 +960,47 @@ function nodeFailure(nodeId, attempts, evidence, output) {
|
|
|
763
960
|
};
|
|
764
961
|
}
|
|
765
962
|
function executeNodeAttempt(node, context, attempt) {
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
963
|
+
return nodeAttemptExecutors[node.kind](node, context, attempt);
|
|
964
|
+
}
|
|
965
|
+
const nodeAttemptExecutors = {
|
|
966
|
+
agent: executeAgentAttempt,
|
|
967
|
+
builtin: executeBuiltinAttempt,
|
|
968
|
+
command: executeCommandAttempt,
|
|
969
|
+
group: executeGroupAttempt,
|
|
970
|
+
parallel: executeParallelAttempt
|
|
971
|
+
};
|
|
972
|
+
function executeAgentAttempt(node, context, attempt) {
|
|
973
|
+
return Effect.tryPromise(() => executeAgentNode(node, context, attempt));
|
|
974
|
+
}
|
|
975
|
+
function executeCommandAttempt(node, context) {
|
|
976
|
+
return Effect.tryPromise(() => executeCommand(node.command ?? [], context, { timeout: node.timeoutMs }));
|
|
977
|
+
}
|
|
978
|
+
function executeBuiltinAttempt(node, context) {
|
|
979
|
+
return Effect.tryPromise(() => executeBuiltin(node.builtin ?? "", context, node));
|
|
980
|
+
}
|
|
981
|
+
function executeGroupAttempt(node) {
|
|
982
|
+
return Effect.succeed({
|
|
983
|
+
evidence: [`group '${node.id}' completed`],
|
|
984
|
+
exitCode: 0,
|
|
985
|
+
output: ""
|
|
986
|
+
});
|
|
987
|
+
}
|
|
988
|
+
function executeParallelAttempt(node, context) {
|
|
989
|
+
return Effect.tryPromise(() => executeParallelNode(node, context, {
|
|
990
|
+
executeNode: (child, childContext) => Effect.runPromise(executeNode(child, childContext)),
|
|
991
|
+
markNodeReady: (childContext, childId) => recordNodeEvent(childContext, childId, {
|
|
992
|
+
at: now(),
|
|
993
|
+
type: "READY"
|
|
994
|
+
})
|
|
995
|
+
}));
|
|
787
996
|
}
|
|
788
|
-
|
|
789
|
-
|
|
997
|
+
function dispatchGateFailureHook(context, node, result) {
|
|
998
|
+
return Effect.asVoid(dispatchHooksEffect(context, "gate.failure", {
|
|
790
999
|
evidence: result.evidence,
|
|
791
1000
|
gate: result.gateId,
|
|
792
1001
|
nodeId: node.id,
|
|
793
1002
|
reason: result.reason ?? "gate failed"
|
|
794
|
-
}, node, result.gateId);
|
|
1003
|
+
}, node, result.gateId));
|
|
795
1004
|
}
|
|
796
1005
|
function formatConfigError(err) {
|
|
797
1006
|
return [err.message, ...err.issues.map((issue) => issue.path ? `- ${issue.path}: ${issue.message}` : `- ${issue.message}`)].join("\n");
|