@oisincoveney/pipeline 2.7.0 → 2.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/defaults/pipeline.yaml +25 -0
- package/dist/argo-graph.js +7 -7
- package/dist/argo-submit.d.ts +2 -4
- package/dist/argo-submit.js +80 -80
- package/dist/bench/eval-report.js +27 -0
- package/dist/cli/program.js +19 -3
- package/dist/cluster-doctor.js +89 -101
- package/dist/commands/bench-command.js +18 -0
- package/dist/config/defaults.js +9 -19
- package/dist/config/load.js +47 -37
- package/dist/config/schemas.d.ts +24 -7
- package/dist/config/schemas.js +20 -7
- package/dist/context/repo-map.js +203 -0
- package/dist/install-commands/opencode.js +10 -1
- package/dist/mcp/gateway-error.js +15 -0
- package/dist/mcp/gateway.js +119 -220
- package/dist/moka-global-config.js +20 -20
- package/dist/moka-submit.d.ts +6 -6
- package/dist/pipeline-init.js +18 -12
- package/dist/pipeline-runtime.js +592 -372
- package/dist/planning/compile.d.ts +8 -3
- package/dist/planning/compile.js +7 -7
- package/dist/planning/generate.d.ts +6 -1
- package/dist/planning/generate.js +29 -7
- package/dist/run-state/git-refs.js +124 -94
- package/dist/runner-command-contract.d.ts +6 -1
- package/dist/runner-command-contract.js +6 -5
- package/dist/runner-event-schema.d.ts +6 -6
- package/dist/runner-event-sink.js +37 -68
- package/dist/runner.d.ts +6 -1
- package/dist/runner.js +3 -3
- package/dist/runtime/agent-node/agent-node.js +218 -159
- package/dist/runtime/changed-files/changed-files.js +15 -27
- package/dist/runtime/changed-files/index.js +2 -0
- package/dist/runtime/drain-merge/drain-merge.js +124 -82
- package/dist/runtime/gates/gates.js +45 -27
- package/dist/runtime/hooks/hooks.js +74 -29
- package/dist/runtime/local-scheduler.js +45 -0
- package/dist/runtime/opencode-server.js +32 -23
- package/dist/runtime/opencode-session-executor.js +101 -44
- package/dist/runtime/parallel-node/parallel-node.js +93 -75
- package/dist/runtime/parallel-worktrees/parallel-worktrees.js +49 -4
- package/dist/runtime/run-journal.js +21 -0
- package/dist/runtime/scheduler.js +122 -93
- package/dist/runtime/select-candidate/select-candidate.js +52 -24
- package/dist/runtime/services/agent-node-runtime-service.js +15 -0
- package/dist/runtime/services/command-executor-service.js +8 -0
- package/dist/runtime/services/config-io-service.js +42 -0
- package/dist/runtime/services/drain-merge-git-service.js +10 -0
- package/dist/runtime/services/git-porcelain-service.js +38 -0
- package/dist/runtime/services/kubernetes-argo-service.d.ts +13 -0
- package/dist/runtime/services/kubernetes-argo-service.js +81 -0
- package/dist/runtime/services/mcp-gateway-service.js +184 -0
- package/dist/runtime/services/opencode-sdk-service.js +27 -0
- package/dist/runtime/services/runner-event-sink-http-service.js +80 -0
- package/dist/runtime/services/select-candidate-service.js +13 -0
- package/dist/runtime/services/worktree-service.js +18 -0
- package/dist/schedule/passes/candidates.js +17 -8
- package/docs/config-architecture.md +105 -0
- package/package.json +7 -2
package/dist/pipeline-runtime.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { loadPipelineConfig } from "./config/load.js";
|
|
2
2
|
import "./config.js";
|
|
3
|
-
import { executeCommand } from "./runtime/command-executor/command-executor.js";
|
|
4
|
-
import "./runtime/command-executor/index.js";
|
|
5
3
|
import { parseJsonObject } from "./runtime/json-validation/json-validation.js";
|
|
6
4
|
import "./runtime/json-validation/index.js";
|
|
7
5
|
import { emitNodeFinish, emitNodeOutputRecorded, emitNodeStart, emitWorkflowFinish, emitWorkflowPlanned, emitWorkflowStarted, runtimeNodeActorDescriptor } from "./runtime/events/events.js";
|
|
8
6
|
import "./runtime/events/index.js";
|
|
7
|
+
import { executeCommand } from "./runtime/command-executor/command-executor.js";
|
|
8
|
+
import "./runtime/command-executor/index.js";
|
|
9
9
|
import { dispatchHooks } from "./runtime/hooks/hooks.js";
|
|
10
10
|
import "./runtime/hooks/index.js";
|
|
11
11
|
import { findPlannedNode } from "./planned-node.js";
|
|
@@ -16,21 +16,25 @@ import "./runtime/context/index.js";
|
|
|
16
16
|
import { executeBuiltin } from "./runtime/builtins/builtins.js";
|
|
17
17
|
import "./runtime/builtins/index.js";
|
|
18
18
|
import { diffChangedFiles, snapshotChangedFiles } from "./runtime/changed-files/changed-files.js";
|
|
19
|
+
import "./runtime/changed-files/index.js";
|
|
19
20
|
import { evaluateNodeGates } from "./runtime/gates/gates.js";
|
|
20
21
|
import "./runtime/gates/index.js";
|
|
22
|
+
import { LocalScheduler } from "./runtime/local-scheduler.js";
|
|
21
23
|
import { NodeStateTracker } from "./runtime/node-state-tracker.js";
|
|
22
24
|
import { configUsesOpencode, leaseOpencodeRuntime } from "./runtime/opencode-runtime.js";
|
|
23
25
|
import { executeParallelNode } from "./runtime/parallel-node/parallel-node.js";
|
|
24
26
|
import "./runtime/parallel-node/index.js";
|
|
25
27
|
import { decideNodeRetry, nodeRetryPolicy } from "./runtime/retry.js";
|
|
26
|
-
import {
|
|
28
|
+
import { fileRunJournal } from "./runtime/run-journal.js";
|
|
29
|
+
import { Effect } from "effect";
|
|
30
|
+
import { join } from "node:path";
|
|
27
31
|
//#region src/pipeline-runtime.ts
|
|
28
32
|
function runPipelineFromConfig(options) {
|
|
29
|
-
return withOpencodeRuntime(options, (resolved) => runPipelineWithContext(createRuntimeContext(resolved)));
|
|
33
|
+
return Effect.runPromise(withOpencodeRuntime(options, (resolved) => runPipelineWithContext(createRuntimeContext(resolved))));
|
|
30
34
|
}
|
|
31
35
|
function runScheduledWorkflowTask(options) {
|
|
32
36
|
const { dependencyOutputs, nodeId, ...runtimeOptions } = options;
|
|
33
|
-
return withOpencodeRuntime(runtimeOptions, (resolved) => {
|
|
37
|
+
return Effect.runPromise(withOpencodeRuntime(runtimeOptions, (resolved) => Effect.gen(function* () {
|
|
34
38
|
const context = createRuntimeContext(resolved);
|
|
35
39
|
hydrateScheduledDependencyStates(context, nodeId);
|
|
36
40
|
hydrateDependencyOutputs(context, dependencyOutputs);
|
|
@@ -38,8 +42,8 @@ function runScheduledWorkflowTask(options) {
|
|
|
38
42
|
at: now(),
|
|
39
43
|
type: "READY"
|
|
40
44
|
});
|
|
41
|
-
return executePlannedNode(nodeId, context);
|
|
42
|
-
});
|
|
45
|
+
return yield* executePlannedNode(nodeId, context);
|
|
46
|
+
})));
|
|
43
47
|
}
|
|
44
48
|
/**
|
|
45
49
|
* When the config uses opencode and the caller did not inject an executor,
|
|
@@ -47,12 +51,15 @@ function runScheduledWorkflowTask(options) {
|
|
|
47
51
|
* and tear the server down afterward. Command-only configs and callers that
|
|
48
52
|
* supply their own executor (tests, embedders) are passed through untouched.
|
|
49
53
|
*/
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
function withOpencodeRuntime(options, run) {
|
|
55
|
+
return Effect.gen(function* () {
|
|
56
|
+
if (options.executor) return yield* run(options);
|
|
57
|
+
const { config, worktreePath } = resolveConfigForRun(options);
|
|
58
|
+
if (configUsesOpencode(config)) return yield* runWithLeasedOpencode(options, config, worktreePath, run);
|
|
59
|
+
return yield* run({
|
|
60
|
+
...options,
|
|
61
|
+
config
|
|
62
|
+
});
|
|
56
63
|
});
|
|
57
64
|
}
|
|
58
65
|
function resolveConfigForRun(options) {
|
|
@@ -62,56 +69,82 @@ function resolveConfigForRun(options) {
|
|
|
62
69
|
worktreePath
|
|
63
70
|
};
|
|
64
71
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
return
|
|
72
|
+
function runWithLeasedOpencode(options, config, worktreePath, run) {
|
|
73
|
+
return Effect.scoped(Effect.gen(function* () {
|
|
74
|
+
const lease = yield* Effect.acquireRelease(Effect.tryPromise(() => leaseOpencodeRuntime({
|
|
75
|
+
config,
|
|
76
|
+
...options.signal ? { signal: options.signal } : {},
|
|
77
|
+
worktreePath
|
|
78
|
+
})), (lease) => Effect.promise(() => lease.release()));
|
|
79
|
+
return yield* run({
|
|
73
80
|
...options,
|
|
74
81
|
config,
|
|
75
82
|
executor: lease.executor
|
|
76
83
|
});
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
|
|
84
|
+
}));
|
|
85
|
+
}
|
|
86
|
+
function runJournalPath(context, dir) {
|
|
87
|
+
return join(context.worktreePath ?? process.cwd(), dir, `${context.runId}.jsonl`);
|
|
88
|
+
}
|
|
89
|
+
function resolveRunJournal(context) {
|
|
90
|
+
const durability = context.config.durability;
|
|
91
|
+
if (!(durability?.enabled && context.runId)) return;
|
|
92
|
+
return fileRunJournal(runJournalPath(context, durability.dir));
|
|
80
93
|
}
|
|
81
|
-
|
|
82
|
-
|
|
94
|
+
function runPipelineWithContext(context) {
|
|
95
|
+
const scheduler = new LocalScheduler({
|
|
83
96
|
buildResult: (outcome, nodes, failure) => workflowRuntimeResult(context, outcome, nodes, failure),
|
|
84
97
|
emitWorkflowPlanned: (nextContext) => emitWorkflowPlanned(nextContext),
|
|
85
98
|
emitWorkflowStarted: (nextContext) => emitWorkflowStarted(nextContext),
|
|
86
|
-
executeNode: (nodeId, nextContext) => executePlannedNode(nodeId, nextContext),
|
|
99
|
+
executeNode: (nodeId, nextContext) => Effect.runPromise(executePlannedNode(nodeId, nextContext)),
|
|
87
100
|
isCancelled: (nextContext) => isCancelled(nextContext),
|
|
88
101
|
markNodeReady: (nodeId, nextContext) => recordNodeEvent(nextContext, nodeId, {
|
|
89
102
|
at: now(),
|
|
90
103
|
type: "READY"
|
|
91
104
|
}),
|
|
92
|
-
|
|
105
|
+
resolveJournal: (nextContext) => resolveRunJournal(nextContext),
|
|
106
|
+
runWorkflowHook: (event, failure, nextContext) => Effect.runPromise(dispatchHooksEffect(nextContext, event, failure)),
|
|
93
107
|
shouldContinueAfterNodeResult: (result, nextContext) => shouldContinueAfterNodeResult(result, nextContext),
|
|
94
108
|
skipNode: (nodeId, reason, nextContext) => recordSkippedNodeState(nextContext, nodeId, reason, now())
|
|
95
|
-
})
|
|
109
|
+
});
|
|
110
|
+
return Effect.gen(function* () {
|
|
111
|
+
return finishRuntime(context, yield* Effect.tryPromise(() => scheduler.runWorkflow(context.plan, context)));
|
|
112
|
+
});
|
|
96
113
|
}
|
|
97
114
|
function shouldContinueAfterNodeResult(result, context) {
|
|
98
115
|
if (result.status !== "failed") return true;
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
116
|
+
return isRecoverableParallelFailure(context.plan.graph.node(result.nodeId), result.output, context);
|
|
117
|
+
}
|
|
118
|
+
function isRecoverableParallelFailure(node, output, context) {
|
|
119
|
+
if (!isParallelWithChildren(node, output)) return false;
|
|
120
|
+
return hasOnlyDrainMergeDependents(node, context);
|
|
121
|
+
}
|
|
122
|
+
function isParallelWithChildren(node, output) {
|
|
123
|
+
if (!node) return false;
|
|
124
|
+
return node.kind === "parallel" ? parallelOutputHasChildren(output) : false;
|
|
125
|
+
}
|
|
126
|
+
function hasOnlyDrainMergeDependents(node, context) {
|
|
127
|
+
if (node.dependents.length === 0) return false;
|
|
128
|
+
return node.dependents.every((dependentId) => isDrainMergeNode(context.plan.graph.node(dependentId)));
|
|
102
129
|
}
|
|
103
130
|
function parallelOutputHasChildren(output) {
|
|
104
131
|
return Object.keys(parseJsonObject(parseJsonObject(output).children)).length > 0;
|
|
105
132
|
}
|
|
106
133
|
function isDrainMergeNode(node) {
|
|
107
|
-
|
|
134
|
+
if (!node) return false;
|
|
135
|
+
return node.kind === "builtin" ? node.builtin === "drain-merge" : false;
|
|
136
|
+
}
|
|
137
|
+
function executePlannedNode(nodeId, context) {
|
|
138
|
+
return Effect.gen(function* () {
|
|
139
|
+
const node = plannedNodeById(context, nodeId);
|
|
140
|
+
if (!node) return yield* Effect.fail(/* @__PURE__ */ new Error(`workflow scheduler referenced unknown node '${nodeId}'`));
|
|
141
|
+
const result = yield* executeNode(node, context);
|
|
142
|
+
yield* dispatchHooksEffect(context, "node.finish", result.status === "failed" ? nodeRuntimeFailure(result) : void 0, node);
|
|
143
|
+
return result;
|
|
144
|
+
});
|
|
108
145
|
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
if (!node) throw new Error(`workflow scheduler referenced unknown node '${nodeId}'`);
|
|
112
|
-
const result = await executeNode(node, context);
|
|
113
|
-
await dispatchHooks(context, "node.finish", result.status === "failed" ? nodeRuntimeFailure(result) : void 0, node);
|
|
114
|
-
return result;
|
|
146
|
+
function dispatchHooksEffect(...args) {
|
|
147
|
+
return Effect.tryPromise(() => dispatchHooks(...args));
|
|
115
148
|
}
|
|
116
149
|
function plannedNodeById(context, nodeId) {
|
|
117
150
|
return context.plan.graph.node(nodeId) ?? findPlannedNode(context.plan.topologicalOrder, nodeId);
|
|
@@ -186,24 +219,46 @@ function runtimeStructuredOutputs(context) {
|
|
|
186
219
|
return context.nodeStateStore.structuredOutputList();
|
|
187
220
|
}
|
|
188
221
|
function hydrateDependencyOutputs(context, dependencyOutputs) {
|
|
189
|
-
const outputs =
|
|
222
|
+
const outputs = dependencyOutputMap(dependencyOutputs);
|
|
190
223
|
const finishedAt = now();
|
|
191
224
|
for (const [nodeId, output] of outputs) {
|
|
192
|
-
const existing = context.nodeStateStore.getNodeState(nodeId);
|
|
193
225
|
context.nodeStateStore.recordOutput(nodeId, output);
|
|
194
226
|
context.nodeStateStore.markInheritedOutput(nodeId);
|
|
195
|
-
context.nodeStateStore.setNodeState(nodeId,
|
|
196
|
-
attempts: existing?.attempts ?? 1,
|
|
197
|
-
evidence: [...existing?.evidence ?? [], "dependency output inherited from Argo artifact"],
|
|
198
|
-
exitCode: existing?.exitCode ?? 0,
|
|
199
|
-
finishedAt: existing?.finishedAt ?? finishedAt,
|
|
200
|
-
gates: existing?.gates ?? [],
|
|
201
|
-
id: nodeId,
|
|
202
|
-
output,
|
|
203
|
-
status: "passed"
|
|
204
|
-
});
|
|
227
|
+
context.nodeStateStore.setNodeState(nodeId, inheritedDependencyOutputState(context, nodeId, output, finishedAt));
|
|
205
228
|
}
|
|
206
229
|
}
|
|
230
|
+
function dependencyOutputMap(dependencyOutputs) {
|
|
231
|
+
if (dependencyOutputs instanceof Map) return dependencyOutputs;
|
|
232
|
+
return new Map(Object.entries(dependencyOutputs ?? {}));
|
|
233
|
+
}
|
|
234
|
+
function inheritedDependencyOutputState(context, nodeId, output, finishedAt) {
|
|
235
|
+
const existing = context.nodeStateStore.getNodeState(nodeId);
|
|
236
|
+
return {
|
|
237
|
+
attempts: existingAttempts(existing),
|
|
238
|
+
evidence: inheritedOutputEvidence(existing),
|
|
239
|
+
exitCode: existingExitCode(existing),
|
|
240
|
+
finishedAt: existingFinishedAt(existing, finishedAt),
|
|
241
|
+
gates: existingGates(existing),
|
|
242
|
+
id: nodeId,
|
|
243
|
+
output,
|
|
244
|
+
status: "passed"
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
function existingAttempts(existing) {
|
|
248
|
+
return existing ? existing.attempts : 1;
|
|
249
|
+
}
|
|
250
|
+
function inheritedOutputEvidence(existing) {
|
|
251
|
+
return [...existing ? existing.evidence : [], "dependency output inherited from Argo artifact"];
|
|
252
|
+
}
|
|
253
|
+
function existingExitCode(existing) {
|
|
254
|
+
return existing ? existing.exitCode ?? 0 : 0;
|
|
255
|
+
}
|
|
256
|
+
function existingFinishedAt(existing, fallback) {
|
|
257
|
+
return existing ? existing.finishedAt ?? fallback : fallback;
|
|
258
|
+
}
|
|
259
|
+
function existingGates(existing) {
|
|
260
|
+
return existing ? existing.gates : [];
|
|
261
|
+
}
|
|
207
262
|
function hydrateScheduledDependencyStates(context, nodeId) {
|
|
208
263
|
const finishedAt = now();
|
|
209
264
|
for (const dependencyId of scheduledDependencyNodeIds(context, nodeId)) {
|
|
@@ -279,38 +334,89 @@ function now() {
|
|
|
279
334
|
function isCancelled(context) {
|
|
280
335
|
return context.signal?.aborted === true;
|
|
281
336
|
}
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
337
|
+
function executeNode(node, context) {
|
|
338
|
+
return Effect.gen(function* () {
|
|
339
|
+
const retryPolicy = nodeRetryPolicy(node);
|
|
340
|
+
const state = initialAttemptLoopState();
|
|
341
|
+
const result = yield* runNodeAttempts(node, context, retryPolicy, state);
|
|
342
|
+
if (result) return result;
|
|
343
|
+
const finalRetry = state.retry ?? exhaustedRetry(node, retryPolicy.maxAttempts, state.last);
|
|
344
|
+
return yield* finishFailedNode(node, context, state.last, finalRetry);
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
function initialAttemptLoopState() {
|
|
348
|
+
return { last: {
|
|
285
349
|
evidence: [],
|
|
286
350
|
exitCode: 1,
|
|
287
351
|
output: ""
|
|
288
|
-
};
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
return
|
|
352
|
+
} };
|
|
353
|
+
}
|
|
354
|
+
function runNodeAttempts(node, context, retryPolicy, state) {
|
|
355
|
+
return Effect.gen(function* () {
|
|
356
|
+
for (let attempt = 1;; attempt += 1) {
|
|
357
|
+
const step = yield* runSingleNodeAttempt(node, context, retryPolicy, state, attempt);
|
|
358
|
+
if (step === "retry") continue;
|
|
359
|
+
return step === "failed" ? null : step;
|
|
360
|
+
}
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
function runSingleNodeAttempt(node, context, retryPolicy, state, attempt) {
|
|
364
|
+
return Effect.gen(function* () {
|
|
365
|
+
const outcome = yield* nodeAttemptCycleOrError(node, context, attempt, state.last);
|
|
366
|
+
if ("error" in outcome) {
|
|
367
|
+
state.retry = retryFromAttemptError(node, context, attempt, state.last, outcome.error);
|
|
368
|
+
return "failed";
|
|
296
369
|
}
|
|
297
|
-
|
|
298
|
-
|
|
370
|
+
state.last = outcome.last;
|
|
371
|
+
return yield* continueAfterAttemptCycle(node, context, retryPolicy, state, attempt, outcome);
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
function nodeAttemptCycleOrError(node, context, attempt, last) {
|
|
375
|
+
return Effect.catchAll(executeNodeAttemptCycle(node, context, attempt, last), (error) => Effect.succeed({ error }));
|
|
376
|
+
}
|
|
377
|
+
function continueAfterAttemptCycle(node, context, retryPolicy, state, attempt, cycle) {
|
|
378
|
+
if (cycle.result) {
|
|
379
|
+
emitNodeFinish(context, cycle.result);
|
|
380
|
+
return Effect.succeed(cycle.result);
|
|
381
|
+
}
|
|
382
|
+
state.retry = retryCandidateForCycle(node, cycle, state.last, attempt);
|
|
383
|
+
return continueAfterRetryCandidate(node, context, retryPolicy, state.retry, attempt);
|
|
384
|
+
}
|
|
385
|
+
function continueAfterRetryCandidate(node, context, retryPolicy, retry, attempt) {
|
|
386
|
+
return Effect.gen(function* () {
|
|
387
|
+
const remediation = yield* remediateFailedNode({
|
|
299
388
|
attempt,
|
|
300
389
|
context,
|
|
301
390
|
node,
|
|
302
391
|
retry
|
|
303
392
|
});
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
type: "PASSED"
|
|
309
|
-
});
|
|
310
|
-
emitNodeFinish(context, remediation.result);
|
|
311
|
-
return remediation.result;
|
|
393
|
+
const passed = remediationPassedResult(remediation);
|
|
394
|
+
if (passed) {
|
|
395
|
+
emitRemediationPass(context, node.id, passed);
|
|
396
|
+
return passed;
|
|
312
397
|
}
|
|
313
|
-
if (remediation
|
|
398
|
+
if (remediationRequestsRetry(remediation)) return "retry";
|
|
399
|
+
return yield* scheduleNodeRetry(node, context, retryPolicy, retry, attempt);
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
function remediationPassedResult(remediation) {
|
|
403
|
+
if (!remediation) return null;
|
|
404
|
+
return remediation.result ?? null;
|
|
405
|
+
}
|
|
406
|
+
function remediationRequestsRetry(remediation) {
|
|
407
|
+
if (!remediation) return false;
|
|
408
|
+
return remediation.retryNode === true;
|
|
409
|
+
}
|
|
410
|
+
function emitRemediationPass(context, nodeId, result) {
|
|
411
|
+
recordNodeEvent(context, nodeId, {
|
|
412
|
+
at: now(),
|
|
413
|
+
result,
|
|
414
|
+
type: "PASSED"
|
|
415
|
+
});
|
|
416
|
+
emitNodeFinish(context, result);
|
|
417
|
+
}
|
|
418
|
+
function scheduleNodeRetry(node, context, retryPolicy, retry, attempt) {
|
|
419
|
+
return Effect.gen(function* () {
|
|
314
420
|
const retryDecision = decideNodeRetry({
|
|
315
421
|
attempt,
|
|
316
422
|
evidence: retry.evidence,
|
|
@@ -319,61 +425,77 @@ async function executeNode(node, context) {
|
|
|
319
425
|
reason: retry.reason,
|
|
320
426
|
retryReason: retry.retryReason
|
|
321
427
|
});
|
|
322
|
-
|
|
323
|
-
at: now(),
|
|
324
|
-
attempt,
|
|
325
|
-
evidence: retry.evidence,
|
|
326
|
-
gate: retry.gate,
|
|
327
|
-
reason: retry.reason,
|
|
328
|
-
retry: retryDecision,
|
|
329
|
-
retryReason: retry.retryReason,
|
|
330
|
-
type: "RETRYING"
|
|
331
|
-
});
|
|
428
|
+
recordRetryingNodeEvent(context, node.id, attempt, retry, retryDecision);
|
|
332
429
|
emitRuntimeRetry(context, node.id, retryDecision, retry.retryReason);
|
|
333
|
-
if (!retryDecision?.scheduled)
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
retry
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
430
|
+
if (!retryDecision?.scheduled) return "failed";
|
|
431
|
+
yield* waitForRetryDelay(retryDecision.delayMs, context.signal);
|
|
432
|
+
return "retry";
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
function recordRetryingNodeEvent(context, nodeId, attempt, retry, retryDecision) {
|
|
436
|
+
recordNodeEvent(context, nodeId, {
|
|
437
|
+
at: now(),
|
|
438
|
+
attempt,
|
|
439
|
+
evidence: retry.evidence,
|
|
440
|
+
gate: retry.gate,
|
|
441
|
+
reason: retry.reason,
|
|
442
|
+
retry: retryDecision,
|
|
443
|
+
retryReason: retry.retryReason,
|
|
444
|
+
type: "RETRYING"
|
|
445
|
+
});
|
|
446
|
+
}
|
|
447
|
+
function retryFromAttemptError(node, context, attempt, last, err) {
|
|
448
|
+
return isCancelled(context) ? cancelledRetry(node.id, attempt, last) : failedAttemptRetry(node.id, attempt, last, err);
|
|
449
|
+
}
|
|
450
|
+
function cancelledRetry(nodeId, attempt, last) {
|
|
451
|
+
return {
|
|
452
|
+
attempt,
|
|
453
|
+
evidence: [...last.evidence, ...cancelledFailure().evidence],
|
|
454
|
+
gate: nodeId,
|
|
455
|
+
reason: "pipeline cancelled",
|
|
456
|
+
retryReason: "timeout"
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
function failedAttemptRetry(nodeId, attempt, last, err) {
|
|
460
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
461
|
+
return {
|
|
462
|
+
attempt,
|
|
463
|
+
evidence: [...last.evidence, message],
|
|
464
|
+
gate: nodeId,
|
|
465
|
+
reason: err instanceof Error ? err.message : "node retry failed",
|
|
466
|
+
retryReason: nodeRetryReason(last)
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
function exhaustedRetry(node, maxAttempts, last) {
|
|
470
|
+
return {
|
|
471
|
+
attempt: Math.max(1, maxAttempts),
|
|
357
472
|
evidence: last.evidence,
|
|
358
473
|
gate: node.id,
|
|
359
474
|
reason: `node exited with code ${last.exitCode}`,
|
|
360
475
|
retryReason: nodeRetryReason(last)
|
|
361
476
|
};
|
|
362
|
-
|
|
477
|
+
}
|
|
478
|
+
function finishFailedNode(node, context, last, retry) {
|
|
479
|
+
return Effect.gen(function* () {
|
|
480
|
+
yield* dispatchHooksEffect(context, "node.error", nodeRetryFailure(node, retry), node);
|
|
481
|
+
const result = nodeFailure(node.id, retry.attempt, retry.evidence, last.output);
|
|
482
|
+
recordNodeEvent(context, node.id, {
|
|
483
|
+
at: now(),
|
|
484
|
+
failure: nodeRuntimeFailure(result),
|
|
485
|
+
result,
|
|
486
|
+
type: "FAILED"
|
|
487
|
+
});
|
|
488
|
+
emitNodeFinish(context, result);
|
|
489
|
+
return result;
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
function nodeRetryFailure(node, retry) {
|
|
493
|
+
return {
|
|
363
494
|
evidence: retry.evidence,
|
|
364
495
|
gate: retry.gate,
|
|
365
496
|
nodeId: node.id,
|
|
366
497
|
reason: retry.reason
|
|
367
|
-
}
|
|
368
|
-
const result = nodeFailure(node.id, retry.attempt, retry.evidence, last.output);
|
|
369
|
-
recordNodeEvent(context, node.id, {
|
|
370
|
-
at: now(),
|
|
371
|
-
failure: nodeRuntimeFailure(result),
|
|
372
|
-
result,
|
|
373
|
-
type: "FAILED"
|
|
374
|
-
});
|
|
375
|
-
emitNodeFinish(context, result);
|
|
376
|
-
return result;
|
|
498
|
+
};
|
|
377
499
|
}
|
|
378
500
|
function emitRuntimeRetry(context, nodeId, retry, reason) {
|
|
379
501
|
context.observability?.({
|
|
@@ -385,113 +507,132 @@ function emitRuntimeRetry(context, nodeId, retry, reason) {
|
|
|
385
507
|
type: retry.scheduled ? "runtime.retry.scheduled" : "runtime.retry.exhausted"
|
|
386
508
|
});
|
|
387
509
|
}
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
if (input.retry.retryReason !== "gate_failure" || isRemediationNode(input.node) || !nodeCanWrite(input.context, input.node)) return null;
|
|
397
|
-
const beforeSnapshot = await snapshotChangedFiles(input.context.worktreePath);
|
|
398
|
-
const beforeOutput = input.context.nodeStateStore.getOutput(input.node.id);
|
|
399
|
-
const result = await executeSelfRemediation(input);
|
|
400
|
-
if (result.status !== "passed") return null;
|
|
401
|
-
const changed = diffChangedFiles(beforeSnapshot, await snapshotChangedFiles(input.context.worktreePath), input.context.worktreePath);
|
|
402
|
-
if (changed.files.size === 0 && result.output === beforeOutput) return null;
|
|
403
|
-
input.context.nodeStateStore.setSnapshot(input.node.id, changed);
|
|
404
|
-
input.context.nodeStateStore.recordOutput(input.node.id, result.output);
|
|
405
|
-
return {
|
|
406
|
-
attempts: input.attempt + 1,
|
|
407
|
-
evidence: result.evidence,
|
|
408
|
-
exitCode: result.exitCode,
|
|
409
|
-
nodeId: input.node.id,
|
|
410
|
-
output: result.output,
|
|
411
|
-
status: "passed"
|
|
412
|
-
};
|
|
510
|
+
function remediateFailedNode(input) {
|
|
511
|
+
return Effect.gen(function* () {
|
|
512
|
+
const selfRemediation = yield* remediateWritableNodeFailure(input);
|
|
513
|
+
if (selfRemediation) return { result: selfRemediation };
|
|
514
|
+
if (yield* remediateCoverageFailure(input)) return { retryNode: true };
|
|
515
|
+
if (yield* remediateUpstreamImplementationFailure(input)) return { retryNode: true };
|
|
516
|
+
return null;
|
|
517
|
+
});
|
|
413
518
|
}
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
}
|
|
519
|
+
function remediateWritableNodeFailure(input) {
|
|
520
|
+
return Effect.gen(function* () {
|
|
521
|
+
if (!canSelfRemediateWritableNode(input)) return null;
|
|
522
|
+
const beforeSnapshot = yield* snapshotChangedFilesEffect(input.context.worktreePath);
|
|
523
|
+
const beforeOutput = input.context.nodeStateStore.getOutput(input.node.id);
|
|
524
|
+
const result = yield* executeSelfRemediation(input);
|
|
525
|
+
if (result.status !== "passed") return null;
|
|
526
|
+
const changed = diffChangedFiles(beforeSnapshot, yield* snapshotChangedFilesEffect(input.context.worktreePath), input.context.worktreePath);
|
|
527
|
+
if (remediationChangedNothing(changed.files.size, result, beforeOutput)) return null;
|
|
528
|
+
input.context.nodeStateStore.setSnapshot(input.node.id, changed);
|
|
529
|
+
input.context.nodeStateStore.recordOutput(input.node.id, result.output);
|
|
530
|
+
return {
|
|
531
|
+
attempts: input.attempt + 1,
|
|
532
|
+
evidence: result.evidence,
|
|
533
|
+
exitCode: result.exitCode,
|
|
534
|
+
nodeId: input.node.id,
|
|
535
|
+
output: result.output,
|
|
536
|
+
status: "passed"
|
|
537
|
+
};
|
|
538
|
+
});
|
|
434
539
|
}
|
|
435
|
-
|
|
436
|
-
if (input.retry.retryReason !== "gate_failure"
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
beforeSnapshot,
|
|
464
|
-
context: input.context,
|
|
465
|
-
implementationNode,
|
|
466
|
-
result
|
|
540
|
+
function canSelfRemediateWritableNode(input) {
|
|
541
|
+
if (input.retry.retryReason !== "gate_failure") return false;
|
|
542
|
+
if (isRemediationNode(input.node)) return false;
|
|
543
|
+
return nodeCanWrite(input.context, input.node);
|
|
544
|
+
}
|
|
545
|
+
function remediationChangedNothing(changedFileCount, result, beforeOutput) {
|
|
546
|
+
if (changedFileCount !== 0) return false;
|
|
547
|
+
return result.output === beforeOutput;
|
|
548
|
+
}
|
|
549
|
+
function executeSelfRemediation(input) {
|
|
550
|
+
return Effect.gen(function* () {
|
|
551
|
+
const node = {
|
|
552
|
+
...input.node,
|
|
553
|
+
artifacts: void 0,
|
|
554
|
+
dependents: [],
|
|
555
|
+
id: `${input.node.id}:remediate:${input.retry.gate}:${input.attempt}`,
|
|
556
|
+
needs: [],
|
|
557
|
+
retries: void 0
|
|
558
|
+
};
|
|
559
|
+
const originalTask = input.context.task;
|
|
560
|
+
input.context.task = nodeRemediationTask({
|
|
561
|
+
node: input.node,
|
|
562
|
+
originalTask,
|
|
563
|
+
retry: input.retry
|
|
564
|
+
});
|
|
565
|
+
return yield* Effect.ensuring(executeNode(node, input.context), Effect.sync(() => {
|
|
566
|
+
input.context.task = originalTask;
|
|
567
|
+
}));
|
|
467
568
|
});
|
|
468
569
|
}
|
|
469
|
-
|
|
470
|
-
if (
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
};
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
570
|
+
function remediateCoverageFailure(input) {
|
|
571
|
+
if (input.retry.retryReason !== "gate_failure" || !hasSchedulingRole(input.context, input.node, "coverage")) return Effect.succeed(false);
|
|
572
|
+
return remediatePassedImplementationAncestors(input);
|
|
573
|
+
}
|
|
574
|
+
function remediateUpstreamImplementationFailure(input) {
|
|
575
|
+
if (isRemediationNode(input.node) || nodeCanWrite(input.context, input.node) || hasSchedulingRole(input.context, input.node, "coverage")) return Effect.succeed(false);
|
|
576
|
+
return remediatePassedImplementationAncestors(input);
|
|
577
|
+
}
|
|
578
|
+
function remediatePassedImplementationAncestors(input) {
|
|
579
|
+
return Effect.gen(function* () {
|
|
580
|
+
const implementationNodes = upstreamImplementationNodes(input.context, input.node).filter((candidate) => input.context.nodeStateStore.getNodeState(candidate.id)?.status === "passed");
|
|
581
|
+
if (implementationNodes.length === 0) return false;
|
|
582
|
+
for (const implementationNode of implementationNodes) if (!(yield* remediateImplementationAncestor(input, implementationNode))) return false;
|
|
583
|
+
return true;
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
function remediateImplementationAncestor(input, implementationNode) {
|
|
587
|
+
return Effect.gen(function* () {
|
|
588
|
+
if (isCancelled(input.context)) return false;
|
|
589
|
+
const beforeSnapshot = yield* snapshotChangedFilesEffect(input.context.worktreePath);
|
|
590
|
+
const beforeOutput = input.context.nodeStateStore.getOutput(implementationNode.id);
|
|
591
|
+
const result = yield* executeImplementationRemediation({
|
|
592
|
+
attempt: input.attempt,
|
|
593
|
+
context: input.context,
|
|
594
|
+
coverageNode: input.node,
|
|
595
|
+
implementationNode,
|
|
596
|
+
retry: input.retry
|
|
597
|
+
});
|
|
598
|
+
if (result.status !== "passed") return false;
|
|
599
|
+
return yield* recordImplementationRemediationEffect({
|
|
600
|
+
beforeOutput,
|
|
601
|
+
beforeSnapshot,
|
|
602
|
+
context: input.context,
|
|
603
|
+
implementationNode,
|
|
604
|
+
result
|
|
605
|
+
});
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
function recordImplementationRemediationEffect(input) {
|
|
609
|
+
return Effect.gen(function* () {
|
|
610
|
+
if (diffChangedFiles(input.beforeSnapshot, yield* snapshotChangedFilesEffect(input.context.worktreePath), input.context.worktreePath).files.size === 0 && input.result.output === input.beforeOutput) return false;
|
|
611
|
+
input.context.nodeStateStore.recordOutput(input.implementationNode.id, input.result.output);
|
|
612
|
+
return true;
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
function executeImplementationRemediation(input) {
|
|
616
|
+
return Effect.gen(function* () {
|
|
617
|
+
const node = {
|
|
618
|
+
...input.implementationNode,
|
|
619
|
+
artifacts: void 0,
|
|
620
|
+
dependents: [],
|
|
621
|
+
gates: void 0,
|
|
622
|
+
id: `${input.implementationNode.id}:remediate:${input.coverageNode.id}:${input.attempt}`,
|
|
623
|
+
needs: [],
|
|
624
|
+
retries: void 0
|
|
625
|
+
};
|
|
626
|
+
const originalTask = input.context.task;
|
|
627
|
+
input.context.task = remediationTask({
|
|
628
|
+
coverageNode: input.coverageNode,
|
|
629
|
+
originalTask,
|
|
630
|
+
retry: input.retry
|
|
631
|
+
});
|
|
632
|
+
return yield* Effect.ensuring(executeNode(node, input.context), Effect.sync(() => {
|
|
633
|
+
input.context.task = originalTask;
|
|
634
|
+
}));
|
|
635
|
+
});
|
|
495
636
|
}
|
|
496
637
|
function remediationTask(input) {
|
|
497
638
|
return [
|
|
@@ -516,9 +657,22 @@ function remediationTask(input) {
|
|
|
516
657
|
].join("\n");
|
|
517
658
|
}
|
|
518
659
|
function nodeCanWrite(context, node) {
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
return
|
|
660
|
+
const profileId = node.profile;
|
|
661
|
+
if (!profileId) return false;
|
|
662
|
+
return profileCanWrite(context.config.profiles[profileId]);
|
|
663
|
+
}
|
|
664
|
+
function profileCanWrite(profile) {
|
|
665
|
+
if (!profile) return false;
|
|
666
|
+
return hasWorkspaceWriteMode(profile) ? true : hasWriteTool(profile.tools ?? []);
|
|
667
|
+
}
|
|
668
|
+
function hasWorkspaceWriteMode(profile) {
|
|
669
|
+
return profile.filesystem?.mode === "workspace-write";
|
|
670
|
+
}
|
|
671
|
+
function hasWriteTool(tools) {
|
|
672
|
+
return tools.some(isWriteTool);
|
|
673
|
+
}
|
|
674
|
+
function isWriteTool(tool) {
|
|
675
|
+
return tool === "edit" ? true : tool === "write";
|
|
522
676
|
}
|
|
523
677
|
function isRemediationNode(node) {
|
|
524
678
|
return node.id.includes(":remediate:");
|
|
@@ -548,168 +702,222 @@ function nodeRemediationTask(input) {
|
|
|
548
702
|
function upstreamImplementationNodes(context, node) {
|
|
549
703
|
const visited = /* @__PURE__ */ new Set();
|
|
550
704
|
const ordered = [];
|
|
551
|
-
const visit = (
|
|
552
|
-
if (visited.has(nodeId)) return;
|
|
553
|
-
visited.add(nodeId);
|
|
554
|
-
const candidate = context.plan.graph.node(nodeId);
|
|
555
|
-
if (!candidate) return;
|
|
556
|
-
for (const need of candidate.needs) visit(need);
|
|
557
|
-
if (hasSchedulingRole(context, candidate, "implementation")) ordered.push(candidate);
|
|
558
|
-
};
|
|
705
|
+
const visit = (candidateId) => visitImplementationNode(context, visited, ordered, candidateId, visit);
|
|
559
706
|
for (const need of node.needs) visit(need);
|
|
560
707
|
return ordered;
|
|
561
708
|
}
|
|
709
|
+
function visitImplementationNode(context, visited, ordered, nodeId, visit) {
|
|
710
|
+
if (visited.has(nodeId)) return;
|
|
711
|
+
visited.add(nodeId);
|
|
712
|
+
const candidate = context.plan.graph.node(nodeId);
|
|
713
|
+
if (!candidate) return;
|
|
714
|
+
visitImplementationDependencies(candidate, visit);
|
|
715
|
+
appendImplementationNode(context, ordered, candidate);
|
|
716
|
+
}
|
|
717
|
+
function visitImplementationDependencies(candidate, visit) {
|
|
718
|
+
for (const need of candidate.needs) visit(need);
|
|
719
|
+
}
|
|
720
|
+
function appendImplementationNode(context, ordered, candidate) {
|
|
721
|
+
if (hasSchedulingRole(context, candidate, "implementation")) ordered.push(candidate);
|
|
722
|
+
}
|
|
562
723
|
function hasSchedulingRole(context, node, role) {
|
|
563
724
|
return node.profile ? context.config.profiles[node.profile]?.scheduling_roles?.includes(role) ?? false : false;
|
|
564
725
|
}
|
|
565
|
-
|
|
566
|
-
if (delayMs <= 0 || signal?.aborted) return;
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
726
|
+
function waitForRetryDelay(delayMs, signal) {
|
|
727
|
+
if (delayMs <= 0 || signal?.aborted) return Effect.void;
|
|
728
|
+
return Effect.race(Effect.sleep(delayMs), waitForAbort(signal));
|
|
729
|
+
}
|
|
730
|
+
function waitForAbort(signal) {
|
|
731
|
+
if (!signal) return Effect.never;
|
|
732
|
+
return Effect.async((resume) => {
|
|
733
|
+
const onAbort = () => resume(Effect.void);
|
|
734
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
735
|
+
return Effect.sync(() => signal.removeEventListener("abort", onAbort));
|
|
574
736
|
});
|
|
575
737
|
}
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
738
|
+
function executeNodeAttemptCycle(node, context, attempt, previous) {
|
|
739
|
+
return Effect.gen(function* () {
|
|
740
|
+
const startResult = yield* beginNodeAttempt(node, context, attempt, previous);
|
|
741
|
+
if (startResult) return startResult;
|
|
742
|
+
const last = yield* runNodeAttemptBody(node, context, attempt);
|
|
743
|
+
const cancelledAfterAttempt = cancelledNodeResult(context, node.id, attempt, last);
|
|
744
|
+
if (cancelledAfterAttempt) return {
|
|
745
|
+
last,
|
|
746
|
+
result: cancelledAfterAttempt
|
|
747
|
+
};
|
|
748
|
+
return yield* finishNodeAttemptAfterGates(node, context, attempt, last);
|
|
586
749
|
});
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
750
|
+
}
|
|
751
|
+
function beginNodeAttempt(node, context, attempt, previous) {
|
|
752
|
+
return Effect.gen(function* () {
|
|
753
|
+
if (isCancelled(context)) return cancelledCycle(node.id, attempt, previous);
|
|
754
|
+
emitNodeStart(context, node, attempt);
|
|
590
755
|
recordNodeEvent(context, node.id, {
|
|
591
756
|
at: now(),
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
type: "FAILED"
|
|
757
|
+
attempt,
|
|
758
|
+
type: "STARTED"
|
|
595
759
|
});
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
760
|
+
const startHook = yield* dispatchHooksEffect(context, "node.start", void 0, node);
|
|
761
|
+
if (startHook) return failedHookCycle(node.id, attempt, previous, startHook.evidence, context);
|
|
762
|
+
return isCancelled(context) ? cancelledCycle(node.id, attempt, previous) : null;
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
function cancelledCycle(nodeId, attempt, previous) {
|
|
766
|
+
return {
|
|
602
767
|
last: previous,
|
|
603
|
-
result: nodeFailure(
|
|
768
|
+
result: nodeFailure(nodeId, attempt, cancelledFailure().evidence, previous.output)
|
|
604
769
|
};
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
context.nodeStateStore.setSnapshot(node.id, await snapshotChangedFiles(context.worktreePath));
|
|
610
|
-
recordNodeEvent(context, node.id, {
|
|
770
|
+
}
|
|
771
|
+
function failedHookCycle(nodeId, attempt, previous, evidence, context) {
|
|
772
|
+
const result = nodeFailure(nodeId, attempt, evidence, previous.output);
|
|
773
|
+
recordNodeEvent(context, nodeId, {
|
|
611
774
|
at: now(),
|
|
612
|
-
|
|
775
|
+
failure: nodeRuntimeFailure(result),
|
|
776
|
+
result,
|
|
777
|
+
type: "FAILED"
|
|
613
778
|
});
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
779
|
+
return {
|
|
780
|
+
last: previous,
|
|
781
|
+
result
|
|
782
|
+
};
|
|
783
|
+
}
|
|
784
|
+
function runNodeAttemptBody(node, context, attempt) {
|
|
785
|
+
return Effect.gen(function* () {
|
|
786
|
+
recordNodeEvent(context, node.id, {
|
|
787
|
+
at: now(),
|
|
788
|
+
type: "START_HOOKS_FINISHED"
|
|
789
|
+
});
|
|
790
|
+
context.nodeStateStore.setSnapshot(node.id, yield* snapshotChangedFilesEffect(context.worktreePath));
|
|
791
|
+
recordNodeEvent(context, node.id, {
|
|
792
|
+
at: now(),
|
|
793
|
+
type: "SNAPSHOT_BEFORE_FINISHED"
|
|
794
|
+
});
|
|
795
|
+
recordNodeEvent(context, node.id, {
|
|
796
|
+
at: now(),
|
|
797
|
+
type: "RUNNER_STARTED"
|
|
798
|
+
});
|
|
799
|
+
const last = yield* executeNodeAttempt(node, context, attempt);
|
|
800
|
+
recordNodeEvent(context, node.id, runnerFinishedEvent(last));
|
|
801
|
+
yield* recordAttemptOutput(node, context, attempt, last);
|
|
802
|
+
return last;
|
|
617
803
|
});
|
|
618
|
-
|
|
619
|
-
|
|
804
|
+
}
|
|
805
|
+
function runnerFinishedEvent(last) {
|
|
806
|
+
return {
|
|
620
807
|
at: now(),
|
|
621
808
|
evidence: last.evidence,
|
|
622
809
|
exitCode: last.exitCode,
|
|
623
810
|
output: last.output,
|
|
624
811
|
timedOut: last.timedOut,
|
|
625
812
|
type: "RUNNER_FINISHED"
|
|
626
|
-
});
|
|
627
|
-
const afterSnapshot = await snapshotChangedFiles(context.worktreePath);
|
|
628
|
-
const beforeSnapshot = context.nodeStateStore.getSnapshot(node.id);
|
|
629
|
-
if (beforeSnapshot) context.nodeStateStore.setSnapshot(node.id, diffChangedFiles(beforeSnapshot, afterSnapshot, context.worktreePath));
|
|
630
|
-
context.nodeStateStore.recordOutput(node.id, last.output);
|
|
631
|
-
context.nodeStateStore.recordHandoff(node.id, last.handoff);
|
|
632
|
-
emitNodeOutputRecorded(context, node, attempt, last.output);
|
|
633
|
-
recordNodeEvent(context, node.id, {
|
|
634
|
-
at: now(),
|
|
635
|
-
type: "OUTPUT_RECORDED"
|
|
636
|
-
});
|
|
637
|
-
recordNodeEvent(context, node.id, {
|
|
638
|
-
at: now(),
|
|
639
|
-
type: "SNAPSHOT_AFTER_FINISHED"
|
|
640
|
-
});
|
|
641
|
-
const cancelledAfterAttempt = cancelledNodeResult(context, node.id, attempt, last);
|
|
642
|
-
if (cancelledAfterAttempt) return {
|
|
643
|
-
last,
|
|
644
|
-
result: cancelledAfterAttempt
|
|
645
813
|
};
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
814
|
+
}
|
|
815
|
+
function recordAttemptOutput(node, context, attempt, last) {
|
|
816
|
+
return Effect.gen(function* () {
|
|
817
|
+
const afterSnapshot = yield* snapshotChangedFilesEffect(context.worktreePath);
|
|
818
|
+
const beforeSnapshot = context.nodeStateStore.getSnapshot(node.id);
|
|
819
|
+
if (beforeSnapshot) context.nodeStateStore.setSnapshot(node.id, diffChangedFiles(beforeSnapshot, afterSnapshot, context.worktreePath));
|
|
820
|
+
context.nodeStateStore.recordOutput(node.id, last.output);
|
|
821
|
+
context.nodeStateStore.recordHandoff(node.id, last.handoff);
|
|
822
|
+
emitNodeOutputRecorded(context, node, attempt, last.output);
|
|
823
|
+
recordNodeEvent(context, node.id, {
|
|
824
|
+
at: now(),
|
|
825
|
+
type: "OUTPUT_RECORDED"
|
|
826
|
+
});
|
|
827
|
+
recordNodeEvent(context, node.id, {
|
|
828
|
+
at: now(),
|
|
829
|
+
type: "SNAPSHOT_AFTER_FINISHED"
|
|
830
|
+
});
|
|
655
831
|
});
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
if (!failedGate && last.exitCode === 0) {
|
|
663
|
-
const successHook = await dispatchHooks(context, "node.success", void 0, node);
|
|
664
|
-
if (successHook) {
|
|
665
|
-
const result = nodeFailure(node.id, attempt, successHook.evidence, last.output);
|
|
666
|
-
recordNodeEvent(context, node.id, {
|
|
667
|
-
at: now(),
|
|
668
|
-
failure: nodeRuntimeFailure(result),
|
|
669
|
-
result,
|
|
670
|
-
type: "FAILED"
|
|
671
|
-
});
|
|
672
|
-
return {
|
|
673
|
-
last,
|
|
674
|
-
result
|
|
675
|
-
};
|
|
676
|
-
}
|
|
677
|
-
const cancelledAfterHook = cancelledNodeResult(context, node.id, attempt, last);
|
|
678
|
-
if (cancelledAfterHook) return {
|
|
832
|
+
}
|
|
833
|
+
function finishNodeAttemptAfterGates(node, context, attempt, last) {
|
|
834
|
+
return Effect.gen(function* () {
|
|
835
|
+
const gateResults = yield* evaluateGatesForAttempt(node, context, last);
|
|
836
|
+
const cancelledAfterGates = cancelledNodeResult(context, node.id, attempt, last);
|
|
837
|
+
if (cancelledAfterGates) return {
|
|
679
838
|
last,
|
|
680
|
-
result:
|
|
681
|
-
};
|
|
682
|
-
const result = {
|
|
683
|
-
attempts: attempt,
|
|
684
|
-
evidence: last.evidence,
|
|
685
|
-
exitCode: 0,
|
|
686
|
-
nodeId: node.id,
|
|
687
|
-
output: last.output,
|
|
688
|
-
status: "passed"
|
|
839
|
+
result: cancelledAfterGates
|
|
689
840
|
};
|
|
841
|
+
return yield* finishNodeAttemptWithGate(node, context, attempt, last, gateResults.find((gate) => !gate.passed));
|
|
842
|
+
});
|
|
843
|
+
}
|
|
844
|
+
function evaluateGatesForAttempt(node, context, last) {
|
|
845
|
+
return Effect.gen(function* () {
|
|
690
846
|
recordNodeEvent(context, node.id, {
|
|
691
847
|
at: now(),
|
|
692
|
-
|
|
693
|
-
type: "PASSED"
|
|
848
|
+
type: "GATES_STARTED"
|
|
694
849
|
});
|
|
695
|
-
|
|
850
|
+
const gateResults = yield* Effect.tryPromise(() => evaluateNodeGates(node, context, last, (failedNode, result) => Effect.runPromise(dispatchGateFailureHook(context, failedNode, result))));
|
|
851
|
+
recordNodeEvent(context, node.id, {
|
|
852
|
+
at: now(),
|
|
853
|
+
gates: gateResults,
|
|
854
|
+
type: "GATES_FINISHED"
|
|
855
|
+
});
|
|
856
|
+
return gateResults;
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
function finishNodeAttemptWithGate(node, context, attempt, last, failedGate) {
|
|
860
|
+
if (failedGate || last.exitCode !== 0) return Effect.succeed(retryCycle(node, attempt, last, failedGate));
|
|
861
|
+
return successfulAttemptCycle(node, context, attempt, last);
|
|
862
|
+
}
|
|
863
|
+
function successfulAttemptCycle(node, context, attempt, last) {
|
|
864
|
+
return Effect.gen(function* () {
|
|
865
|
+
const successHook = yield* dispatchHooksEffect(context, "node.success", void 0, node);
|
|
866
|
+
if (successHook) return failedHookCycle(node.id, attempt, last, successHook.evidence, context);
|
|
867
|
+
const cancelledAfterHook = cancelledNodeResult(context, node.id, attempt, last);
|
|
868
|
+
return cancelledAfterHook ? {
|
|
696
869
|
last,
|
|
697
|
-
result
|
|
698
|
-
};
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
|
|
870
|
+
result: cancelledAfterHook
|
|
871
|
+
} : passedCycle(node.id, attempt, last, context);
|
|
872
|
+
});
|
|
873
|
+
}
|
|
874
|
+
function passedCycle(nodeId, attempt, last, context) {
|
|
875
|
+
const result = passedNodeResult(nodeId, attempt, last);
|
|
876
|
+
recordNodeEvent(context, nodeId, {
|
|
877
|
+
at: now(),
|
|
878
|
+
result,
|
|
879
|
+
type: "PASSED"
|
|
880
|
+
});
|
|
881
|
+
return {
|
|
882
|
+
last,
|
|
883
|
+
result
|
|
884
|
+
};
|
|
885
|
+
}
|
|
886
|
+
function passedNodeResult(nodeId, attempt, last) {
|
|
887
|
+
return {
|
|
888
|
+
attempts: attempt,
|
|
889
|
+
evidence: last.evidence,
|
|
890
|
+
exitCode: 0,
|
|
891
|
+
nodeId,
|
|
892
|
+
output: last.output,
|
|
893
|
+
status: "passed"
|
|
894
|
+
};
|
|
895
|
+
}
|
|
896
|
+
function retryCycle(node, attempt, last, failedGate) {
|
|
702
897
|
return {
|
|
703
898
|
last,
|
|
704
899
|
retry: {
|
|
705
900
|
attempt,
|
|
706
|
-
evidence,
|
|
707
|
-
gate:
|
|
708
|
-
reason:
|
|
709
|
-
retryReason
|
|
901
|
+
evidence: retryEvidence(last, failedGate),
|
|
902
|
+
gate: retryGateId(node.id, failedGate),
|
|
903
|
+
reason: retryReasonText(last.exitCode, failedGate),
|
|
904
|
+
retryReason: nodeRetryReason(last, failedGate)
|
|
710
905
|
}
|
|
711
906
|
};
|
|
712
907
|
}
|
|
908
|
+
function retryGateId(nodeId, failedGate) {
|
|
909
|
+
return failedGate ? failedGate.gateId : nodeId;
|
|
910
|
+
}
|
|
911
|
+
function retryReasonText(exitCode, failedGate) {
|
|
912
|
+
if (!failedGate) return `node exited with code ${exitCode}`;
|
|
913
|
+
return failedGate.reason ?? `node exited with code ${exitCode}`;
|
|
914
|
+
}
|
|
915
|
+
function retryEvidence(last, failedGate) {
|
|
916
|
+
return failedGate ? [...last.evidence, ...failedGate.evidence] : last.evidence.concat(`node exited with code ${last.exitCode}`);
|
|
917
|
+
}
|
|
918
|
+
function snapshotChangedFilesEffect(worktreePath) {
|
|
919
|
+
return Effect.sync(() => snapshotChangedFiles(worktreePath));
|
|
920
|
+
}
|
|
713
921
|
function nodeRetryReason(attempt, failedGate) {
|
|
714
922
|
if (attempt.timedOut) return "timeout";
|
|
715
923
|
if (failedGate) return "gate_failure";
|
|
@@ -752,35 +960,47 @@ function nodeFailure(nodeId, attempts, evidence, output) {
|
|
|
752
960
|
};
|
|
753
961
|
}
|
|
754
962
|
function executeNodeAttempt(node, context, attempt) {
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
963
|
+
return nodeAttemptExecutors[node.kind](node, context, attempt);
|
|
964
|
+
}
|
|
965
|
+
const nodeAttemptExecutors = {
|
|
966
|
+
agent: executeAgentAttempt,
|
|
967
|
+
builtin: executeBuiltinAttempt,
|
|
968
|
+
command: executeCommandAttempt,
|
|
969
|
+
group: executeGroupAttempt,
|
|
970
|
+
parallel: executeParallelAttempt
|
|
971
|
+
};
|
|
972
|
+
function executeAgentAttempt(node, context, attempt) {
|
|
973
|
+
return Effect.tryPromise(() => executeAgentNode(node, context, attempt));
|
|
974
|
+
}
|
|
975
|
+
function executeCommandAttempt(node, context) {
|
|
976
|
+
return Effect.tryPromise(() => executeCommand(node.command ?? [], context, { timeout: node.timeoutMs }));
|
|
977
|
+
}
|
|
978
|
+
function executeBuiltinAttempt(node, context) {
|
|
979
|
+
return Effect.tryPromise(() => executeBuiltin(node.builtin ?? "", context, node));
|
|
980
|
+
}
|
|
981
|
+
function executeGroupAttempt(node) {
|
|
982
|
+
return Effect.succeed({
|
|
983
|
+
evidence: [`group '${node.id}' completed`],
|
|
984
|
+
exitCode: 0,
|
|
985
|
+
output: ""
|
|
986
|
+
});
|
|
987
|
+
}
|
|
988
|
+
function executeParallelAttempt(node, context) {
|
|
989
|
+
return Effect.tryPromise(() => executeParallelNode(node, context, {
|
|
990
|
+
executeNode: (child, childContext) => Effect.runPromise(executeNode(child, childContext)),
|
|
991
|
+
markNodeReady: (childContext, childId) => recordNodeEvent(childContext, childId, {
|
|
992
|
+
at: now(),
|
|
993
|
+
type: "READY"
|
|
994
|
+
})
|
|
995
|
+
}));
|
|
776
996
|
}
|
|
777
|
-
|
|
778
|
-
|
|
997
|
+
function dispatchGateFailureHook(context, node, result) {
|
|
998
|
+
return Effect.asVoid(dispatchHooksEffect(context, "gate.failure", {
|
|
779
999
|
evidence: result.evidence,
|
|
780
1000
|
gate: result.gateId,
|
|
781
1001
|
nodeId: node.id,
|
|
782
1002
|
reason: result.reason ?? "gate failed"
|
|
783
|
-
}, node, result.gateId);
|
|
1003
|
+
}, node, result.gateId));
|
|
784
1004
|
}
|
|
785
1005
|
function formatConfigError(err) {
|
|
786
1006
|
return [err.message, ...err.issues.map((issue) => issue.path ? `- ${issue.path}: ${issue.message}` : `- ${issue.message}`)].join("\n");
|