@oisincoveney/pipeline 3.15.0 → 3.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/bench-command.js +1 -1
- package/dist/commands/pipeline-command.js +1 -1
- package/dist/commands/runner-command-command.js +1 -1
- package/dist/config/schemas.d.ts +17 -17
- package/dist/loop/argo-poll.js +3 -3
- package/dist/loop/merge.js +2 -2
- package/dist/mcp/gateway.js +3 -3
- package/dist/moka-submit.d.ts +7 -7
- package/dist/pipeline-runtime.js +32 -232
- package/dist/planning/generate.d.ts +20 -20
- package/dist/run-control/commands.js +2 -2
- package/dist/run-control/detach.js +1 -1
- package/dist/run-control/run-state-lock.js +4 -0
- package/dist/run-control/runtime-event-projection.js +98 -0
- package/dist/run-control/runtime-reporter.js +26 -89
- package/dist/run-control/store.js +3 -3
- package/dist/run-control/supervisor.js +7 -6
- package/dist/runner-command/finalize.js +1 -1
- package/dist/runner-command/lifecycle.js +1 -1
- package/dist/runner-command/run.js +4 -4
- package/dist/runner-command-contract.d.ts +2 -2
- package/dist/runner-event-schema.d.ts +10 -10
- package/dist/runtime/agent-node/agent-node.js +3 -3
- package/dist/runtime/changed-files/changed-files.js +1 -1
- package/dist/runtime/drain-merge/drain-merge.js +6 -6
- package/dist/runtime/durable-store/postgres/postgres-store.js +4 -3
- package/dist/runtime/json-validation/json-validation.js +1 -1
- package/dist/runtime/local-scheduler.js +1 -1
- package/dist/runtime/node-state-tracker.js +133 -58
- package/dist/runtime/open-pull-request/open-pull-request.js +11 -11
- package/dist/runtime/opencode-server.js +1 -1
- package/dist/runtime/opencode-session-executor.js +22 -16
- package/dist/runtime/parallel-node/parallel-node.js +2 -2
- package/dist/runtime/remediation/remediation.js +246 -0
- package/dist/runtime/scheduler.js +1 -1
- package/dist/runtime/services/agent-node-runtime-service.js +1 -1
- package/dist/runtime/services/backlog-service.d.ts +1 -1
- package/dist/runtime/services/backlog-service.js +1 -1
- package/dist/runtime/services/command-executor-service.js +1 -1
- package/dist/runtime/services/config-io-service.js +2 -2
- package/dist/runtime/services/drain-merge-git-service.js +1 -1
- package/dist/runtime/services/file-system-service.js +2 -2
- package/dist/runtime/services/git-porcelain-service.js +1 -1
- package/dist/runtime/services/kubernetes-argo-service.js +2 -2
- package/dist/runtime/services/mcp-gateway-service.js +2 -2
- package/dist/runtime/services/open-pull-request-git-service.js +1 -1
- package/dist/runtime/services/opencode-runtime-server-service.js +1 -1
- package/dist/runtime/services/opencode-sdk-service.js +1 -1
- package/dist/runtime/services/repo-io-service.js +2 -2
- package/dist/runtime/services/runner-command-io-service.js +4 -4
- package/dist/runtime/services/runner-event-sink-http-service.js +1 -1
- package/dist/runtime/services/worktree-service.js +2 -2
- package/dist/runtime/workflow-lifecycle.js +2 -2
- package/dist/serialized-write-queue.js +35 -0
- package/dist/tickets/ticket-graph-dto.js +1 -1
- package/docs/runtime-actor-model.md +30 -0
- package/package.json +3 -3
|
@@ -2,7 +2,7 @@ import { buildEvalReport, renderEvalReport } from "../bench/eval-report.js";
|
|
|
2
2
|
import { Context, Effect, Layer } from "effect";
|
|
3
3
|
import { readFileSync } from "node:fs";
|
|
4
4
|
//#region src/commands/bench-command.ts
|
|
5
|
-
var BenchCommandService = class extends Context.
|
|
5
|
+
var BenchCommandService = class extends Context.Service()("BenchCommandService") {};
|
|
6
6
|
const BenchCommandServiceLive = Layer.succeed(BenchCommandService, {
|
|
7
7
|
readResults: (path) => Effect.try(() => JSON.parse(readFileSync(path, "utf8"))),
|
|
8
8
|
writeReport: (report) => Effect.try(() => process.stdout.write(`${report}\n`))
|
|
@@ -12,7 +12,7 @@ const BUILTIN_PIPE_COMMANDS = new Set([
|
|
|
12
12
|
"runner-command",
|
|
13
13
|
"ticket"
|
|
14
14
|
]);
|
|
15
|
-
var EntrypointCommandService = class extends Context.
|
|
15
|
+
var EntrypointCommandService = class extends Context.Service()("EntrypointCommandService") {};
|
|
16
16
|
const createEntrypointCommandServiceLive = (runEntrypoint) => Layer.succeed(EntrypointCommandService, { runEntrypoint: (entrypoint, task, opts) => Effect.tryPromise({
|
|
17
17
|
catch: (error) => error,
|
|
18
18
|
try: () => runEntrypoint(entrypoint, task, opts)
|
|
@@ -3,7 +3,7 @@ import { runRunnerFinalize } from "../runner-command/finalize.js";
|
|
|
3
3
|
import { runRunnerLifecycle } from "../runner-command/lifecycle.js";
|
|
4
4
|
import { Context, Effect, Layer } from "effect";
|
|
5
5
|
//#region src/commands/runner-command-command.ts
|
|
6
|
-
var RunnerCommandService = class extends Context.
|
|
6
|
+
var RunnerCommandService = class extends Context.Service()("RunnerCommandService") {};
|
|
7
7
|
const RunnerCommandServiceLive = Layer.succeed(RunnerCommandService, {
|
|
8
8
|
finalize: (options) => Effect.tryPromise({
|
|
9
9
|
catch: (error) => error,
|
package/dist/config/schemas.d.ts
CHANGED
|
@@ -99,10 +99,10 @@ declare const workflowNodeBaseSchema: z.ZodObject<{
|
|
|
99
99
|
models: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
100
100
|
needs: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
101
101
|
reasoning_effort: z.ZodOptional<z.ZodEnum<{
|
|
102
|
-
none: "none";
|
|
103
|
-
low: "low";
|
|
104
|
-
medium: "medium";
|
|
105
102
|
high: "high";
|
|
103
|
+
medium: "medium";
|
|
104
|
+
low: "low";
|
|
105
|
+
none: "none";
|
|
106
106
|
xhigh: "xhigh";
|
|
107
107
|
}>>;
|
|
108
108
|
retries: z.ZodOptional<z.ZodObject<{
|
|
@@ -233,8 +233,8 @@ declare const configSchema: z.ZodObject<{
|
|
|
233
233
|
policy: z.ZodOptional<z.ZodObject<{
|
|
234
234
|
commands: z.ZodOptional<z.ZodEnum<{
|
|
235
235
|
allow: "allow";
|
|
236
|
-
deny: "deny";
|
|
237
236
|
"trusted-only": "trusted-only";
|
|
237
|
+
deny: "deny";
|
|
238
238
|
}>>;
|
|
239
239
|
modules: z.ZodOptional<z.ZodEnum<{
|
|
240
240
|
allow: "allow";
|
|
@@ -262,8 +262,8 @@ declare const configSchema: z.ZodObject<{
|
|
|
262
262
|
global: "global";
|
|
263
263
|
}>>;
|
|
264
264
|
mode: z.ZodEnum<{
|
|
265
|
-
local: "local";
|
|
266
265
|
hosted: "hosted";
|
|
266
|
+
local: "local";
|
|
267
267
|
}>;
|
|
268
268
|
provider: z.ZodLiteral<"toolhive">;
|
|
269
269
|
authorization_env: z.ZodDefault<z.ZodString>;
|
|
@@ -307,10 +307,10 @@ declare const configSchema: z.ZodObject<{
|
|
|
307
307
|
}, z.core.$strict>>;
|
|
308
308
|
output: z.ZodOptional<z.ZodObject<{
|
|
309
309
|
format: z.ZodEnum<{
|
|
310
|
-
json_schema: "json_schema";
|
|
311
310
|
text: "text";
|
|
312
311
|
json: "json";
|
|
313
312
|
jsonl: "jsonl";
|
|
313
|
+
json_schema: "json_schema";
|
|
314
314
|
}>;
|
|
315
315
|
repair: z.ZodOptional<z.ZodObject<{
|
|
316
316
|
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -320,10 +320,10 @@ declare const configSchema: z.ZodObject<{
|
|
|
320
320
|
schema_path: z.ZodOptional<z.ZodString>;
|
|
321
321
|
}, z.core.$strict>>;
|
|
322
322
|
reasoning_effort: z.ZodOptional<z.ZodEnum<{
|
|
323
|
-
none: "none";
|
|
324
|
-
low: "low";
|
|
325
|
-
medium: "medium";
|
|
326
323
|
high: "high";
|
|
324
|
+
medium: "medium";
|
|
325
|
+
low: "low";
|
|
326
|
+
none: "none";
|
|
327
327
|
xhigh: "xhigh";
|
|
328
328
|
}>>;
|
|
329
329
|
rules: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
@@ -386,10 +386,10 @@ declare const configSchema: z.ZodObject<{
|
|
|
386
386
|
disabled: "disabled";
|
|
387
387
|
}>>>;
|
|
388
388
|
output_formats: z.ZodOptional<z.ZodArray<z.ZodEnum<{
|
|
389
|
-
json_schema: "json_schema";
|
|
390
389
|
text: "text";
|
|
391
390
|
json: "json";
|
|
392
391
|
jsonl: "jsonl";
|
|
392
|
+
json_schema: "json_schema";
|
|
393
393
|
}>>>;
|
|
394
394
|
rules: z.ZodOptional<z.ZodBoolean>;
|
|
395
395
|
skills: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -408,10 +408,10 @@ declare const configSchema: z.ZodObject<{
|
|
|
408
408
|
host_models: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
409
409
|
model: z.ZodOptional<z.ZodString>;
|
|
410
410
|
reasoning_effort: z.ZodOptional<z.ZodEnum<{
|
|
411
|
-
none: "none";
|
|
412
|
-
low: "low";
|
|
413
|
-
medium: "medium";
|
|
414
411
|
high: "high";
|
|
412
|
+
medium: "medium";
|
|
413
|
+
low: "low";
|
|
414
|
+
none: "none";
|
|
415
415
|
xhigh: "xhigh";
|
|
416
416
|
}>>;
|
|
417
417
|
type: z.ZodEnum<{
|
|
@@ -497,10 +497,10 @@ declare const configSchema: z.ZodObject<{
|
|
|
497
497
|
models: z.ZodArray<z.ZodString>;
|
|
498
498
|
profile: z.ZodString;
|
|
499
499
|
reasoning_effort: z.ZodOptional<z.ZodEnum<{
|
|
500
|
-
none: "none";
|
|
501
|
-
low: "low";
|
|
502
|
-
medium: "medium";
|
|
503
500
|
high: "high";
|
|
501
|
+
medium: "medium";
|
|
502
|
+
low: "low";
|
|
503
|
+
none: "none";
|
|
504
504
|
xhigh: "xhigh";
|
|
505
505
|
}>>;
|
|
506
506
|
}, z.core.$strict>>>;
|
|
@@ -510,8 +510,8 @@ declare const configSchema: z.ZodObject<{
|
|
|
510
510
|
schedules: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
511
511
|
description: z.ZodOptional<z.ZodString>;
|
|
512
512
|
baseline: z.ZodEnum<{
|
|
513
|
-
execute: "execute";
|
|
514
513
|
quick: "quick";
|
|
514
|
+
execute: "execute";
|
|
515
515
|
}>;
|
|
516
516
|
max_parallel_nodes: z.ZodOptional<z.ZodNumber>;
|
|
517
517
|
node_catalog: z.ZodOptional<z.ZodString>;
|
package/dist/loop/argo-poll.js
CHANGED
|
@@ -104,18 +104,18 @@ function pollLoop(state) {
|
|
|
104
104
|
workflowReadApi: state.workflowReadApi
|
|
105
105
|
}).pipe(Effect.flatMap((phase) => {
|
|
106
106
|
if (isTerminal(phase)) return Effect.succeed(phase);
|
|
107
|
-
return Effect.sleep(Duration.millis(state.pollIntervalMs)).pipe(Effect.
|
|
107
|
+
return Effect.sleep(Duration.millis(state.pollIntervalMs)).pipe(Effect.andThen(pollLoop({
|
|
108
108
|
...state,
|
|
109
109
|
errorCount: 0
|
|
110
110
|
})));
|
|
111
|
-
}), Effect.
|
|
111
|
+
}), Effect.catch((error) => handlePollError(state, error)));
|
|
112
112
|
}
|
|
113
113
|
function handlePollError(state, error) {
|
|
114
114
|
const nextErrorCount = state.errorCount + 1;
|
|
115
115
|
state.onTransientError?.(error, nextErrorCount);
|
|
116
116
|
if (nextErrorCount > state.maxRetries) return Effect.fail(error);
|
|
117
117
|
const delay = Duration.millis(RETRY_BASE_DELAY_MS * 2 ** (nextErrorCount - 1));
|
|
118
|
-
return Effect.sleep(delay).pipe(Effect.
|
|
118
|
+
return Effect.sleep(delay).pipe(Effect.andThen(pollLoop({
|
|
119
119
|
...state,
|
|
120
120
|
errorCount: nextErrorCount
|
|
121
121
|
})));
|
package/dist/loop/merge.js
CHANGED
|
@@ -49,7 +49,7 @@ function enableAutoMerge(pr, gh) {
|
|
|
49
49
|
return gh.text(args).pipe(Effect.map(() => ({
|
|
50
50
|
_tag: "pending",
|
|
51
51
|
pr: pr.number
|
|
52
|
-
})), Effect.
|
|
52
|
+
})), Effect.catch((error) => Effect.succeed(toBlocked(pr, error))));
|
|
53
53
|
}
|
|
54
54
|
/**
|
|
55
55
|
* Admin-merge through branch protection using the bypass token. The token is
|
|
@@ -71,7 +71,7 @@ function adminMerge(pr, token, gh) {
|
|
|
71
71
|
return gh.text(args, { secretEnv: { GH_TOKEN: token.reveal() } }).pipe(Effect.map(() => ({
|
|
72
72
|
_tag: "merged",
|
|
73
73
|
pr: pr.number
|
|
74
|
-
})), Effect.
|
|
74
|
+
})), Effect.catch((error) => Effect.succeed(toBlocked(pr, error))));
|
|
75
75
|
}
|
|
76
76
|
function toBlocked(pr, error) {
|
|
77
77
|
const message = error instanceof Error ? error.message : String(error);
|
package/dist/mcp/gateway.js
CHANGED
|
@@ -188,7 +188,7 @@ function checkGatewayRequiredTools(gateway) {
|
|
|
188
188
|
name: "gateway-required-tools",
|
|
189
189
|
passed: false
|
|
190
190
|
};
|
|
191
|
-
}).pipe(Effect.
|
|
191
|
+
}).pipe(Effect.catch((error) => Effect.succeed({
|
|
192
192
|
detail: error instanceof Error ? error.message : String(error),
|
|
193
193
|
name: "gateway-required-tools",
|
|
194
194
|
passed: false
|
|
@@ -317,7 +317,7 @@ function checkThv(cwd) {
|
|
|
317
317
|
name: "toolhive",
|
|
318
318
|
passed: true
|
|
319
319
|
};
|
|
320
|
-
}).pipe(Effect.
|
|
320
|
+
}).pipe(Effect.catch((error) => Effect.succeed({
|
|
321
321
|
detail: error.message || "not available",
|
|
322
322
|
name: "toolhive",
|
|
323
323
|
passed: false
|
|
@@ -342,7 +342,7 @@ function checkGatewayHealth(gateway) {
|
|
|
342
342
|
name: "gateway-health",
|
|
343
343
|
passed
|
|
344
344
|
};
|
|
345
|
-
}).pipe(Effect.
|
|
345
|
+
}).pipe(Effect.catch((error) => Effect.succeed({
|
|
346
346
|
detail: error instanceof Error ? error.message : String(error),
|
|
347
347
|
name: "gateway-health",
|
|
348
348
|
passed: false
|
package/dist/moka-submit.d.ts
CHANGED
|
@@ -6,13 +6,13 @@ import { z } from "zod";
|
|
|
6
6
|
//#region src/moka-submit.d.ts
|
|
7
7
|
declare const mokaSubmitDirectHooksSchema: z.ZodRecord<z.ZodEnum<{
|
|
8
8
|
"workflow.start": "workflow.start";
|
|
9
|
+
"node.finish": "node.finish";
|
|
10
|
+
"node.start": "node.start";
|
|
9
11
|
"workflow.success": "workflow.success";
|
|
10
12
|
"workflow.failure": "workflow.failure";
|
|
11
13
|
"workflow.complete": "workflow.complete";
|
|
12
|
-
"node.start": "node.start";
|
|
13
14
|
"node.success": "node.success";
|
|
14
15
|
"node.error": "node.error";
|
|
15
|
-
"node.finish": "node.finish";
|
|
16
16
|
"gate.failure": "gate.failure";
|
|
17
17
|
}> & z.core.$partial, z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
18
18
|
failure: z.ZodDefault<z.ZodEnum<{
|
|
@@ -99,13 +99,13 @@ declare const mokaSubmitOptionsSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
99
99
|
}, z.core.$strict>>;
|
|
100
100
|
hooks: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
101
101
|
"workflow.start": "workflow.start";
|
|
102
|
+
"node.finish": "node.finish";
|
|
103
|
+
"node.start": "node.start";
|
|
102
104
|
"workflow.success": "workflow.success";
|
|
103
105
|
"workflow.failure": "workflow.failure";
|
|
104
106
|
"workflow.complete": "workflow.complete";
|
|
105
|
-
"node.start": "node.start";
|
|
106
107
|
"node.success": "node.success";
|
|
107
108
|
"node.error": "node.error";
|
|
108
|
-
"node.finish": "node.finish";
|
|
109
109
|
"gate.failure": "gate.failure";
|
|
110
110
|
}> & z.core.$partial, z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
111
111
|
failure: z.ZodDefault<z.ZodEnum<{
|
|
@@ -170,8 +170,8 @@ declare const mokaSubmitOptionsSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
170
170
|
}, z.core.$strict>>;
|
|
171
171
|
serviceAccountName: z.ZodOptional<z.ZodString>;
|
|
172
172
|
mode: z.ZodEnum<{
|
|
173
|
-
quick: "quick";
|
|
174
173
|
full: "full";
|
|
174
|
+
quick: "quick";
|
|
175
175
|
}>;
|
|
176
176
|
schedulePath: z.ZodOptional<z.ZodString>;
|
|
177
177
|
scheduleYaml: z.ZodOptional<z.ZodString>;
|
|
@@ -220,13 +220,13 @@ declare const mokaSubmitOptionsSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
220
220
|
}, z.core.$strict>>;
|
|
221
221
|
hooks: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
222
222
|
"workflow.start": "workflow.start";
|
|
223
|
+
"node.finish": "node.finish";
|
|
224
|
+
"node.start": "node.start";
|
|
223
225
|
"workflow.success": "workflow.success";
|
|
224
226
|
"workflow.failure": "workflow.failure";
|
|
225
227
|
"workflow.complete": "workflow.complete";
|
|
226
|
-
"node.start": "node.start";
|
|
227
228
|
"node.success": "node.success";
|
|
228
229
|
"node.error": "node.error";
|
|
229
|
-
"node.finish": "node.finish";
|
|
230
230
|
"gate.failure": "gate.failure";
|
|
231
231
|
}> & z.core.$partial, z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
232
232
|
failure: z.ZodDefault<z.ZodEnum<{
|
package/dist/pipeline-runtime.js
CHANGED
|
@@ -28,6 +28,7 @@ import { NodeStateTracker } from "./runtime/node-state-tracker.js";
|
|
|
28
28
|
import { configUsesOpencode, leaseOpencodeRuntime } from "./runtime/opencode-runtime.js";
|
|
29
29
|
import { executeParallelNode } from "./runtime/parallel-node/parallel-node.js";
|
|
30
30
|
import "./runtime/parallel-node/index.js";
|
|
31
|
+
import { remediateFailedNode } from "./runtime/remediation/remediation.js";
|
|
31
32
|
import { decideNodeRetry, nodeRetryPolicy } from "./runtime/retry.js";
|
|
32
33
|
import { Effect } from "effect";
|
|
33
34
|
//#region src/pipeline-runtime.ts
|
|
@@ -192,6 +193,18 @@ function executePlannedNode(nodeId, context) {
|
|
|
192
193
|
function dispatchHooksEffect(...args) {
|
|
193
194
|
return Effect.tryPromise(() => dispatchHooks(...args));
|
|
194
195
|
}
|
|
196
|
+
const runtimeRemediationDependencies = {
|
|
197
|
+
executeNode: executeReadyNode,
|
|
198
|
+
isCancelled,
|
|
199
|
+
snapshotChangedFiles: snapshotChangedFilesEffect
|
|
200
|
+
};
|
|
201
|
+
function executeReadyNode(node, context) {
|
|
202
|
+
recordNodeEvent(context, node.id, {
|
|
203
|
+
at: now(),
|
|
204
|
+
type: "READY"
|
|
205
|
+
});
|
|
206
|
+
return executeNode(node, context);
|
|
207
|
+
}
|
|
195
208
|
function plannedNodeById(context, nodeId) {
|
|
196
209
|
return context.plan.graph.node(nodeId) ?? findPlannedNode(context.plan.topologicalOrder, nodeId);
|
|
197
210
|
}
|
|
@@ -418,7 +431,7 @@ function runSingleNodeAttempt(node, context, retryPolicy, state, attempt) {
|
|
|
418
431
|
});
|
|
419
432
|
}
|
|
420
433
|
function nodeAttemptCycleOrError(node, context, attempt, last) {
|
|
421
|
-
return Effect.
|
|
434
|
+
return Effect.catch(executeNodeAttemptCycle(node, context, attempt, last), (error) => Effect.succeed({ error }));
|
|
422
435
|
}
|
|
423
436
|
function continueAfterAttemptCycle(node, context, retryPolicy, state, attempt, cycle) {
|
|
424
437
|
if (cycle.result) {
|
|
@@ -433,6 +446,7 @@ function continueAfterRetryCandidate(node, context, retryPolicy, retry, attempt)
|
|
|
433
446
|
const remediation = yield* remediateFailedNode({
|
|
434
447
|
attempt,
|
|
435
448
|
context,
|
|
449
|
+
dependencies: runtimeRemediationDependencies,
|
|
436
450
|
node,
|
|
437
451
|
retry
|
|
438
452
|
});
|
|
@@ -441,10 +455,25 @@ function continueAfterRetryCandidate(node, context, retryPolicy, retry, attempt)
|
|
|
441
455
|
emitRemediationPass(context, node.id, passed);
|
|
442
456
|
return passed;
|
|
443
457
|
}
|
|
444
|
-
if (remediationRequestsRetry(remediation))
|
|
458
|
+
if (remediationRequestsRetry(remediation)) {
|
|
459
|
+
recordRemediationRetryingNodeEvent(context, node.id, attempt, retry);
|
|
460
|
+
return "retry";
|
|
461
|
+
}
|
|
445
462
|
return yield* scheduleNodeRetry(node, context, retryPolicy, retry, attempt);
|
|
446
463
|
});
|
|
447
464
|
}
|
|
465
|
+
function recordRemediationRetryingNodeEvent(context, nodeId, attempt, retry) {
|
|
466
|
+
recordRetryingNodeEvent(context, nodeId, attempt, retry, {
|
|
467
|
+
attempt,
|
|
468
|
+
delayMs: 0,
|
|
469
|
+
evidence: retry.evidence,
|
|
470
|
+
exhausted: false,
|
|
471
|
+
gate: retry.gate,
|
|
472
|
+
reason: retry.reason,
|
|
473
|
+
retryReason: retry.retryReason,
|
|
474
|
+
scheduled: true
|
|
475
|
+
});
|
|
476
|
+
}
|
|
448
477
|
function remediationPassedResult(remediation) {
|
|
449
478
|
if (!remediation) return null;
|
|
450
479
|
return remediation.result ?? null;
|
|
@@ -562,242 +591,13 @@ function emitRuntimeRetry(context, nodeId, retry, reason) {
|
|
|
562
591
|
type: retry.scheduled ? "runtime.retry.scheduled" : "runtime.retry.exhausted"
|
|
563
592
|
});
|
|
564
593
|
}
|
|
565
|
-
function remediateFailedNode(input) {
|
|
566
|
-
return Effect.gen(function* () {
|
|
567
|
-
const selfRemediation = yield* remediateWritableNodeFailure(input);
|
|
568
|
-
if (selfRemediation) return { result: selfRemediation };
|
|
569
|
-
if (yield* remediateCoverageFailure(input)) return { retryNode: true };
|
|
570
|
-
if (yield* remediateUpstreamImplementationFailure(input)) return { retryNode: true };
|
|
571
|
-
return null;
|
|
572
|
-
});
|
|
573
|
-
}
|
|
574
|
-
function remediateWritableNodeFailure(input) {
|
|
575
|
-
return Effect.gen(function* () {
|
|
576
|
-
if (!canSelfRemediateWritableNode(input)) return null;
|
|
577
|
-
const beforeSnapshot = yield* snapshotChangedFilesEffect(input.context.worktreePath);
|
|
578
|
-
const beforeOutput = input.context.nodeStateStore.getOutput(input.node.id);
|
|
579
|
-
const result = yield* executeSelfRemediation(input);
|
|
580
|
-
if (result.status !== "passed") return null;
|
|
581
|
-
const changed = diffChangedFiles(beforeSnapshot, yield* snapshotChangedFilesEffect(input.context.worktreePath), input.context.worktreePath);
|
|
582
|
-
if (remediationChangedNothing(changed.files.size, result, beforeOutput)) return null;
|
|
583
|
-
input.context.nodeStateStore.setSnapshot(input.node.id, changed);
|
|
584
|
-
input.context.nodeStateStore.recordOutput(input.node.id, result.output);
|
|
585
|
-
return {
|
|
586
|
-
attempts: input.attempt + 1,
|
|
587
|
-
evidence: result.evidence,
|
|
588
|
-
exitCode: result.exitCode,
|
|
589
|
-
nodeId: input.node.id,
|
|
590
|
-
output: result.output,
|
|
591
|
-
status: "passed"
|
|
592
|
-
};
|
|
593
|
-
});
|
|
594
|
-
}
|
|
595
|
-
function canSelfRemediateWritableNode(input) {
|
|
596
|
-
if (input.retry.retryReason !== "gate_failure") return false;
|
|
597
|
-
if (isRemediationNode(input.node)) return false;
|
|
598
|
-
return nodeCanWrite(input.context, input.node);
|
|
599
|
-
}
|
|
600
|
-
function remediationChangedNothing(changedFileCount, result, beforeOutput) {
|
|
601
|
-
if (changedFileCount !== 0) return false;
|
|
602
|
-
return result.output === beforeOutput;
|
|
603
|
-
}
|
|
604
|
-
function executeSelfRemediation(input) {
|
|
605
|
-
return Effect.gen(function* () {
|
|
606
|
-
const node = {
|
|
607
|
-
...input.node,
|
|
608
|
-
artifacts: void 0,
|
|
609
|
-
dependents: [],
|
|
610
|
-
id: `${input.node.id}:remediate:${input.retry.gate}:${input.attempt}`,
|
|
611
|
-
needs: [],
|
|
612
|
-
retries: void 0
|
|
613
|
-
};
|
|
614
|
-
const originalTask = input.context.task;
|
|
615
|
-
input.context.task = nodeRemediationTask({
|
|
616
|
-
node: input.node,
|
|
617
|
-
originalTask,
|
|
618
|
-
retry: input.retry
|
|
619
|
-
});
|
|
620
|
-
return yield* Effect.ensuring(executeNode(node, input.context), Effect.sync(() => {
|
|
621
|
-
input.context.task = originalTask;
|
|
622
|
-
}));
|
|
623
|
-
});
|
|
624
|
-
}
|
|
625
|
-
function remediateCoverageFailure(input) {
|
|
626
|
-
if (input.retry.retryReason !== "gate_failure" || !hasSchedulingRole(input.context, input.node, "coverage")) return Effect.succeed(false);
|
|
627
|
-
return remediatePassedImplementationAncestors(input);
|
|
628
|
-
}
|
|
629
|
-
function remediateUpstreamImplementationFailure(input) {
|
|
630
|
-
if (isRemediationNode(input.node) || nodeCanWrite(input.context, input.node) || hasSchedulingRole(input.context, input.node, "coverage")) return Effect.succeed(false);
|
|
631
|
-
return remediatePassedImplementationAncestors(input);
|
|
632
|
-
}
|
|
633
|
-
function remediatePassedImplementationAncestors(input) {
|
|
634
|
-
return Effect.gen(function* () {
|
|
635
|
-
const implementationNodes = upstreamImplementationNodes(input.context, input.node);
|
|
636
|
-
if (implementationNodes.length === 0) return false;
|
|
637
|
-
let remediated = false;
|
|
638
|
-
for (const implementationNode of implementationNodes) if (yield* remediateImplementationAncestor(input, implementationNode)) remediated = true;
|
|
639
|
-
return remediated;
|
|
640
|
-
});
|
|
641
|
-
}
|
|
642
|
-
function remediateImplementationAncestor(input, implementationNode) {
|
|
643
|
-
return Effect.gen(function* () {
|
|
644
|
-
if (isCancelled(input.context)) return false;
|
|
645
|
-
const beforeSnapshot = yield* snapshotChangedFilesEffect(input.context.worktreePath);
|
|
646
|
-
const beforeOutput = input.context.nodeStateStore.getOutput(implementationNode.id);
|
|
647
|
-
const result = yield* executeImplementationRemediation({
|
|
648
|
-
attempt: input.attempt,
|
|
649
|
-
context: input.context,
|
|
650
|
-
coverageNode: input.node,
|
|
651
|
-
implementationNode,
|
|
652
|
-
retry: input.retry
|
|
653
|
-
});
|
|
654
|
-
if (result.status !== "passed") return false;
|
|
655
|
-
return yield* recordImplementationRemediationEffect({
|
|
656
|
-
beforeOutput,
|
|
657
|
-
beforeSnapshot,
|
|
658
|
-
context: input.context,
|
|
659
|
-
implementationNode,
|
|
660
|
-
result
|
|
661
|
-
});
|
|
662
|
-
});
|
|
663
|
-
}
|
|
664
|
-
function recordImplementationRemediationEffect(input) {
|
|
665
|
-
return Effect.gen(function* () {
|
|
666
|
-
if (diffChangedFiles(input.beforeSnapshot, yield* snapshotChangedFilesEffect(input.context.worktreePath), input.context.worktreePath).files.size === 0 && input.result.output === input.beforeOutput) return false;
|
|
667
|
-
input.context.nodeStateStore.recordOutput(input.implementationNode.id, input.result.output);
|
|
668
|
-
return true;
|
|
669
|
-
});
|
|
670
|
-
}
|
|
671
|
-
function executeImplementationRemediation(input) {
|
|
672
|
-
return Effect.gen(function* () {
|
|
673
|
-
const node = {
|
|
674
|
-
...input.implementationNode,
|
|
675
|
-
artifacts: void 0,
|
|
676
|
-
dependents: [],
|
|
677
|
-
gates: void 0,
|
|
678
|
-
id: `${input.implementationNode.id}:remediate:${input.coverageNode.id}:${input.attempt}`,
|
|
679
|
-
needs: [],
|
|
680
|
-
retries: void 0
|
|
681
|
-
};
|
|
682
|
-
const originalTask = input.context.task;
|
|
683
|
-
input.context.task = remediationTask({
|
|
684
|
-
coverageNode: input.coverageNode,
|
|
685
|
-
originalTask,
|
|
686
|
-
retry: input.retry
|
|
687
|
-
});
|
|
688
|
-
return yield* Effect.ensuring(executeNode(node, input.context), Effect.sync(() => {
|
|
689
|
-
input.context.task = originalTask;
|
|
690
|
-
}));
|
|
691
|
-
});
|
|
692
|
-
}
|
|
693
|
-
function remediationTask(input) {
|
|
694
|
-
return [
|
|
695
|
-
"Remediate a pipeline coverage failure.",
|
|
696
|
-
"",
|
|
697
|
-
"Original task:",
|
|
698
|
-
input.originalTask,
|
|
699
|
-
"",
|
|
700
|
-
"Coverage node:",
|
|
701
|
-
input.coverageNode.id,
|
|
702
|
-
"",
|
|
703
|
-
"Failed gate:",
|
|
704
|
-
input.retry.gate,
|
|
705
|
-
"",
|
|
706
|
-
"Failure reason:",
|
|
707
|
-
input.retry.reason,
|
|
708
|
-
"",
|
|
709
|
-
"Coverage failure feedback:",
|
|
710
|
-
...input.retry.evidence.map((item) => `- ${item}`),
|
|
711
|
-
"",
|
|
712
|
-
"Update the implementation so the coverage node can pass on its next run."
|
|
713
|
-
].join("\n");
|
|
714
|
-
}
|
|
715
|
-
function nodeCanWrite(context, node) {
|
|
716
|
-
const profileId = node.profile;
|
|
717
|
-
if (!profileId) return false;
|
|
718
|
-
return profileCanWrite(context.config.profiles[profileId]);
|
|
719
|
-
}
|
|
720
|
-
function profileCanWrite(profile) {
|
|
721
|
-
if (!profile) return false;
|
|
722
|
-
return hasWorkspaceWriteMode(profile) ? true : hasWriteTool(profile.tools ?? []);
|
|
723
|
-
}
|
|
724
|
-
function hasWorkspaceWriteMode(profile) {
|
|
725
|
-
return profile.filesystem?.mode === "workspace-write";
|
|
726
|
-
}
|
|
727
|
-
function hasWriteTool(tools) {
|
|
728
|
-
return tools.some(isWriteTool);
|
|
729
|
-
}
|
|
730
|
-
function isWriteTool(tool) {
|
|
731
|
-
return tool === "edit" ? true : tool === "write";
|
|
732
|
-
}
|
|
733
|
-
function isRemediationNode(node) {
|
|
734
|
-
return node.id.includes(":remediate:");
|
|
735
|
-
}
|
|
736
|
-
function nodeRemediationTask(input) {
|
|
737
|
-
return [
|
|
738
|
-
"Remediate a pipeline node gate failure.",
|
|
739
|
-
"",
|
|
740
|
-
"Original task:",
|
|
741
|
-
input.originalTask,
|
|
742
|
-
"",
|
|
743
|
-
"Node:",
|
|
744
|
-
input.node.id,
|
|
745
|
-
"",
|
|
746
|
-
"Failed gate:",
|
|
747
|
-
input.retry.gate,
|
|
748
|
-
"",
|
|
749
|
-
"Failure reason:",
|
|
750
|
-
input.retry.reason,
|
|
751
|
-
"",
|
|
752
|
-
"Gate failure feedback:",
|
|
753
|
-
...input.retry.evidence.map((item) => `- ${item}`),
|
|
754
|
-
"",
|
|
755
|
-
"Update the node output and files so this gate can pass."
|
|
756
|
-
].join("\n");
|
|
757
|
-
}
|
|
758
|
-
function upstreamImplementationNodes(context, node) {
|
|
759
|
-
const visited = /* @__PURE__ */ new Set();
|
|
760
|
-
const ordered = [];
|
|
761
|
-
const visit = (candidateId) => visitImplementationNode(context, visited, ordered, candidateId, visit);
|
|
762
|
-
for (const need of node.needs) visit(need);
|
|
763
|
-
return ordered;
|
|
764
|
-
}
|
|
765
|
-
function visitImplementationNode(context, visited, ordered, nodeId, visit) {
|
|
766
|
-
if (visited.has(nodeId)) return;
|
|
767
|
-
visited.add(nodeId);
|
|
768
|
-
const candidate = context.plan.graph.node(nodeId);
|
|
769
|
-
if (!candidate) return;
|
|
770
|
-
visitImplementationDependencies(candidate, visit);
|
|
771
|
-
appendImplementationNode(context, ordered, candidate);
|
|
772
|
-
}
|
|
773
|
-
function visitImplementationDependencies(candidate, visit) {
|
|
774
|
-
for (const need of candidate.needs) visit(need);
|
|
775
|
-
}
|
|
776
|
-
function appendImplementationNode(context, ordered, candidate) {
|
|
777
|
-
if (!nodeStatePassed(context, candidate.id)) return;
|
|
778
|
-
pushIfImplementation(context, ordered, candidate);
|
|
779
|
-
for (const child of candidate.children ?? []) appendPassedImplementationChild(context, ordered, child);
|
|
780
|
-
}
|
|
781
|
-
function appendPassedImplementationChild(context, ordered, child) {
|
|
782
|
-
pushIfImplementation(context, ordered, child);
|
|
783
|
-
for (const grandchild of child.children ?? []) appendPassedImplementationChild(context, ordered, grandchild);
|
|
784
|
-
}
|
|
785
|
-
function pushIfImplementation(context, ordered, node) {
|
|
786
|
-
if (hasSchedulingRole(context, node, "implementation")) ordered.push(node);
|
|
787
|
-
}
|
|
788
|
-
function nodeStatePassed(context, nodeId) {
|
|
789
|
-
return context.nodeStateStore.getNodeState(nodeId)?.status === "passed";
|
|
790
|
-
}
|
|
791
|
-
function hasSchedulingRole(context, node, role) {
|
|
792
|
-
return node.profile ? context.config.profiles[node.profile]?.scheduling_roles?.includes(role) ?? false : false;
|
|
793
|
-
}
|
|
794
594
|
function waitForRetryDelay(delayMs, signal) {
|
|
795
595
|
if (delayMs <= 0 || signal?.aborted) return Effect.void;
|
|
796
596
|
return Effect.race(Effect.sleep(delayMs), waitForAbort(signal));
|
|
797
597
|
}
|
|
798
598
|
function waitForAbort(signal) {
|
|
799
599
|
if (!signal) return Effect.never;
|
|
800
|
-
return Effect.
|
|
600
|
+
return Effect.callback((resume) => {
|
|
801
601
|
const onAbort = () => resume(Effect.void);
|
|
802
602
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
803
603
|
return Effect.sync(() => signal.removeEventListener("abort", onAbort));
|