@oisincoveney/pipeline 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/defaults/pipeline.yaml +25 -0
- package/dist/argo-graph.js +7 -7
- package/dist/bench/eval-report.js +27 -0
- package/dist/cli/program.js +19 -3
- package/dist/commands/bench-command.js +18 -0
- package/dist/config/load.js +17 -0
- package/dist/config/schemas.d.ts +22 -5
- package/dist/config/schemas.js +20 -7
- package/dist/context/repo-map.js +203 -0
- package/dist/install-commands/opencode.js +10 -1
- package/dist/mcp/gateway.js +3 -3
- package/dist/moka-submit.d.ts +6 -6
- package/dist/pipeline-init.js +18 -12
- package/dist/pipeline-runtime.js +12 -1
- package/dist/planning/compile.d.ts +8 -3
- package/dist/planning/compile.js +7 -7
- package/dist/planning/generate.d.ts +6 -1
- package/dist/planning/generate.js +29 -7
- package/dist/runner-command-contract.d.ts +6 -1
- package/dist/runner-command-contract.js +6 -5
- package/dist/runner-event-schema.d.ts +6 -6
- package/dist/runner-event-sink.js +6 -5
- package/dist/runner.d.ts +6 -1
- package/dist/runner.js +3 -3
- package/dist/runtime/agent-node/agent-node.js +22 -4
- package/dist/runtime/local-scheduler.js +45 -0
- package/dist/runtime/opencode-server.js +6 -3
- package/dist/runtime/parallel-node/parallel-node.js +77 -58
- package/dist/runtime/parallel-worktrees/parallel-worktrees.js +49 -4
- package/dist/runtime/run-journal.js +21 -0
- package/dist/runtime/scheduler.js +122 -93
- package/dist/runtime/select-candidate/select-candidate.js +13 -1
- package/dist/runtime/services/worktree-service.js +18 -0
- package/dist/schedule/passes/candidates.js +17 -8
- package/docs/config-architecture.md +105 -0
- package/package.json +7 -2
|
@@ -1,60 +1,97 @@
|
|
|
1
1
|
import { childReporter } from "../events/events.js";
|
|
2
2
|
import "../events/index.js";
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
3
|
+
import { WorktreeService, WorktreeServiceLive } from "../services/worktree-service.js";
|
|
4
|
+
import { Effect } from "effect";
|
|
5
5
|
//#region src/runtime/parallel-node/parallel-node.ts
|
|
6
|
-
|
|
6
|
+
function executeParallelNode(node, context, runtime) {
|
|
7
|
+
return Effect.runPromise(Effect.provide(parallelNodeProgram(node, context, runtime), WorktreeServiceLive));
|
|
8
|
+
}
|
|
9
|
+
function parallelNodeProgram(node, context, runtime) {
|
|
7
10
|
const children = node.children ?? [];
|
|
8
|
-
if (children.length === 0) return {
|
|
11
|
+
if (children.length === 0) return Effect.succeed({
|
|
9
12
|
evidence: [`parallel node '${node.id}' has no children`],
|
|
10
13
|
exitCode: 1,
|
|
11
14
|
output: ""
|
|
15
|
+
});
|
|
16
|
+
return Effect.gen(function* () {
|
|
17
|
+
yield* gcStaleWorktrees(context);
|
|
18
|
+
const failFast = context.plan.execution.failFast;
|
|
19
|
+
const linkedAbort = createLinkedAbortController(context.signal);
|
|
20
|
+
const childContext = createParallelChildContext(context, node.id, children, failFast ? linkedAbort.controller.signal : context.signal);
|
|
21
|
+
const gate = failFast ? makeFailFastGate(linkedAbort.controller) : void 0;
|
|
22
|
+
for (const child of children) runtime.markNodeReady(childContext, child.id);
|
|
23
|
+
const settled = yield* runAllChildren(children, childContext, runtime, yield* makeCategorySemaphores(childContext), gate).pipe(Effect.ensuring(Effect.sync(linkedAbort.cleanup)));
|
|
24
|
+
return aggregateParallelResult(node.id, children, settled);
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
function makeFailFastGate(controller) {
|
|
28
|
+
return {
|
|
29
|
+
abort: () => controller.abort(),
|
|
30
|
+
aborted: () => controller.signal.aborted
|
|
12
31
|
};
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
32
|
+
}
|
|
33
|
+
function aggregateParallelResult(nodeId, children, settled) {
|
|
34
|
+
const results = settled.filter((result) => result !== void 0);
|
|
35
|
+
const failed = results.filter((result) => result.status === "failed");
|
|
36
|
+
return {
|
|
37
|
+
evidence: parallelEvidence(nodeId, results, failed),
|
|
38
|
+
exitCode: failed.length > 0 ? 1 : 0,
|
|
39
|
+
output: parallelOutput(children, results)
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
function runAllChildren(children, context, runtime, caps, gate) {
|
|
43
|
+
return Effect.forEach(children, (child) => runChildCapped(child, context, runtime, caps, gate), { concurrency: context.maxParallelNodes ?? "unbounded" });
|
|
44
|
+
}
|
|
45
|
+
function runChildCapped(child, context, runtime, caps, gate) {
|
|
46
|
+
return Effect.gen(function* () {
|
|
47
|
+
if (gate?.aborted()) return;
|
|
48
|
+
const result = yield* withCategoryCap(caps, child.id, context, runChildInWorktree(child, context, runtime));
|
|
49
|
+
if (gate && result.status === "failed") gate.abort();
|
|
50
|
+
return result;
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
function withCategoryCap(caps, childId, context, effect) {
|
|
54
|
+
const category = childCategory(childId, context.config.token_budget?.fan_out_width);
|
|
55
|
+
const semaphore = category ? caps.get(category) : void 0;
|
|
56
|
+
return semaphore ? semaphore.withPermits(1)(effect) : effect;
|
|
57
|
+
}
|
|
58
|
+
function makeCategorySemaphores(context) {
|
|
59
|
+
const fanOut = context.config.token_budget?.fan_out_width;
|
|
60
|
+
if (!fanOut) return Effect.succeed(/* @__PURE__ */ new Map());
|
|
61
|
+
return Effect.gen(function* () {
|
|
62
|
+
const caps = /* @__PURE__ */ new Map();
|
|
63
|
+
for (const [category, permits] of Object.entries(fanOut.by_category)) caps.set(category, yield* Effect.makeSemaphore(permits));
|
|
64
|
+
return caps;
|
|
65
|
+
});
|
|
27
66
|
}
|
|
28
67
|
function gcStaleWorktrees(context) {
|
|
29
|
-
|
|
68
|
+
return Effect.gen(function* () {
|
|
69
|
+
if (context.config.parallel_worktrees?.enabled) yield* (yield* WorktreeService).gc(context.worktreePath);
|
|
70
|
+
});
|
|
30
71
|
}
|
|
31
72
|
/**
|
|
32
73
|
* PIPE-83.4: run a parallel child in its own git worktree when enabled, so
|
|
33
|
-
* concurrent candidate edits can't collide. The lease is
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
* selection. Default-off path is byte-identical to the prior behaviour.
|
|
74
|
+
* concurrent candidate edits can't collide. The worktree lease is acquired and
|
|
75
|
+
* released as an Effect-scoped resource (released on success, failure, or
|
|
76
|
+
* interruption); release retains dirty/unpushed work for downstream selection.
|
|
37
77
|
*/
|
|
38
78
|
function runChildInWorktree(child, context, runtime) {
|
|
39
|
-
|
|
79
|
+
if (!context.config.parallel_worktrees?.enabled) return Effect.promise(() => runtime.executeNode(child, context));
|
|
80
|
+
return Effect.gen(function* () {
|
|
81
|
+
const worktree = yield* WorktreeService;
|
|
82
|
+
return yield* Effect.acquireUseRelease(worktree.createChild(childLeaseOptions(child, context)), (lease) => Effect.promise(() => runtime.executeNode(child, {
|
|
83
|
+
...context,
|
|
84
|
+
worktreePath: lease.path
|
|
85
|
+
})), (lease) => Effect.sync(() => lease.release()));
|
|
86
|
+
});
|
|
40
87
|
}
|
|
41
|
-
function
|
|
42
|
-
return
|
|
88
|
+
function childLeaseOptions(child, context) {
|
|
89
|
+
return {
|
|
43
90
|
childNodeId: child.id,
|
|
44
91
|
parentNodeId: context.parentParallelNodeId ?? "parallel",
|
|
45
92
|
repoRoot: context.worktreePath,
|
|
46
93
|
...context.runId ? { runId: context.runId } : {}
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
async function runInLease(child, context, runtime, lease) {
|
|
50
|
-
try {
|
|
51
|
-
return await runtime.executeNode(child, {
|
|
52
|
-
...context,
|
|
53
|
-
worktreePath: lease.path
|
|
54
|
-
});
|
|
55
|
-
} finally {
|
|
56
|
-
lease.release();
|
|
57
|
-
}
|
|
94
|
+
};
|
|
58
95
|
}
|
|
59
96
|
function createParallelChildContext(context, parentNodeId, children, signal) {
|
|
60
97
|
return {
|
|
@@ -91,26 +128,8 @@ function createLinkedAbortController(signal) {
|
|
|
91
128
|
controller
|
|
92
129
|
};
|
|
93
130
|
}
|
|
94
|
-
function
|
|
95
|
-
|
|
96
|
-
if (!context.maxParallelNodes) return Promise.all(children.map((child) => runChildInWorktree(child, context, runtime)));
|
|
97
|
-
const limit = pLimit(context.maxParallelNodes);
|
|
98
|
-
return Promise.all(children.map((child) => limit(() => runChildInWorktree(child, context, runtime))));
|
|
99
|
-
}
|
|
100
|
-
async function executeFailFastParallelChildren(children, context, abortController, runtime) {
|
|
101
|
-
for (const child of children) runtime.markNodeReady(context, child.id);
|
|
102
|
-
const limit = pLimit({
|
|
103
|
-
concurrency: context.maxParallelNodes ?? children.length,
|
|
104
|
-
rejectOnClear: true
|
|
105
|
-
});
|
|
106
|
-
return (await Promise.allSettled(children.map((child) => limit(async () => {
|
|
107
|
-
const result = await runChildInWorktree(child, context, runtime);
|
|
108
|
-
if (result.status === "failed") {
|
|
109
|
-
abortController.abort();
|
|
110
|
-
limit.clearQueue();
|
|
111
|
-
}
|
|
112
|
-
return result;
|
|
113
|
-
})))).flatMap((result) => result.status === "fulfilled" ? [result.value] : []);
|
|
131
|
+
function childCategory(childId, fanOut) {
|
|
132
|
+
return fanOut ? Object.keys(fanOut.by_category).find((category) => childId.includes(category)) : void 0;
|
|
114
133
|
}
|
|
115
134
|
function parallelEvidence(nodeId, results, failed) {
|
|
116
135
|
if (failed.length === 0) return [`parallel node '${nodeId}' completed ${results.length} child nodes`];
|
|
@@ -121,4 +140,4 @@ function parallelOutput(children, results) {
|
|
|
121
140
|
return JSON.stringify({ children: Object.fromEntries(children.filter((child) => outputsByNode.has(child.id)).map((child) => [child.id, outputsByNode.get(child.id)])) });
|
|
122
141
|
}
|
|
123
142
|
//#endregion
|
|
124
|
-
export { executeParallelNode, parallelEvidence, parallelOutput };
|
|
143
|
+
export { childCategory, executeParallelNode, parallelEvidence, parallelOutput };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { join } from "node:path";
|
|
1
|
+
import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
3
|
import { execFileSync } from "node:child_process";
|
|
4
4
|
//#region src/runtime/parallel-worktrees/parallel-worktrees.ts
|
|
5
5
|
/**
|
|
@@ -13,6 +13,50 @@ import { execFileSync } from "node:child_process";
|
|
|
13
13
|
const WORKTREE_ROOT = ".pipeline/worktrees";
|
|
14
14
|
const REGISTRY_DIR = join(WORKTREE_ROOT, "registry");
|
|
15
15
|
const OWNER = "oisin-pipeline";
|
|
16
|
+
const GENERATED_WORKTREE_RESOURCES = [join(".opencode", "agents"), join(".opencode", "command")];
|
|
17
|
+
function provisionGeneratedResources(repoRoot, worktreePath) {
|
|
18
|
+
for (const relativePath of GENERATED_WORKTREE_RESOURCES) {
|
|
19
|
+
const source = join(repoRoot, relativePath);
|
|
20
|
+
const target = join(worktreePath, relativePath);
|
|
21
|
+
if (existsSync(source) && !existsSync(target)) cpSync(source, target, { recursive: true });
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function childWorktreeRelPath(runId, parentNodeId, childNodeId) {
|
|
25
|
+
return join(WORKTREE_ROOT, "trees", sanitize(runId ?? "local"), sanitize(parentNodeId), sanitize(childNodeId));
|
|
26
|
+
}
|
|
27
|
+
function changedWorktreeFiles(worktreePath) {
|
|
28
|
+
const modified = git(worktreePath, [
|
|
29
|
+
"diff",
|
|
30
|
+
"--name-only",
|
|
31
|
+
"HEAD"
|
|
32
|
+
]);
|
|
33
|
+
const untracked = git(worktreePath, [
|
|
34
|
+
"ls-files",
|
|
35
|
+
"--others",
|
|
36
|
+
"--exclude-standard"
|
|
37
|
+
]);
|
|
38
|
+
return [...modified.split("\n"), ...untracked.split("\n")].filter(Boolean);
|
|
39
|
+
}
|
|
40
|
+
function copyFileInto(fromRoot, toRoot, relativePath) {
|
|
41
|
+
const source = join(fromRoot, relativePath);
|
|
42
|
+
if (!existsSync(source)) return;
|
|
43
|
+
const dest = join(toRoot, relativePath);
|
|
44
|
+
mkdirSync(dirname(dest), { recursive: true });
|
|
45
|
+
cpSync(source, dest, { recursive: true });
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* PIPE-83.14: promote a best-of-N winner's edits from its (retained) worktree
|
|
49
|
+
* back into the main worktree, so downstream nodes (tests, verification) see the
|
|
50
|
+
* selected candidate's changes. Copies modified + untracked files; no-op if the
|
|
51
|
+
* worktree is gone. Returns the promoted file paths.
|
|
52
|
+
*/
|
|
53
|
+
function promoteWorktreeChanges(repoRoot, runId, parentNodeId, childNodeId) {
|
|
54
|
+
const worktreePath = join(repoRoot, childWorktreeRelPath(runId, parentNodeId, childNodeId));
|
|
55
|
+
if (!existsSync(worktreePath)) return [];
|
|
56
|
+
const files = changedWorktreeFiles(worktreePath);
|
|
57
|
+
for (const relativePath of files) copyFileInto(worktreePath, repoRoot, relativePath);
|
|
58
|
+
return files;
|
|
59
|
+
}
|
|
16
60
|
function git(cwd, args) {
|
|
17
61
|
return execFileSync("git", args, {
|
|
18
62
|
cwd,
|
|
@@ -33,7 +77,7 @@ function createChildWorktree(opts) {
|
|
|
33
77
|
const parentSeg = sanitize(opts.parentNodeId);
|
|
34
78
|
const childSeg = sanitize(opts.childNodeId);
|
|
35
79
|
const baseSha = git(opts.repoRoot, ["rev-parse", "HEAD"]);
|
|
36
|
-
const relPath =
|
|
80
|
+
const relPath = childWorktreeRelPath(opts.runId, opts.parentNodeId, opts.childNodeId);
|
|
37
81
|
const absPath = join(opts.repoRoot, relPath);
|
|
38
82
|
const branch = `pipeline/worktrees/${runSeg}/${parentSeg}/${childSeg}`;
|
|
39
83
|
const leaseId = `${runSeg}__${parentSeg}__${childSeg}`;
|
|
@@ -61,6 +105,7 @@ function createChildWorktree(opts) {
|
|
|
61
105
|
absPath,
|
|
62
106
|
baseSha
|
|
63
107
|
]);
|
|
108
|
+
provisionGeneratedResources(opts.repoRoot, absPath);
|
|
64
109
|
writeManifest(manifestPath, {
|
|
65
110
|
...manifest,
|
|
66
111
|
state: "active"
|
|
@@ -129,4 +174,4 @@ function gcParallelWorktrees(repoRoot) {
|
|
|
129
174
|
return results;
|
|
130
175
|
}
|
|
131
176
|
//#endregion
|
|
132
|
-
export { createChildWorktree, gcParallelWorktrees };
|
|
177
|
+
export { createChildWorktree, gcParallelWorktrees, promoteWorktreeChanges };
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
//#region src/runtime/run-journal.ts
|
|
4
|
+
function passedOnly(results) {
|
|
5
|
+
return results.filter((result) => result.status === "passed");
|
|
6
|
+
}
|
|
7
|
+
function readJournalFile(path) {
|
|
8
|
+
if (!existsSync(path)) return [];
|
|
9
|
+
return readFileSync(path, "utf8").split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
|
|
10
|
+
}
|
|
11
|
+
function fileRunJournal(path) {
|
|
12
|
+
return {
|
|
13
|
+
record: (result) => {
|
|
14
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
15
|
+
appendFileSync(path, `${JSON.stringify(result)}\n`);
|
|
16
|
+
},
|
|
17
|
+
resumeCompleted: () => passedOnly(readJournalFile(path))
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
//#endregion
|
|
21
|
+
export { fileRunJournal };
|
|
@@ -1,44 +1,16 @@
|
|
|
1
1
|
import { uniqueStrings } from "../strings.js";
|
|
2
|
-
import {
|
|
2
|
+
import { Effect, Fiber, Queue } from "effect";
|
|
3
3
|
//#region src/runtime/scheduler.ts
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
return (await runWorkflowLifecycle({
|
|
13
|
-
buildResult: options.buildResult,
|
|
14
|
-
emitWorkflowPlanned: () => options.emitWorkflowPlanned(context),
|
|
15
|
-
emitWorkflowStarted: () => options.emitWorkflowStarted(context),
|
|
16
|
-
executeWorkflow: () => runWorkflowScheduler({
|
|
17
|
-
failFast: plan.execution.failFast,
|
|
18
|
-
fanOutWidth: context.config.token_budget?.fan_out_width,
|
|
19
|
-
isCancelled: () => options.isCancelled(context),
|
|
20
|
-
markNodeReady: (nodeId) => options.markNodeReady(nodeId, context),
|
|
21
|
-
maxParallelNodes: context.maxParallelNodes,
|
|
22
|
-
nodes: plan.topologicalOrder.map((node) => ({
|
|
23
|
-
category: node.category,
|
|
24
|
-
dependents: node.dependents,
|
|
25
|
-
id: node.id,
|
|
26
|
-
index: node.index,
|
|
27
|
-
needs: node.needs
|
|
28
|
-
})),
|
|
29
|
-
runNode: (nodeId) => options.executeNode(nodeId, context),
|
|
30
|
-
shouldContinueAfterNodeResult: (result) => options.shouldContinueAfterNodeResult(result, context),
|
|
31
|
-
skipNode: (nodeId, reason) => options.skipNode(nodeId, reason, context)
|
|
32
|
-
}),
|
|
33
|
-
isCancelled: () => options.isCancelled(context),
|
|
34
|
-
runWorkflowHook: (event, failure) => options.runWorkflowHook(event, failure, context)
|
|
35
|
-
})).result;
|
|
36
|
-
}
|
|
37
|
-
};
|
|
38
|
-
async function runWorkflowScheduler(input) {
|
|
39
|
-
const state = {
|
|
4
|
+
function resumeFromJournal(input) {
|
|
5
|
+
return input.journal?.resumeCompleted() ?? [];
|
|
6
|
+
}
|
|
7
|
+
function recordToJournal(input, result) {
|
|
8
|
+
input.journal?.record(result);
|
|
9
|
+
}
|
|
10
|
+
function initialSchedulerState(input) {
|
|
11
|
+
return {
|
|
40
12
|
blocked: [],
|
|
41
|
-
completed:
|
|
13
|
+
completed: resumeFromJournal(input),
|
|
42
14
|
failFast: input.failFast,
|
|
43
15
|
fanOutWidth: input.fanOutWidth,
|
|
44
16
|
maxParallelNodes: input.maxParallelNodes,
|
|
@@ -46,50 +18,120 @@ async function runWorkflowScheduler(input) {
|
|
|
46
18
|
running: [],
|
|
47
19
|
shouldContinueAfterNodeResult: input.shouldContinueAfterNodeResult
|
|
48
20
|
};
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
21
|
+
}
|
|
22
|
+
function cancelledResult(state) {
|
|
23
|
+
return {
|
|
24
|
+
completed: state.completed,
|
|
25
|
+
outcome: "CANCELLED"
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function terminalResult(state, failure) {
|
|
29
|
+
return {
|
|
30
|
+
completed: state.completed,
|
|
31
|
+
failure,
|
|
32
|
+
outcome: failure ? "FAIL" : "PASS"
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
function nodeErrorResult(state, error) {
|
|
36
|
+
return {
|
|
37
|
+
completed: state.completed,
|
|
38
|
+
failure: workflowServiceFailure(error, "workflow.node"),
|
|
39
|
+
outcome: "FAIL"
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
function runNodeFiber(ctx, nodeId) {
|
|
43
|
+
return Effect.tryPromise({
|
|
44
|
+
catch: (error) => error,
|
|
45
|
+
try: () => ctx.input.runNode(nodeId)
|
|
46
|
+
}).pipe(Effect.matchEffect({
|
|
47
|
+
onFailure: (error) => Queue.offer(ctx.completions, {
|
|
48
|
+
error,
|
|
49
|
+
kind: "error"
|
|
50
|
+
}),
|
|
51
|
+
onSuccess: (result) => Queue.offer(ctx.completions, {
|
|
52
|
+
kind: "ok",
|
|
53
|
+
result
|
|
54
|
+
})
|
|
55
|
+
}));
|
|
56
|
+
}
|
|
57
|
+
function launchReady(ctx) {
|
|
58
|
+
return Effect.gen(function* () {
|
|
59
|
+
const capacity = workflowNodeCapacity(ctx.state);
|
|
60
|
+
if (capacity <= 0) return;
|
|
61
|
+
for (const nodeId of selectLaunchableNodes(ctx.state, capacity)) {
|
|
62
|
+
ctx.input.markNodeReady(nodeId);
|
|
63
|
+
ctx.state.running = [...ctx.state.running, nodeId];
|
|
64
|
+
const fiber = yield* Effect.fork(runNodeFiber(ctx, nodeId));
|
|
65
|
+
ctx.running.set(nodeId, fiber);
|
|
83
66
|
}
|
|
84
|
-
|
|
85
|
-
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
function applyFailFastSkip(ctx, result) {
|
|
70
|
+
const reason = `skipped because workflow fail_fast stopped after node '${result.nodeId}' failed`;
|
|
71
|
+
const skipped = unstartedNodeIds(ctx.state);
|
|
72
|
+
ctx.state.blocked = uniqueStrings([...ctx.state.blocked, ...skipped]);
|
|
73
|
+
for (const nodeId of skipped) ctx.input.skipNode(nodeId, reason);
|
|
74
|
+
}
|
|
75
|
+
function applyCompletion(ctx, result) {
|
|
76
|
+
ctx.running.delete(result.nodeId);
|
|
77
|
+
ctx.state.running = ctx.state.running.filter((id) => id !== result.nodeId);
|
|
78
|
+
ctx.state.completed = [...ctx.state.completed, result];
|
|
79
|
+
recordToJournal(ctx.input, result);
|
|
80
|
+
if (!isBlockingFailure(result, ctx.state)) return;
|
|
81
|
+
ctx.failure ??= nodeRuntimeFailure(result);
|
|
82
|
+
if (ctx.input.failFast) {
|
|
83
|
+
applyFailFastSkip(ctx, result);
|
|
84
|
+
return;
|
|
86
85
|
}
|
|
86
|
+
const blocked = unstartedBlockingDescendants(result.nodeId, ctx.state);
|
|
87
|
+
ctx.state.blocked = uniqueStrings([...ctx.state.blocked, ...blocked]);
|
|
88
|
+
}
|
|
89
|
+
function applyOutcome(ctx, outcome) {
|
|
90
|
+
return Effect.gen(function* () {
|
|
91
|
+
if (outcome.kind === "error") {
|
|
92
|
+
yield* Fiber.interruptAll(ctx.running.values());
|
|
93
|
+
return nodeErrorResult(ctx.state, outcome.error);
|
|
94
|
+
}
|
|
95
|
+
applyCompletion(ctx, outcome.result);
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
function schedulerTick(ctx) {
|
|
99
|
+
return Effect.gen(function* () {
|
|
100
|
+
if (ctx.input.isCancelled()) {
|
|
101
|
+
yield* Fiber.interruptAll(ctx.running.values());
|
|
102
|
+
return cancelledResult(ctx.state);
|
|
103
|
+
}
|
|
104
|
+
yield* launchReady(ctx);
|
|
105
|
+
if (ctx.running.size === 0) return terminalResult(ctx.state, ctx.failure);
|
|
106
|
+
return yield* applyOutcome(ctx, yield* Queue.take(ctx.completions));
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
function schedulerProgram(input) {
|
|
110
|
+
return Effect.gen(function* () {
|
|
111
|
+
const ctx = {
|
|
112
|
+
completions: yield* Queue.unbounded(),
|
|
113
|
+
input,
|
|
114
|
+
running: /* @__PURE__ */ new Map(),
|
|
115
|
+
state: initialSchedulerState(input)
|
|
116
|
+
};
|
|
117
|
+
while (true) {
|
|
118
|
+
const done = yield* schedulerTick(ctx);
|
|
119
|
+
if (done) return done;
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
function runWorkflowScheduler(input) {
|
|
124
|
+
return Effect.runPromise(schedulerProgram(input));
|
|
125
|
+
}
|
|
126
|
+
function settledNodeIds(context) {
|
|
127
|
+
const ids = new Set((context.completed ?? []).map((result) => result.nodeId));
|
|
128
|
+
for (const nodeId of context.running) ids.add(nodeId);
|
|
129
|
+
return ids;
|
|
87
130
|
}
|
|
88
131
|
function readyNodeIds(context) {
|
|
132
|
+
const settled = settledNodeIds(context);
|
|
89
133
|
const blocked = new Set(context.blocked ?? []);
|
|
90
|
-
|
|
91
|
-
const running = new Set(context.running);
|
|
92
|
-
return orderedNodes(context.nodes).filter((node) => !completed.has(node.id)).filter((node) => !running.has(node.id)).filter((node) => !blocked.has(node.id)).filter((node) => node.needs.every((need) => dependencyPassed(need, context))).map((node) => node.id);
|
|
134
|
+
return orderedNodes(context.nodes).filter((node) => !(settled.has(node.id) || blocked.has(node.id))).filter((node) => node.needs.every((need) => dependencyPassed(need, context))).map((node) => node.id);
|
|
93
135
|
}
|
|
94
136
|
function workflowNodeCapacity(context) {
|
|
95
137
|
const limit = context.failFast ? 1 : context.maxParallelNodes ?? context.nodes.length;
|
|
@@ -107,18 +149,6 @@ function unstartedBlockingDescendants(nodeId, context) {
|
|
|
107
149
|
}
|
|
108
150
|
return orderedNodes(context.nodes).map((node) => node.id).filter((descendantId) => descendants.has(descendantId)).filter((descendantId) => unstarted.has(descendantId));
|
|
109
151
|
}
|
|
110
|
-
function launchReadyNodes(input, state, running) {
|
|
111
|
-
const capacity = workflowNodeCapacity(state);
|
|
112
|
-
if (capacity <= 0) return;
|
|
113
|
-
for (const nodeId of selectLaunchableNodes(state, capacity)) {
|
|
114
|
-
input.markNodeReady(nodeId);
|
|
115
|
-
state.running = [...state.running, nodeId];
|
|
116
|
-
running.set(nodeId, {
|
|
117
|
-
nodeId,
|
|
118
|
-
promise: input.runNode(nodeId)
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
152
|
/**
|
|
123
153
|
* Choose which ready nodes to launch this tick within the global capacity and
|
|
124
154
|
* the per-category fan-out caps. A category at its cap defers its remaining
|
|
@@ -171,9 +201,8 @@ function isBlockingFailure(result, context) {
|
|
|
171
201
|
return result.status === "failed" && !(context.shouldContinueAfterNodeResult?.(result) ?? false);
|
|
172
202
|
}
|
|
173
203
|
function unstartedNodeIds(context) {
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
return orderedNodes(context.nodes).map((node) => node.id).filter((nodeId) => !completed.has(nodeId)).filter((nodeId) => !running.has(nodeId));
|
|
204
|
+
const settled = settledNodeIds(context);
|
|
205
|
+
return orderedNodes(context.nodes).map((node) => node.id).filter((nodeId) => !settled.has(nodeId));
|
|
177
206
|
}
|
|
178
207
|
function directDependents(nodeId, nodes) {
|
|
179
208
|
const declared = nodes.find((node) => node.id === nodeId)?.dependents ?? [];
|
|
@@ -201,4 +230,4 @@ function workflowServiceFailure(error, gate) {
|
|
|
201
230
|
};
|
|
202
231
|
}
|
|
203
232
|
//#endregion
|
|
204
|
-
export {
|
|
233
|
+
export { runWorkflowScheduler };
|
|
@@ -2,6 +2,7 @@ import { createRunnerLaunchPlan } from "../../runner.js";
|
|
|
2
2
|
import { normalizeRunnerOutput } from "../../runner-output.js";
|
|
3
3
|
import { parseJsonObject } from "../json-validation/json-validation.js";
|
|
4
4
|
import "../json-validation/index.js";
|
|
5
|
+
import { promoteWorktreeChanges } from "../parallel-worktrees/parallel-worktrees.js";
|
|
5
6
|
//#region src/runtime/select-candidate/select-candidate.ts
|
|
6
7
|
const SCORE_RE = /-?\d+(?:\.\d+)?/;
|
|
7
8
|
function selectBestCandidate(candidates) {
|
|
@@ -17,12 +18,23 @@ async function executeSelectCandidateBuiltin(context, node) {
|
|
|
17
18
|
exitCode: 1,
|
|
18
19
|
output: ""
|
|
19
20
|
};
|
|
21
|
+
const promoted = promoteWinner(context, node, selected.nodeId);
|
|
20
22
|
return {
|
|
21
|
-
evidence:
|
|
23
|
+
evidence: selectionEvidence(selected, candidates.length, promoted),
|
|
22
24
|
exitCode: 0,
|
|
23
25
|
output: selected.output
|
|
24
26
|
};
|
|
25
27
|
}
|
|
28
|
+
function selectionEvidence(selected, candidateCount, promoted) {
|
|
29
|
+
const lines = [`select-candidate: selected '${selected.nodeId}' (judge=${selected.judgeScore ?? "n/a"}) from ${candidateCount} candidates`];
|
|
30
|
+
if (promoted.length > 0) lines.push(`promoted ${promoted.length} file(s) from the winning worktree`);
|
|
31
|
+
return lines;
|
|
32
|
+
}
|
|
33
|
+
function promoteWinner(context, node, winnerNodeId) {
|
|
34
|
+
const parentNodeId = node?.needs.at(0);
|
|
35
|
+
if (!(context.config.parallel_worktrees?.enabled && parentNodeId)) return [];
|
|
36
|
+
return promoteWorktreeChanges(context.worktreePath, context.runId, parentNodeId, winnerNodeId);
|
|
37
|
+
}
|
|
26
38
|
async function scoreCandidates(context, candidates) {
|
|
27
39
|
const model = context.config.best_of_n?.judge_model;
|
|
28
40
|
const runner = Object.keys(context.config.runners).at(0);
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { createChildWorktree, gcParallelWorktrees } from "../parallel-worktrees/parallel-worktrees.js";
|
|
2
|
+
import { Context, Effect, Layer } from "effect";
|
|
3
|
+
//#region src/runtime/services/worktree-service.ts
|
|
4
|
+
/**
|
|
5
|
+
* Effect service over the git-worktree lifecycle (PIPE-83 follow-up). The Live
|
|
6
|
+
* layer delegates to the synchronous porcelain helpers in parallel-worktrees;
|
|
7
|
+
* the Effect-native parallel-node runtime composes `createChild`/`gc` through
|
|
8
|
+
* this injectable seam instead of calling the helpers directly. This is the
|
|
9
|
+
* canonical service shape for the Effect conversion: a `Context.Tag` whose Live
|
|
10
|
+
* `Layer` wraps the underlying IO, provided once at the runPromise boundary.
|
|
11
|
+
*/
|
|
12
|
+
var WorktreeService = class extends Context.Tag("WorktreeService")() {};
|
|
13
|
+
const WorktreeServiceLive = Layer.succeed(WorktreeService, {
|
|
14
|
+
createChild: (opts) => Effect.sync(() => createChildWorktree(opts)),
|
|
15
|
+
gc: (repoRoot) => Effect.sync(() => gcParallelWorktrees(repoRoot))
|
|
16
|
+
});
|
|
17
|
+
//#endregion
|
|
18
|
+
export { WorktreeService, WorktreeServiceLive };
|
|
@@ -19,24 +19,33 @@ function expandBestOfNCandidates(config, artifact) {
|
|
|
19
19
|
}]))
|
|
20
20
|
};
|
|
21
21
|
}
|
|
22
|
+
function candidateChild(node, index) {
|
|
23
|
+
const child = {
|
|
24
|
+
...node,
|
|
25
|
+
id: `${node.id}--c${index + 1}`,
|
|
26
|
+
needs: []
|
|
27
|
+
};
|
|
28
|
+
child.task_context = void 0;
|
|
29
|
+
return child;
|
|
30
|
+
}
|
|
22
31
|
function expandNode(node, categories, n) {
|
|
23
32
|
if (node.kind !== "agent" || !categories.some((category) => node.id.includes(category))) return [node];
|
|
24
33
|
const candidatesId = `${node.id}--candidates`;
|
|
25
34
|
return [{
|
|
26
35
|
id: candidatesId,
|
|
27
36
|
kind: "parallel",
|
|
28
|
-
nodes: Array.from({ length: n }, (_, index) => (
|
|
29
|
-
...node,
|
|
30
|
-
id: `${node.id}--c${index + 1}`,
|
|
31
|
-
needs: []
|
|
32
|
-
})),
|
|
37
|
+
nodes: Array.from({ length: n }, (_, index) => candidateChild(node, index)),
|
|
33
38
|
...node.needs ? { needs: node.needs } : {}
|
|
34
|
-
},
|
|
39
|
+
}, selectCandidateNode(node, candidatesId)];
|
|
40
|
+
}
|
|
41
|
+
function selectCandidateNode(node, candidatesId) {
|
|
42
|
+
return {
|
|
35
43
|
builtin: "select-candidate",
|
|
36
44
|
id: node.id,
|
|
37
45
|
kind: "builtin",
|
|
38
|
-
needs: [candidatesId]
|
|
39
|
-
|
|
46
|
+
needs: [candidatesId],
|
|
47
|
+
...node.task_context ? { task_context: node.task_context } : {}
|
|
48
|
+
};
|
|
40
49
|
}
|
|
41
50
|
//#endregion
|
|
42
51
|
export { expandBestOfNCandidates };
|