@oisincoveney/pipeline 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/defaults/pipeline.yaml +25 -0
- package/dist/argo-graph.js +7 -7
- package/dist/bench/eval-report.js +27 -0
- package/dist/cli/program.js +19 -3
- package/dist/commands/bench-command.js +18 -0
- package/dist/config/load.js +17 -0
- package/dist/config/schemas.d.ts +21 -4
- package/dist/config/schemas.js +20 -7
- package/dist/context/repo-map.js +203 -0
- package/dist/install-commands/opencode.js +10 -1
- package/dist/mcp/gateway.js +3 -3
- package/dist/moka-submit.d.ts +1 -1
- package/dist/pipeline-init.js +18 -12
- package/dist/pipeline-runtime.js +12 -1
- package/dist/planning/compile.d.ts +8 -3
- package/dist/planning/compile.js +7 -7
- package/dist/planning/generate.d.ts +6 -1
- package/dist/planning/generate.js +29 -7
- package/dist/runner-command-contract.d.ts +8 -3
- package/dist/runner-command-contract.js +6 -5
- package/dist/runner-event-sink.js +6 -5
- package/dist/runner.d.ts +6 -1
- package/dist/runner.js +3 -3
- package/dist/runtime/agent-node/agent-node.js +22 -4
- package/dist/runtime/local-scheduler.js +45 -0
- package/dist/runtime/opencode-server.js +6 -3
- package/dist/runtime/parallel-node/parallel-node.js +74 -75
- package/dist/runtime/parallel-worktrees/parallel-worktrees.js +49 -4
- package/dist/runtime/run-journal.js +21 -0
- package/dist/runtime/scheduler.js +122 -93
- package/dist/runtime/select-candidate/select-candidate.js +13 -1
- package/dist/runtime/services/worktree-service.js +18 -0
- package/dist/schedule/passes/candidates.js +17 -8
- package/docs/config-architecture.md +105 -0
- package/package.json +7 -2
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { join } from "node:path";
|
|
1
|
+
import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
3
|
import { execFileSync } from "node:child_process";
|
|
4
4
|
//#region src/runtime/parallel-worktrees/parallel-worktrees.ts
|
|
5
5
|
/**
|
|
@@ -13,6 +13,50 @@ import { execFileSync } from "node:child_process";
|
|
|
13
13
|
const WORKTREE_ROOT = ".pipeline/worktrees";
|
|
14
14
|
const REGISTRY_DIR = join(WORKTREE_ROOT, "registry");
|
|
15
15
|
const OWNER = "oisin-pipeline";
|
|
16
|
+
const GENERATED_WORKTREE_RESOURCES = [join(".opencode", "agents"), join(".opencode", "command")];
|
|
17
|
+
function provisionGeneratedResources(repoRoot, worktreePath) {
|
|
18
|
+
for (const relativePath of GENERATED_WORKTREE_RESOURCES) {
|
|
19
|
+
const source = join(repoRoot, relativePath);
|
|
20
|
+
const target = join(worktreePath, relativePath);
|
|
21
|
+
if (existsSync(source) && !existsSync(target)) cpSync(source, target, { recursive: true });
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function childWorktreeRelPath(runId, parentNodeId, childNodeId) {
|
|
25
|
+
return join(WORKTREE_ROOT, "trees", sanitize(runId ?? "local"), sanitize(parentNodeId), sanitize(childNodeId));
|
|
26
|
+
}
|
|
27
|
+
function changedWorktreeFiles(worktreePath) {
|
|
28
|
+
const modified = git(worktreePath, [
|
|
29
|
+
"diff",
|
|
30
|
+
"--name-only",
|
|
31
|
+
"HEAD"
|
|
32
|
+
]);
|
|
33
|
+
const untracked = git(worktreePath, [
|
|
34
|
+
"ls-files",
|
|
35
|
+
"--others",
|
|
36
|
+
"--exclude-standard"
|
|
37
|
+
]);
|
|
38
|
+
return [...modified.split("\n"), ...untracked.split("\n")].filter(Boolean);
|
|
39
|
+
}
|
|
40
|
+
function copyFileInto(fromRoot, toRoot, relativePath) {
|
|
41
|
+
const source = join(fromRoot, relativePath);
|
|
42
|
+
if (!existsSync(source)) return;
|
|
43
|
+
const dest = join(toRoot, relativePath);
|
|
44
|
+
mkdirSync(dirname(dest), { recursive: true });
|
|
45
|
+
cpSync(source, dest, { recursive: true });
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* PIPE-83.14: promote a best-of-N winner's edits from its (retained) worktree
|
|
49
|
+
* back into the main worktree, so downstream nodes (tests, verification) see the
|
|
50
|
+
* selected candidate's changes. Copies modified + untracked files; no-op if the
|
|
51
|
+
* worktree is gone. Returns the promoted file paths.
|
|
52
|
+
*/
|
|
53
|
+
function promoteWorktreeChanges(repoRoot, runId, parentNodeId, childNodeId) {
|
|
54
|
+
const worktreePath = join(repoRoot, childWorktreeRelPath(runId, parentNodeId, childNodeId));
|
|
55
|
+
if (!existsSync(worktreePath)) return [];
|
|
56
|
+
const files = changedWorktreeFiles(worktreePath);
|
|
57
|
+
for (const relativePath of files) copyFileInto(worktreePath, repoRoot, relativePath);
|
|
58
|
+
return files;
|
|
59
|
+
}
|
|
16
60
|
function git(cwd, args) {
|
|
17
61
|
return execFileSync("git", args, {
|
|
18
62
|
cwd,
|
|
@@ -33,7 +77,7 @@ function createChildWorktree(opts) {
|
|
|
33
77
|
const parentSeg = sanitize(opts.parentNodeId);
|
|
34
78
|
const childSeg = sanitize(opts.childNodeId);
|
|
35
79
|
const baseSha = git(opts.repoRoot, ["rev-parse", "HEAD"]);
|
|
36
|
-
const relPath =
|
|
80
|
+
const relPath = childWorktreeRelPath(opts.runId, opts.parentNodeId, opts.childNodeId);
|
|
37
81
|
const absPath = join(opts.repoRoot, relPath);
|
|
38
82
|
const branch = `pipeline/worktrees/${runSeg}/${parentSeg}/${childSeg}`;
|
|
39
83
|
const leaseId = `${runSeg}__${parentSeg}__${childSeg}`;
|
|
@@ -61,6 +105,7 @@ function createChildWorktree(opts) {
|
|
|
61
105
|
absPath,
|
|
62
106
|
baseSha
|
|
63
107
|
]);
|
|
108
|
+
provisionGeneratedResources(opts.repoRoot, absPath);
|
|
64
109
|
writeManifest(manifestPath, {
|
|
65
110
|
...manifest,
|
|
66
111
|
state: "active"
|
|
@@ -129,4 +174,4 @@ function gcParallelWorktrees(repoRoot) {
|
|
|
129
174
|
return results;
|
|
130
175
|
}
|
|
131
176
|
//#endregion
|
|
132
|
-
export { createChildWorktree, gcParallelWorktrees };
|
|
177
|
+
export { createChildWorktree, gcParallelWorktrees, promoteWorktreeChanges };
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
//#region src/runtime/run-journal.ts
|
|
4
|
+
function passedOnly(results) {
|
|
5
|
+
return results.filter((result) => result.status === "passed");
|
|
6
|
+
}
|
|
7
|
+
function readJournalFile(path) {
|
|
8
|
+
if (!existsSync(path)) return [];
|
|
9
|
+
return readFileSync(path, "utf8").split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
|
|
10
|
+
}
|
|
11
|
+
function fileRunJournal(path) {
|
|
12
|
+
return {
|
|
13
|
+
record: (result) => {
|
|
14
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
15
|
+
appendFileSync(path, `${JSON.stringify(result)}\n`);
|
|
16
|
+
},
|
|
17
|
+
resumeCompleted: () => passedOnly(readJournalFile(path))
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
//#endregion
|
|
21
|
+
export { fileRunJournal };
|
|
@@ -1,44 +1,16 @@
|
|
|
1
1
|
import { uniqueStrings } from "../strings.js";
|
|
2
|
-
import {
|
|
2
|
+
import { Effect, Fiber, Queue } from "effect";
|
|
3
3
|
//#region src/runtime/scheduler.ts
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
return (await runWorkflowLifecycle({
|
|
13
|
-
buildResult: options.buildResult,
|
|
14
|
-
emitWorkflowPlanned: () => options.emitWorkflowPlanned(context),
|
|
15
|
-
emitWorkflowStarted: () => options.emitWorkflowStarted(context),
|
|
16
|
-
executeWorkflow: () => runWorkflowScheduler({
|
|
17
|
-
failFast: plan.execution.failFast,
|
|
18
|
-
fanOutWidth: context.config.token_budget?.fan_out_width,
|
|
19
|
-
isCancelled: () => options.isCancelled(context),
|
|
20
|
-
markNodeReady: (nodeId) => options.markNodeReady(nodeId, context),
|
|
21
|
-
maxParallelNodes: context.maxParallelNodes,
|
|
22
|
-
nodes: plan.topologicalOrder.map((node) => ({
|
|
23
|
-
category: node.category,
|
|
24
|
-
dependents: node.dependents,
|
|
25
|
-
id: node.id,
|
|
26
|
-
index: node.index,
|
|
27
|
-
needs: node.needs
|
|
28
|
-
})),
|
|
29
|
-
runNode: (nodeId) => options.executeNode(nodeId, context),
|
|
30
|
-
shouldContinueAfterNodeResult: (result) => options.shouldContinueAfterNodeResult(result, context),
|
|
31
|
-
skipNode: (nodeId, reason) => options.skipNode(nodeId, reason, context)
|
|
32
|
-
}),
|
|
33
|
-
isCancelled: () => options.isCancelled(context),
|
|
34
|
-
runWorkflowHook: (event, failure) => options.runWorkflowHook(event, failure, context)
|
|
35
|
-
})).result;
|
|
36
|
-
}
|
|
37
|
-
};
|
|
38
|
-
async function runWorkflowScheduler(input) {
|
|
39
|
-
const state = {
|
|
4
|
+
function resumeFromJournal(input) {
|
|
5
|
+
return input.journal?.resumeCompleted() ?? [];
|
|
6
|
+
}
|
|
7
|
+
function recordToJournal(input, result) {
|
|
8
|
+
input.journal?.record(result);
|
|
9
|
+
}
|
|
10
|
+
function initialSchedulerState(input) {
|
|
11
|
+
return {
|
|
40
12
|
blocked: [],
|
|
41
|
-
completed:
|
|
13
|
+
completed: resumeFromJournal(input),
|
|
42
14
|
failFast: input.failFast,
|
|
43
15
|
fanOutWidth: input.fanOutWidth,
|
|
44
16
|
maxParallelNodes: input.maxParallelNodes,
|
|
@@ -46,50 +18,120 @@ async function runWorkflowScheduler(input) {
|
|
|
46
18
|
running: [],
|
|
47
19
|
shouldContinueAfterNodeResult: input.shouldContinueAfterNodeResult
|
|
48
20
|
};
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
21
|
+
}
|
|
22
|
+
function cancelledResult(state) {
|
|
23
|
+
return {
|
|
24
|
+
completed: state.completed,
|
|
25
|
+
outcome: "CANCELLED"
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function terminalResult(state, failure) {
|
|
29
|
+
return {
|
|
30
|
+
completed: state.completed,
|
|
31
|
+
failure,
|
|
32
|
+
outcome: failure ? "FAIL" : "PASS"
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
function nodeErrorResult(state, error) {
|
|
36
|
+
return {
|
|
37
|
+
completed: state.completed,
|
|
38
|
+
failure: workflowServiceFailure(error, "workflow.node"),
|
|
39
|
+
outcome: "FAIL"
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
function runNodeFiber(ctx, nodeId) {
|
|
43
|
+
return Effect.tryPromise({
|
|
44
|
+
catch: (error) => error,
|
|
45
|
+
try: () => ctx.input.runNode(nodeId)
|
|
46
|
+
}).pipe(Effect.matchEffect({
|
|
47
|
+
onFailure: (error) => Queue.offer(ctx.completions, {
|
|
48
|
+
error,
|
|
49
|
+
kind: "error"
|
|
50
|
+
}),
|
|
51
|
+
onSuccess: (result) => Queue.offer(ctx.completions, {
|
|
52
|
+
kind: "ok",
|
|
53
|
+
result
|
|
54
|
+
})
|
|
55
|
+
}));
|
|
56
|
+
}
|
|
57
|
+
function launchReady(ctx) {
|
|
58
|
+
return Effect.gen(function* () {
|
|
59
|
+
const capacity = workflowNodeCapacity(ctx.state);
|
|
60
|
+
if (capacity <= 0) return;
|
|
61
|
+
for (const nodeId of selectLaunchableNodes(ctx.state, capacity)) {
|
|
62
|
+
ctx.input.markNodeReady(nodeId);
|
|
63
|
+
ctx.state.running = [...ctx.state.running, nodeId];
|
|
64
|
+
const fiber = yield* Effect.fork(runNodeFiber(ctx, nodeId));
|
|
65
|
+
ctx.running.set(nodeId, fiber);
|
|
83
66
|
}
|
|
84
|
-
|
|
85
|
-
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
function applyFailFastSkip(ctx, result) {
|
|
70
|
+
const reason = `skipped because workflow fail_fast stopped after node '${result.nodeId}' failed`;
|
|
71
|
+
const skipped = unstartedNodeIds(ctx.state);
|
|
72
|
+
ctx.state.blocked = uniqueStrings([...ctx.state.blocked, ...skipped]);
|
|
73
|
+
for (const nodeId of skipped) ctx.input.skipNode(nodeId, reason);
|
|
74
|
+
}
|
|
75
|
+
function applyCompletion(ctx, result) {
|
|
76
|
+
ctx.running.delete(result.nodeId);
|
|
77
|
+
ctx.state.running = ctx.state.running.filter((id) => id !== result.nodeId);
|
|
78
|
+
ctx.state.completed = [...ctx.state.completed, result];
|
|
79
|
+
recordToJournal(ctx.input, result);
|
|
80
|
+
if (!isBlockingFailure(result, ctx.state)) return;
|
|
81
|
+
ctx.failure ??= nodeRuntimeFailure(result);
|
|
82
|
+
if (ctx.input.failFast) {
|
|
83
|
+
applyFailFastSkip(ctx, result);
|
|
84
|
+
return;
|
|
86
85
|
}
|
|
86
|
+
const blocked = unstartedBlockingDescendants(result.nodeId, ctx.state);
|
|
87
|
+
ctx.state.blocked = uniqueStrings([...ctx.state.blocked, ...blocked]);
|
|
88
|
+
}
|
|
89
|
+
function applyOutcome(ctx, outcome) {
|
|
90
|
+
return Effect.gen(function* () {
|
|
91
|
+
if (outcome.kind === "error") {
|
|
92
|
+
yield* Fiber.interruptAll(ctx.running.values());
|
|
93
|
+
return nodeErrorResult(ctx.state, outcome.error);
|
|
94
|
+
}
|
|
95
|
+
applyCompletion(ctx, outcome.result);
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
function schedulerTick(ctx) {
|
|
99
|
+
return Effect.gen(function* () {
|
|
100
|
+
if (ctx.input.isCancelled()) {
|
|
101
|
+
yield* Fiber.interruptAll(ctx.running.values());
|
|
102
|
+
return cancelledResult(ctx.state);
|
|
103
|
+
}
|
|
104
|
+
yield* launchReady(ctx);
|
|
105
|
+
if (ctx.running.size === 0) return terminalResult(ctx.state, ctx.failure);
|
|
106
|
+
return yield* applyOutcome(ctx, yield* Queue.take(ctx.completions));
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
function schedulerProgram(input) {
|
|
110
|
+
return Effect.gen(function* () {
|
|
111
|
+
const ctx = {
|
|
112
|
+
completions: yield* Queue.unbounded(),
|
|
113
|
+
input,
|
|
114
|
+
running: /* @__PURE__ */ new Map(),
|
|
115
|
+
state: initialSchedulerState(input)
|
|
116
|
+
};
|
|
117
|
+
while (true) {
|
|
118
|
+
const done = yield* schedulerTick(ctx);
|
|
119
|
+
if (done) return done;
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
function runWorkflowScheduler(input) {
|
|
124
|
+
return Effect.runPromise(schedulerProgram(input));
|
|
125
|
+
}
|
|
126
|
+
function settledNodeIds(context) {
|
|
127
|
+
const ids = new Set((context.completed ?? []).map((result) => result.nodeId));
|
|
128
|
+
for (const nodeId of context.running) ids.add(nodeId);
|
|
129
|
+
return ids;
|
|
87
130
|
}
|
|
88
131
|
function readyNodeIds(context) {
|
|
132
|
+
const settled = settledNodeIds(context);
|
|
89
133
|
const blocked = new Set(context.blocked ?? []);
|
|
90
|
-
|
|
91
|
-
const running = new Set(context.running);
|
|
92
|
-
return orderedNodes(context.nodes).filter((node) => !completed.has(node.id)).filter((node) => !running.has(node.id)).filter((node) => !blocked.has(node.id)).filter((node) => node.needs.every((need) => dependencyPassed(need, context))).map((node) => node.id);
|
|
134
|
+
return orderedNodes(context.nodes).filter((node) => !(settled.has(node.id) || blocked.has(node.id))).filter((node) => node.needs.every((need) => dependencyPassed(need, context))).map((node) => node.id);
|
|
93
135
|
}
|
|
94
136
|
function workflowNodeCapacity(context) {
|
|
95
137
|
const limit = context.failFast ? 1 : context.maxParallelNodes ?? context.nodes.length;
|
|
@@ -107,18 +149,6 @@ function unstartedBlockingDescendants(nodeId, context) {
|
|
|
107
149
|
}
|
|
108
150
|
return orderedNodes(context.nodes).map((node) => node.id).filter((descendantId) => descendants.has(descendantId)).filter((descendantId) => unstarted.has(descendantId));
|
|
109
151
|
}
|
|
110
|
-
function launchReadyNodes(input, state, running) {
|
|
111
|
-
const capacity = workflowNodeCapacity(state);
|
|
112
|
-
if (capacity <= 0) return;
|
|
113
|
-
for (const nodeId of selectLaunchableNodes(state, capacity)) {
|
|
114
|
-
input.markNodeReady(nodeId);
|
|
115
|
-
state.running = [...state.running, nodeId];
|
|
116
|
-
running.set(nodeId, {
|
|
117
|
-
nodeId,
|
|
118
|
-
promise: input.runNode(nodeId)
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
152
|
/**
|
|
123
153
|
* Choose which ready nodes to launch this tick within the global capacity and
|
|
124
154
|
* the per-category fan-out caps. A category at its cap defers its remaining
|
|
@@ -171,9 +201,8 @@ function isBlockingFailure(result, context) {
|
|
|
171
201
|
return result.status === "failed" && !(context.shouldContinueAfterNodeResult?.(result) ?? false);
|
|
172
202
|
}
|
|
173
203
|
function unstartedNodeIds(context) {
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
return orderedNodes(context.nodes).map((node) => node.id).filter((nodeId) => !completed.has(nodeId)).filter((nodeId) => !running.has(nodeId));
|
|
204
|
+
const settled = settledNodeIds(context);
|
|
205
|
+
return orderedNodes(context.nodes).map((node) => node.id).filter((nodeId) => !settled.has(nodeId));
|
|
177
206
|
}
|
|
178
207
|
function directDependents(nodeId, nodes) {
|
|
179
208
|
const declared = nodes.find((node) => node.id === nodeId)?.dependents ?? [];
|
|
@@ -201,4 +230,4 @@ function workflowServiceFailure(error, gate) {
|
|
|
201
230
|
};
|
|
202
231
|
}
|
|
203
232
|
//#endregion
|
|
204
|
-
export {
|
|
233
|
+
export { runWorkflowScheduler };
|
|
@@ -2,6 +2,7 @@ import { createRunnerLaunchPlan } from "../../runner.js";
|
|
|
2
2
|
import { normalizeRunnerOutput } from "../../runner-output.js";
|
|
3
3
|
import { parseJsonObject } from "../json-validation/json-validation.js";
|
|
4
4
|
import "../json-validation/index.js";
|
|
5
|
+
import { promoteWorktreeChanges } from "../parallel-worktrees/parallel-worktrees.js";
|
|
5
6
|
//#region src/runtime/select-candidate/select-candidate.ts
|
|
6
7
|
const SCORE_RE = /-?\d+(?:\.\d+)?/;
|
|
7
8
|
function selectBestCandidate(candidates) {
|
|
@@ -17,12 +18,23 @@ async function executeSelectCandidateBuiltin(context, node) {
|
|
|
17
18
|
exitCode: 1,
|
|
18
19
|
output: ""
|
|
19
20
|
};
|
|
21
|
+
const promoted = promoteWinner(context, node, selected.nodeId);
|
|
20
22
|
return {
|
|
21
|
-
evidence:
|
|
23
|
+
evidence: selectionEvidence(selected, candidates.length, promoted),
|
|
22
24
|
exitCode: 0,
|
|
23
25
|
output: selected.output
|
|
24
26
|
};
|
|
25
27
|
}
|
|
28
|
+
function selectionEvidence(selected, candidateCount, promoted) {
|
|
29
|
+
const lines = [`select-candidate: selected '${selected.nodeId}' (judge=${selected.judgeScore ?? "n/a"}) from ${candidateCount} candidates`];
|
|
30
|
+
if (promoted.length > 0) lines.push(`promoted ${promoted.length} file(s) from the winning worktree`);
|
|
31
|
+
return lines;
|
|
32
|
+
}
|
|
33
|
+
function promoteWinner(context, node, winnerNodeId) {
|
|
34
|
+
const parentNodeId = node?.needs.at(0);
|
|
35
|
+
if (!(context.config.parallel_worktrees?.enabled && parentNodeId)) return [];
|
|
36
|
+
return promoteWorktreeChanges(context.worktreePath, context.runId, parentNodeId, winnerNodeId);
|
|
37
|
+
}
|
|
26
38
|
async function scoreCandidates(context, candidates) {
|
|
27
39
|
const model = context.config.best_of_n?.judge_model;
|
|
28
40
|
const runner = Object.keys(context.config.runners).at(0);
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { createChildWorktree, gcParallelWorktrees } from "../parallel-worktrees/parallel-worktrees.js";
|
|
2
|
+
import { Context, Effect, Layer } from "effect";
|
|
3
|
+
//#region src/runtime/services/worktree-service.ts
|
|
4
|
+
/**
|
|
5
|
+
* Effect service over the git-worktree lifecycle (PIPE-83 follow-up). The Live
|
|
6
|
+
* layer delegates to the synchronous porcelain helpers in parallel-worktrees;
|
|
7
|
+
* the Effect-native parallel-node runtime composes `createChild`/`gc` through
|
|
8
|
+
* this injectable seam instead of calling the helpers directly. This is the
|
|
9
|
+
* canonical service shape for the Effect conversion: a `Context.Tag` whose Live
|
|
10
|
+
* `Layer` wraps the underlying IO, provided once at the runPromise boundary.
|
|
11
|
+
*/
|
|
12
|
+
var WorktreeService = class extends Context.Tag("WorktreeService")() {};
|
|
13
|
+
const WorktreeServiceLive = Layer.succeed(WorktreeService, {
|
|
14
|
+
createChild: (opts) => Effect.sync(() => createChildWorktree(opts)),
|
|
15
|
+
gc: (repoRoot) => Effect.sync(() => gcParallelWorktrees(repoRoot))
|
|
16
|
+
});
|
|
17
|
+
//#endregion
|
|
18
|
+
export { WorktreeService, WorktreeServiceLive };
|
|
@@ -19,24 +19,33 @@ function expandBestOfNCandidates(config, artifact) {
|
|
|
19
19
|
}]))
|
|
20
20
|
};
|
|
21
21
|
}
|
|
22
|
+
function candidateChild(node, index) {
|
|
23
|
+
const child = {
|
|
24
|
+
...node,
|
|
25
|
+
id: `${node.id}--c${index + 1}`,
|
|
26
|
+
needs: []
|
|
27
|
+
};
|
|
28
|
+
child.task_context = void 0;
|
|
29
|
+
return child;
|
|
30
|
+
}
|
|
22
31
|
function expandNode(node, categories, n) {
|
|
23
32
|
if (node.kind !== "agent" || !categories.some((category) => node.id.includes(category))) return [node];
|
|
24
33
|
const candidatesId = `${node.id}--candidates`;
|
|
25
34
|
return [{
|
|
26
35
|
id: candidatesId,
|
|
27
36
|
kind: "parallel",
|
|
28
|
-
nodes: Array.from({ length: n }, (_, index) => (
|
|
29
|
-
...node,
|
|
30
|
-
id: `${node.id}--c${index + 1}`,
|
|
31
|
-
needs: []
|
|
32
|
-
})),
|
|
37
|
+
nodes: Array.from({ length: n }, (_, index) => candidateChild(node, index)),
|
|
33
38
|
...node.needs ? { needs: node.needs } : {}
|
|
34
|
-
},
|
|
39
|
+
}, selectCandidateNode(node, candidatesId)];
|
|
40
|
+
}
|
|
41
|
+
function selectCandidateNode(node, candidatesId) {
|
|
42
|
+
return {
|
|
35
43
|
builtin: "select-candidate",
|
|
36
44
|
id: node.id,
|
|
37
45
|
kind: "builtin",
|
|
38
|
-
needs: [candidatesId]
|
|
39
|
-
|
|
46
|
+
needs: [candidatesId],
|
|
47
|
+
...node.task_context ? { task_context: node.task_context } : {}
|
|
48
|
+
};
|
|
40
49
|
}
|
|
41
50
|
//#endregion
|
|
42
51
|
export { expandBestOfNCandidates };
|
|
@@ -164,6 +164,13 @@ Project-authored skill and rule paths resolve from the project root and must
|
|
|
164
164
|
exist for runtime use. If default skill files are missing, run `moka init` to
|
|
165
165
|
install them before executing workflows.
|
|
166
166
|
|
|
167
|
+
`moka init --skill-scope` (PIPE-83.12) chooses how the default set is installed:
|
|
168
|
+
`project` (default) vendors a repo-local copy (`skills add … --copy`,
|
|
169
|
+
`skills-lock.json`); `personal` installs once at user/global scope
|
|
170
|
+
(`skills add … --global`) so every repo the user opens inherits the skills with
|
|
171
|
+
no per-repo copy and no project lockfile — the standardization path for a single
|
|
172
|
+
user across many projects.
|
|
173
|
+
|
|
167
174
|
MCP-enabled profiles use one gateway grant:
|
|
168
175
|
|
|
169
176
|
```yaml
|
|
@@ -352,6 +359,104 @@ The toposort uses `@dagrejs/graphlib` for the graph model but an iterative
|
|
|
352
359
|
traversal for the topological sort, because graphlib's recursive topsort can
|
|
353
360
|
overflow the call stack on deep generated workflow chains.
|
|
354
361
|
|
|
362
|
+
## Context, durability & best-of-N features (PIPE-83)
|
|
363
|
+
|
|
364
|
+
The shipped `defaults/pipeline.yaml` turns **`context_handoff`, `repo_map`,
|
|
365
|
+
`durability`, `best_of_n` (n=2 on green), and `parallel_worktrees` ON** — moka
|
|
366
|
+
uses its architecture by default. `best_of_n` is the cost/quality dial: n=2
|
|
367
|
+
~doubles green-node spend, so lower `n` or set `enabled: false` to spend the
|
|
368
|
+
minimum. Each block can be overridden in `pipeline.yaml`; the per-block
|
|
369
|
+
`# default …` notes below mark the *schema* default that applies when a block is
|
|
370
|
+
omitted entirely.
|
|
371
|
+
|
|
372
|
+
### `context_handoff` — curated node-to-node handoffs
|
|
373
|
+
|
|
374
|
+
```yaml
|
|
375
|
+
context_handoff:
|
|
376
|
+
enabled: true # default false
|
|
377
|
+
model: openai/gpt-5.5-fast # optional cheap model for the deriving call
|
|
378
|
+
```
|
|
379
|
+
|
|
380
|
+
When enabled, each agent node derives a structured `NodeHandoff`
|
|
381
|
+
(`summary`, `decisions`, `artifacts`, `testNames`, `openQuestions`) from its raw
|
|
382
|
+
output via a cheap read-only finalizer. Downstream nodes then receive that
|
|
383
|
+
curated handoff in their prompt instead of the upstream node's full transcript
|
|
384
|
+
(`renderAgentPrompt`), which kills the prompt-bloat from re-hydrating every
|
|
385
|
+
ancestor's output. When a node produces no handoff, consumers fall back to the
|
|
386
|
+
raw output text, so disabling the flag is byte-identical to prior behaviour.
|
|
387
|
+
|
|
388
|
+
### `parallel_worktrees` — isolated parallel children
|
|
389
|
+
|
|
390
|
+
```yaml
|
|
391
|
+
parallel_worktrees:
|
|
392
|
+
enabled: true # default false
|
|
393
|
+
```
|
|
394
|
+
|
|
395
|
+
When enabled, every child of a `kind: parallel` node runs in its own git
|
|
396
|
+
worktree on an auto-named branch (`.pipeline/worktrees/...`), so concurrent
|
|
397
|
+
candidate edits cannot collide. Teardown is idempotent and crash-safe: a
|
|
398
|
+
worktree with dirty or unpushed work is retained (never deleted), and orphaned
|
|
399
|
+
worktrees are GC'd on the next parallel node under the same guard. A worktree is
|
|
400
|
+
*not* a sandbox (node_modules/build state are shared) — real isolation remains
|
|
401
|
+
k8s mode. SDK-runner nodes honour the per-child directory via `plan.cwd`.
|
|
402
|
+
|
|
403
|
+
### `best_of_n` — generate N candidates and select the winner
|
|
404
|
+
|
|
405
|
+
```yaml
|
|
406
|
+
best_of_n:
|
|
407
|
+
enabled: true # default false
|
|
408
|
+
n: 3 # candidates per matching node (default 1 = off)
|
|
409
|
+
categories: [green] # which node categories fan out (id-substring match)
|
|
410
|
+
judge_model: openai/gpt-5.5 # optional LLM judge; omit for status-only
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
When enabled with `n > 1`, the `candidates` schedule pass expands each matching
|
|
414
|
+
agent node into a `kind: parallel` of N candidate children
|
|
415
|
+
(`<node>--candidates`) feeding a `select-candidate` builtin that keeps the
|
|
416
|
+
original node id. Each candidate builds in its own worktree (pair with
|
|
417
|
+
`parallel_worktrees`) and is throttled by its category's
|
|
418
|
+
`token_budget.fan_out_width` cap. The selector scores candidates by a hybrid of
|
|
419
|
+
execution status (PASS/FAIL) and, when `judge_model` is set, a 0..1 LLM judge
|
|
420
|
+
score; it emits the winner's output and **never self-fixes** — if no candidate
|
|
421
|
+
passes, the node fails with evidence.
|
|
422
|
+
|
|
423
|
+
### `durability` — durable crash-resume
|
|
424
|
+
|
|
425
|
+
```yaml
|
|
426
|
+
durability:
|
|
427
|
+
enabled: true # default false
|
|
428
|
+
dir: .pipeline/journal # default
|
|
429
|
+
```
|
|
430
|
+
|
|
431
|
+
When enabled, the scheduler journals each terminal node result to an
|
|
432
|
+
append-only JSONL log under `dir`, keyed by run id (`<dir>/<runId>.jsonl`). A
|
|
433
|
+
killed run, re-invoked with the same run id, resumes from the last **passed**
|
|
434
|
+
node — finished nodes are not re-run and their tokens are not re-spent — while a
|
|
435
|
+
failed node and everything downstream replay so fail-fast and blocked-descendant
|
|
436
|
+
handling stay live. The journal is a swappable seam (`RunJournal`): the shipped
|
|
437
|
+
`fileRunJournal` needs zero external infra; a future `@effect/workflow` / cluster
|
|
438
|
+
provider can implement the same interface without touching the scheduler. Off by
|
|
439
|
+
default → the scheduler runs purely in-memory, exactly as before.
|
|
440
|
+
|
|
441
|
+
### `mcp_gateway.host_scope` — register the gateway once globally
|
|
442
|
+
|
|
443
|
+
```yaml
|
|
444
|
+
mcp_gateway:
|
|
445
|
+
provider: toolhive
|
|
446
|
+
mode: hosted
|
|
447
|
+
host_scope: global # default "project"
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
The singleton pipeline gateway is normally synthesized into each repo's
|
|
451
|
+
`.opencode/opencode.json` (`host_scope: project`, the default — unchanged
|
|
452
|
+
goldens). Setting `host_scope: global` stops that per-project synthesis: the
|
|
453
|
+
generated project config omits the `pipeline-gateway` MCP entry and instead
|
|
454
|
+
inherits one global registration written once via
|
|
455
|
+
`moka gateway configure-host --scope global` (which targets
|
|
456
|
+
`$OPENCODE_CONFIG_DIR`/`$XDG_CONFIG_HOME/opencode/opencode.json`). This is the
|
|
457
|
+
PIPE-83.11 standardization path — register the MCP gateway once per machine
|
|
458
|
+
rather than re-emitting it into every project.
|
|
459
|
+
|
|
355
460
|
## Troubleshooting
|
|
356
461
|
|
|
357
462
|
- Missing host resources: run `moka install-commands`; `moka run` loads the
|
package/package.json
CHANGED
|
@@ -6,18 +6,23 @@
|
|
|
6
6
|
"ajv": "^8.20.0",
|
|
7
7
|
"ajv-formats": "^3.0.1",
|
|
8
8
|
"commander": "^14.0.3",
|
|
9
|
+
"effect": "^3.21.3",
|
|
9
10
|
"execa": "^9.5.2",
|
|
10
11
|
"git-url-parse": "^16.1.0",
|
|
12
|
+
"graphology": "0.26.0",
|
|
13
|
+
"graphology-metrics": "2.4.0",
|
|
11
14
|
"gray-matter": "^4.0.3",
|
|
12
15
|
"js-tiktoken": "^1.0.21",
|
|
13
16
|
"jsonc-parser": "^3.3.1",
|
|
14
17
|
"ky": "^2.0.2",
|
|
15
18
|
"micromatch": "^4.0.8",
|
|
16
|
-
"p-limit": "^7.3.0",
|
|
17
19
|
"package-manager-detector": "^1.6.0",
|
|
18
20
|
"pino": "^10.3.1",
|
|
19
21
|
"secure-json-parse": "^4.1.0",
|
|
20
22
|
"simple-git": "^3.36.0",
|
|
23
|
+
"tree-sitter-javascript": "0.25.0",
|
|
24
|
+
"tree-sitter-typescript": "0.23.2",
|
|
25
|
+
"web-tree-sitter": "0.26.9",
|
|
21
26
|
"yaml": "^2.9.0",
|
|
22
27
|
"zod": "^4.4.3"
|
|
23
28
|
},
|
|
@@ -121,7 +126,7 @@
|
|
|
121
126
|
"prepack": "bun run build:cli"
|
|
122
127
|
},
|
|
123
128
|
"type": "module",
|
|
124
|
-
"version": "2.
|
|
129
|
+
"version": "2.8.0",
|
|
125
130
|
"description": "Config-driven multi-agent pipeline runner for repository work",
|
|
126
131
|
"main": "./dist/index.js",
|
|
127
132
|
"types": "./dist/index.d.ts",
|