@oisincoveney/pipeline 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/orchestrate/SKILL.md +80 -0
- package/defaults/profiles.yaml +5 -2
- package/dist/argo-submit.d.ts +0 -1
- package/dist/argo-submit.js +1 -4
- package/dist/argo-workflow.d.ts +1 -2
- package/dist/argo-workflow.js +0 -2
- package/dist/cli/program.js +2 -3
- package/dist/cli/submit-options.js +1 -2
- package/dist/cluster-doctor.js +0 -12
- package/dist/commands/pipeline-command.js +1 -1
- package/dist/config/schemas.d.ts +4 -4
- package/dist/install-commands/opencode.js +17 -7
- package/dist/moka-global-config.d.ts +0 -1
- package/dist/moka-global-config.js +0 -1
- package/dist/moka-submit.d.ts +1 -4
- package/dist/moka-submit.js +1 -4
- package/dist/pipeline-runtime.d.ts +9 -0
- package/dist/planned-node.js +2 -5
- package/dist/{workflow-planner.d.ts → planning/compile.d.ts} +2 -2
- package/dist/{workflow-planner.js → planning/compile.js} +6 -83
- package/dist/{schedule/planner.d.ts → planning/generate.d.ts} +17 -3
- package/dist/{schedule/planner.js → planning/generate.js} +24 -56
- package/dist/planning/graph.js +138 -0
- package/dist/runner-command/lifecycle-context.js +2 -3
- package/dist/runner-command/run.js +2 -3
- package/dist/runner.d.ts +27 -0
- package/dist/runtime/context/context.js +1 -1
- package/dist/runtime/contracts/contracts.d.ts +16 -1
- package/dist/schedule/passes/coverage.js +7 -51
- package/dist/schedule/passes/ids.js +3 -23
- package/dist/schedule/scheduling-roles.js +19 -0
- package/dist/strings.js +30 -1
- package/docs/config-architecture.md +32 -0
- package/docs/operator-guide.md +2 -3
- package/docs/pipeline-console-runner-contract.md +3 -4
- package/package.json +5 -5
- package/dist/schedule-planner.d.ts +0 -2
- package/dist/schedule-planner.js +0 -2
|
@@ -3,20 +3,22 @@ import { validatePipelineConfig } from "../config/validate.js";
|
|
|
3
3
|
import "../config.js";
|
|
4
4
|
import { createRunnerLaunchPlan, runLaunchPlan } from "../runner.js";
|
|
5
5
|
import { normalizeRunnerOutput } from "../runner-output.js";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
6
|
+
import { loadBacklogPlanningContext } from "../schedule/backlog-context.js";
|
|
7
|
+
import { baselineScheduleArtifact } from "../schedule/baseline.js";
|
|
8
|
+
import { dependentsByNeed, flattenNodes, hasReachableDependent } from "./graph.js";
|
|
9
|
+
import { isCoverageNode, isImplementationNode } from "../schedule/scheduling-roles.js";
|
|
10
|
+
import { addGeneratedImplementationCoverage } from "../schedule/passes/coverage.js";
|
|
11
|
+
import { canonicalizeGeneratedScheduleIds } from "../schedule/passes/ids.js";
|
|
12
|
+
import { SCHEDULE_PASS_ORDER } from "../schedule/passes/index.js";
|
|
13
|
+
import { applyNodeCatalogModelFallbacks } from "../schedule/passes/models.js";
|
|
14
|
+
import { namespaceScheduleWorkflows } from "../schedule/passes/references.js";
|
|
15
|
+
import { plannerPrompt, plannerRepairPrompt } from "../schedule/prompts.js";
|
|
16
|
+
import { compileWorkflowPlan } from "./compile.js";
|
|
15
17
|
import { parseDocument, stringify } from "yaml";
|
|
16
18
|
import { z } from "zod";
|
|
17
19
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
18
20
|
import { join } from "node:path";
|
|
19
|
-
//#region src/
|
|
21
|
+
//#region src/planning/generate.ts
|
|
20
22
|
const SCHEDULE_KIND = "pipeline-schedule";
|
|
21
23
|
const ID_RE = /^[a-z][a-z0-9-]*$/;
|
|
22
24
|
const SCHEDULE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
@@ -253,9 +255,9 @@ function unsafeParallelWorktreeIssues(config, artifact) {
|
|
|
253
255
|
}
|
|
254
256
|
function workflowNodeIssues(artifact, collectIssues) {
|
|
255
257
|
return Object.entries(artifact.workflows).flatMap(([workflowId, workflow]) => {
|
|
256
|
-
const nodes = workflow.nodes
|
|
258
|
+
const nodes = flattenWorkflowNodes(workflow.nodes);
|
|
257
259
|
return collectIssues({
|
|
258
|
-
dependentsByNeed:
|
|
260
|
+
dependentsByNeed: dependentsByNeed(nodes),
|
|
259
261
|
nodes,
|
|
260
262
|
workflowId
|
|
261
263
|
});
|
|
@@ -267,20 +269,8 @@ function isWriteCapableParallelChild(config, node) {
|
|
|
267
269
|
if (node.kind === "parallel") return node.nodes.some((child) => isWriteCapableParallelChild(config, child));
|
|
268
270
|
return false;
|
|
269
271
|
}
|
|
270
|
-
function hasDownstreamDrainMerge(nodeId,
|
|
271
|
-
return hasReachableDependent(nodeId,
|
|
272
|
-
}
|
|
273
|
-
function hasReachableDependent(nodeId, dependentsByNeed, matches) {
|
|
274
|
-
const queue = [...dependentsByNeed.get(nodeId) ?? []];
|
|
275
|
-
const seen = /* @__PURE__ */ new Set();
|
|
276
|
-
while (queue.length > 0) {
|
|
277
|
-
const node = queue.shift();
|
|
278
|
-
if (!node || seen.has(node.id)) continue;
|
|
279
|
-
seen.add(node.id);
|
|
280
|
-
if (matches(node)) return true;
|
|
281
|
-
queue.push(...dependentsByNeed.get(node.id) ?? []);
|
|
282
|
-
}
|
|
283
|
-
return false;
|
|
272
|
+
function hasDownstreamDrainMerge(nodeId, index) {
|
|
273
|
+
return hasReachableDependent(nodeId, index, (node) => node.kind === "builtin" && node.builtin === "drain-merge");
|
|
284
274
|
}
|
|
285
275
|
function generatedRootWorkflowIssues(artifact) {
|
|
286
276
|
const workflowIds = Object.keys(artifact.workflows);
|
|
@@ -331,15 +321,15 @@ function workUnitDependencyIssues(config, artifact, workUnits) {
|
|
|
331
321
|
const workUnitIds = new Set(workUnits.map((unit) => unit.id));
|
|
332
322
|
const dependenciesByUnit = new Map(workUnits.map((unit) => [unit.id, (unit.dependencies ?? []).filter((id) => workUnitIds.has(id))]));
|
|
333
323
|
return Object.entries(artifact.workflows).flatMap(([workflowId, workflow]) => {
|
|
334
|
-
const nodes = workflow.nodes
|
|
335
|
-
const
|
|
324
|
+
const nodes = flattenWorkflowNodes(workflow.nodes);
|
|
325
|
+
const index = dependentsByNeed(nodes);
|
|
336
326
|
const nodesByWorkUnit = nodesByAssignedWorkUnit(nodes);
|
|
337
327
|
return nodes.filter((node) => isImplementationNode(config, node)).flatMap((node) => {
|
|
338
328
|
const dependentId = node.task_context?.id;
|
|
339
329
|
if (!dependentId) return [];
|
|
340
330
|
return (dependenciesByUnit.get(dependentId) ?? []).flatMap((prerequisiteId) => {
|
|
341
331
|
const prerequisiteNodes = nodesByWorkUnit.get(prerequisiteId) ?? [];
|
|
342
|
-
return prerequisiteNodes.some((source) =>
|
|
332
|
+
return prerequisiteNodes.some((source) => hasReachableDependent(source.id, index, (candidate) => candidate.id === node.id)) ? [] : [`work unit dependency edge missing in '${workflowId}': '${dependentId}' node '${node.id}' must depend on prerequisite '${prerequisiteId}' nodes ${prerequisiteNodes.map((prerequisite) => `'${prerequisite.id}'`).join(", ")}`];
|
|
343
333
|
});
|
|
344
334
|
});
|
|
345
335
|
});
|
|
@@ -355,9 +345,6 @@ function nodesByAssignedWorkUnit(nodes) {
|
|
|
355
345
|
}
|
|
356
346
|
return grouped;
|
|
357
347
|
}
|
|
358
|
-
function hasPathToNode(sourceId, targetId, dependentsByNeed) {
|
|
359
|
-
return hasReachableDependent(sourceId, dependentsByNeed, (node) => node.id === targetId);
|
|
360
|
-
}
|
|
361
348
|
function unsupportedGeneratedBuiltinIssues(artifact) {
|
|
362
349
|
const allowed = new Set(SCHEDULE_BUILTINS);
|
|
363
350
|
return allWorkflowNodes(artifact.workflows).flatMap((node) => {
|
|
@@ -369,33 +356,14 @@ function unsupportedGeneratedBuiltinIssues(artifact) {
|
|
|
369
356
|
function implementationCoverageIssues(config, artifact) {
|
|
370
357
|
return workflowNodeIssues(artifact, ({ dependentsByNeed, nodes, workflowId }) => nodes.filter((node) => isImplementationNode(config, node)).filter((node) => !hasDownstreamCoverage(config, node.id, dependentsByNeed)).map((node) => `implementation node '${workflowId}.${node.id}' is without downstream verification or review`));
|
|
371
358
|
}
|
|
372
|
-
function
|
|
373
|
-
return
|
|
374
|
-
}
|
|
375
|
-
function workflowDependentsByNeed(nodes) {
|
|
376
|
-
const dependentsByNeed = /* @__PURE__ */ new Map();
|
|
377
|
-
for (const node of nodes) for (const need of node.needs ?? []) {
|
|
378
|
-
const dependents = dependentsByNeed.get(need) ?? [];
|
|
379
|
-
dependents.push(node);
|
|
380
|
-
dependentsByNeed.set(need, dependents);
|
|
381
|
-
}
|
|
382
|
-
return dependentsByNeed;
|
|
383
|
-
}
|
|
384
|
-
function hasDownstreamCoverage(config, nodeId, dependentsByNeed) {
|
|
385
|
-
return hasReachableDependent(nodeId, dependentsByNeed, (node) => isCoverageNode(config, node));
|
|
386
|
-
}
|
|
387
|
-
function isCoverageNode(config, node) {
|
|
388
|
-
return hasSchedulingRole(config, node, "coverage");
|
|
389
|
-
}
|
|
390
|
-
function hasSchedulingRole(config, node, role) {
|
|
391
|
-
if (node.kind !== "agent") return false;
|
|
392
|
-
return config.profiles[node.profile]?.scheduling_roles?.includes(role) ?? false;
|
|
359
|
+
function hasDownstreamCoverage(config, nodeId, index) {
|
|
360
|
+
return hasReachableDependent(nodeId, index, (node) => isCoverageNode(config, node));
|
|
393
361
|
}
|
|
394
362
|
function allWorkflowNodes(workflows) {
|
|
395
|
-
return Object.values(workflows).flatMap((workflow) => workflow.nodes
|
|
363
|
+
return Object.values(workflows).flatMap((workflow) => flattenWorkflowNodes(workflow.nodes));
|
|
396
364
|
}
|
|
397
|
-
function
|
|
398
|
-
return node.kind === "parallel" ?
|
|
365
|
+
function flattenWorkflowNodes(nodes) {
|
|
366
|
+
return flattenNodes(nodes, (node) => node.kind === "parallel" ? node.nodes : void 0);
|
|
399
367
|
}
|
|
400
368
|
//#endregion
|
|
401
369
|
export { ScheduleArtifactError, compileScheduleArtifact, generateScheduleArtifact, parseScheduleArtifact, scheduleArtifactPath };
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
//#region src/planning/graph.ts
|
|
2
|
+
/**
|
|
3
|
+
* Depth-first flatten of a node tree into a flat list (parents before
|
|
4
|
+
* children), using the caller-supplied accessor to descend into nested
|
|
5
|
+
* containers (e.g. parallel children). Nodes without children are returned
|
|
6
|
+
* as-is.
|
|
7
|
+
*/
|
|
8
|
+
function flattenNodes(nodes, childrenOf) {
|
|
9
|
+
return nodes.flatMap((node) => {
|
|
10
|
+
const children = childrenOf(node);
|
|
11
|
+
return children && children.length > 0 ? [node, ...flattenNodes(children, childrenOf)] : [node];
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Build an index mapping each `need` id to the (flat) nodes that declare it as a
|
|
16
|
+
* dependency. The input is expected to already be flattened by the caller so
|
|
17
|
+
* that the resulting index spans the whole graph.
|
|
18
|
+
*/
|
|
19
|
+
function dependentsByNeed(nodes) {
|
|
20
|
+
const index = /* @__PURE__ */ new Map();
|
|
21
|
+
for (const node of nodes) for (const need of node.needs ?? []) {
|
|
22
|
+
const dependents = index.get(need) ?? [];
|
|
23
|
+
dependents.push(node);
|
|
24
|
+
index.set(need, dependents);
|
|
25
|
+
}
|
|
26
|
+
return index;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Breadth-first search over the dependents index to determine whether any node
|
|
30
|
+
* transitively reachable downstream of `nodeId` satisfies `matches`. Cycle-safe
|
|
31
|
+
* via a visited set.
|
|
32
|
+
*/
|
|
33
|
+
function hasReachableDependent(nodeId, index, matches) {
|
|
34
|
+
const visited = /* @__PURE__ */ new Set();
|
|
35
|
+
const queue = [...index.get(nodeId) ?? []];
|
|
36
|
+
while (queue.length > 0) {
|
|
37
|
+
const node = queue.shift();
|
|
38
|
+
if (!node || visited.has(node.id)) continue;
|
|
39
|
+
visited.add(node.id);
|
|
40
|
+
if (matches(node)) return true;
|
|
41
|
+
queue.push(...index.get(node.id) ?? []);
|
|
42
|
+
}
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Recursively find a node by id within a node tree, descending into children
|
|
47
|
+
* via the caller-supplied accessor. Returns `undefined` when absent.
|
|
48
|
+
*/
|
|
49
|
+
function findNode(nodes, nodeId, childrenOf) {
|
|
50
|
+
for (const node of nodes) {
|
|
51
|
+
if (node.id === nodeId) return node;
|
|
52
|
+
const child = findNode(childrenOf(node) ?? [], nodeId, childrenOf);
|
|
53
|
+
if (child) return child;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Find all dependency cycles in a flat node list using an iterative DFS over the
|
|
58
|
+
* dependents-by-need graph. Each cycle is returned once (canonicalized by its
|
|
59
|
+
* member set) as the ordered list of node ids forming the loop. Only `needs`
|
|
60
|
+
* that reference declared nodes participate, so callers can pass the full flat
|
|
61
|
+
* node set without pre-filtering missing dependencies.
|
|
62
|
+
*
|
|
63
|
+
* The traversal is iterative (an explicit frame stack) rather than recursive so
|
|
64
|
+
* that deep generated workflow chains cannot overflow the call stack.
|
|
65
|
+
*/
|
|
66
|
+
function findDependencyCycles(nodes) {
|
|
67
|
+
const nodeIds = new Set(nodes.map((node) => node.id));
|
|
68
|
+
const index = dependentsByNeed(nodes);
|
|
69
|
+
const adjacency = /* @__PURE__ */ new Map();
|
|
70
|
+
for (const node of nodes) adjacency.set(node.id, (index.get(node.id) ?? []).map((dependent) => dependent.id).filter((id) => nodeIds.has(id)));
|
|
71
|
+
const state = /* @__PURE__ */ new Map();
|
|
72
|
+
const path = [];
|
|
73
|
+
const pathIndex = /* @__PURE__ */ new Map();
|
|
74
|
+
const cycles = [];
|
|
75
|
+
const cycleKeys = /* @__PURE__ */ new Set();
|
|
76
|
+
for (const node of nodes) {
|
|
77
|
+
if (state.has(node.id)) continue;
|
|
78
|
+
visitForCycles(node.id, {
|
|
79
|
+
adjacency,
|
|
80
|
+
cycleKeys,
|
|
81
|
+
cycles,
|
|
82
|
+
path,
|
|
83
|
+
pathIndex,
|
|
84
|
+
state
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
return cycles;
|
|
88
|
+
}
|
|
89
|
+
function visitForCycles(startId, visitState) {
|
|
90
|
+
const frames = [{
|
|
91
|
+
index: 0,
|
|
92
|
+
nodeId: startId
|
|
93
|
+
}];
|
|
94
|
+
markVisiting(startId, visitState);
|
|
95
|
+
while (frames.length > 0) {
|
|
96
|
+
const frame = frames.at(-1);
|
|
97
|
+
if (!frame) return;
|
|
98
|
+
const dependentId = (visitState.adjacency.get(frame.nodeId) ?? [])[frame.index];
|
|
99
|
+
if (!dependentId) {
|
|
100
|
+
markDone(frame.nodeId, visitState);
|
|
101
|
+
frames.pop();
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
frame.index += 1;
|
|
105
|
+
const dependentState = visitState.state.get(dependentId);
|
|
106
|
+
if (dependentState === "visiting") {
|
|
107
|
+
recordCycle(dependentId, visitState);
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (dependentState === "done") continue;
|
|
111
|
+
markVisiting(dependentId, visitState);
|
|
112
|
+
frames.push({
|
|
113
|
+
index: 0,
|
|
114
|
+
nodeId: dependentId
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
function markVisiting(nodeId, visitState) {
|
|
119
|
+
visitState.state.set(nodeId, "visiting");
|
|
120
|
+
visitState.pathIndex.set(nodeId, visitState.path.length);
|
|
121
|
+
visitState.path.push(nodeId);
|
|
122
|
+
}
|
|
123
|
+
function markDone(nodeId, visitState) {
|
|
124
|
+
visitState.state.set(nodeId, "done");
|
|
125
|
+
visitState.pathIndex.delete(nodeId);
|
|
126
|
+
visitState.path.pop();
|
|
127
|
+
}
|
|
128
|
+
function recordCycle(nodeId, visitState) {
|
|
129
|
+
const startIndex = visitState.pathIndex.get(nodeId);
|
|
130
|
+
if (startIndex === void 0) return;
|
|
131
|
+
const cycle = visitState.path.slice(startIndex);
|
|
132
|
+
const key = [...cycle].sort().join("\0");
|
|
133
|
+
if (visitState.cycleKeys.has(key)) return;
|
|
134
|
+
visitState.cycleKeys.add(key);
|
|
135
|
+
visitState.cycles.push(cycle);
|
|
136
|
+
}
|
|
137
|
+
//#endregion
|
|
138
|
+
export { dependentsByNeed, findDependencyCycles, findNode, flattenNodes, hasReachableDependent };
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import { parseRunnerCommandPayload, resolveRunnerEventSinkAuthToken } from "../runner-command-contract.js";
|
|
2
1
|
import { PIPELINE_CONFIG_PATH, PROFILES_CONFIG_PATH, RUNNERS_CONFIG_PATH } from "../config/defaults.js";
|
|
3
2
|
import { loadPipelineConfig, parsePipelineConfigParts } from "../config/load.js";
|
|
4
3
|
import "../config.js";
|
|
5
|
-
import { compileScheduleArtifact, parseScheduleArtifact } from "../
|
|
6
|
-
import "../
|
|
4
|
+
import { compileScheduleArtifact, parseScheduleArtifact } from "../planning/generate.js";
|
|
5
|
+
import { parseRunnerCommandPayload, resolveRunnerEventSinkAuthToken } from "../runner-command-contract.js";
|
|
7
6
|
import { prepareRunnerGitWorkspace } from "../run-state/git-refs.js";
|
|
8
7
|
import { createRunnerEventSink } from "../runner-event-sink.js";
|
|
9
8
|
import { initialNodeStateStore } from "../runtime/node-state-store.js";
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { readRunnerTaskDescriptor } from "./task-descriptor.js";
|
|
2
|
-
import { RunnerCommandPayloadValidationError, parseRunnerCommandPayload, resolveRunnerEventSinkAuthToken } from "../runner-command-contract.js";
|
|
3
2
|
import { loadPipelineConfig } from "../config/load.js";
|
|
4
3
|
import "../config.js";
|
|
5
|
-
import { compileScheduleArtifact, parseScheduleArtifact } from "../
|
|
6
|
-
import "../
|
|
4
|
+
import { compileScheduleArtifact, parseScheduleArtifact } from "../planning/generate.js";
|
|
5
|
+
import { RunnerCommandPayloadValidationError, parseRunnerCommandPayload, resolveRunnerEventSinkAuthToken } from "../runner-command-contract.js";
|
|
7
6
|
import { commitAndPushNodeRef, mergeDependencyRefs, prepareRunnerGitWorkspace } from "../run-state/git-refs.js";
|
|
8
7
|
import { createRunnerEventSink } from "../runner-event-sink.js";
|
|
9
8
|
import { findPlannedNode } from "../planned-node.js";
|
package/dist/runner.d.ts
CHANGED
|
@@ -2,6 +2,18 @@ import { PipelineConfig, RunnerType } from "./config/schemas.js";
|
|
|
2
2
|
//#region src/runner.d.ts
|
|
3
3
|
type Harness = "opencode";
|
|
4
4
|
type AgentRole = "researcher" | "test-writer" | "code-writer" | "verifier";
|
|
5
|
+
/**
|
|
6
|
+
* Agent-output boundary, layer 1 of 4 (PIPE-74 B3). `AgentResult` is the RAW
|
|
7
|
+
* terminal result of one runner subprocess/session: exit code, accumulated
|
|
8
|
+
* stdout/stderr, and execution metadata. It carries no parsing or semantic
|
|
9
|
+
* interpretation — downstream layers refine it:
|
|
10
|
+
* 1. {@link AgentResult} — raw subprocess result (this type)
|
|
11
|
+
* 2. {@link RunnerOutputEvent} — a live stream chunk during execution
|
|
12
|
+
* 3. RuntimeNormalizedOutput — adapter-extracted text + evidence
|
|
13
|
+
* (src/runtime/opencode-adapter.ts)
|
|
14
|
+
* 4. RuntimeStructuredOutput — parsed + schema-validated output
|
|
15
|
+
* (src/runtime/contracts/contracts.ts)
|
|
16
|
+
*/
|
|
5
17
|
interface AgentResult {
|
|
6
18
|
argv?: string[];
|
|
7
19
|
exitCode: number;
|
|
@@ -11,11 +23,26 @@ interface AgentResult {
|
|
|
11
23
|
stdout: string;
|
|
12
24
|
timedOut?: boolean;
|
|
13
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* Agent-output boundary, layer 2 of 4 (PIPE-74 B3). A single incremental chunk
|
|
28
|
+
* of a runner's live output stream, surfaced via
|
|
29
|
+
* {@link RunnerExecutionOptions.onOutput} while the subprocess is still
|
|
30
|
+
* running — distinct from {@link AgentResult}, which is the final accumulated
|
|
31
|
+
* result.
|
|
32
|
+
*/
|
|
14
33
|
interface RunnerOutputEvent {
|
|
15
34
|
chunk: string;
|
|
16
35
|
nodeId: string;
|
|
17
36
|
stream: "stderr" | "stdout";
|
|
18
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* Lowest layer of the runtime-options stack (PIPE-74 B3): the per-invocation
|
|
40
|
+
* controls a runner executor needs — cancellation and live-output streaming.
|
|
41
|
+
* Widened by the runtime layers above it:
|
|
42
|
+
* RunnerExecutionOptions (this type)
|
|
43
|
+
* < PipelineRuntimeOptions (src/runtime/contracts/contracts.ts)
|
|
44
|
+
* < ScheduledWorkflowTaskRuntimeOptions (src/pipeline-runtime.ts)
|
|
45
|
+
*/
|
|
19
46
|
interface RunnerExecutionOptions {
|
|
20
47
|
onOutput?: (event: RunnerOutputEvent) => void;
|
|
21
48
|
signal?: AbortSignal;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { loadPipelineConfig } from "../../config/load.js";
|
|
2
2
|
import "../../config.js";
|
|
3
3
|
import { runLaunchPlan } from "../../runner.js";
|
|
4
|
-
import { compileWorkflowPlan } from "../../
|
|
4
|
+
import { compileWorkflowPlan } from "../../planning/compile.js";
|
|
5
5
|
import { createPublicRuntimeObservabilityEmitter } from "../events/events.js";
|
|
6
6
|
import "../events/index.js";
|
|
7
7
|
import { initialNodeStateStore } from "../node-state-store.js";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { HookEvent, PipelineConfig } from "../../config/schemas.js";
|
|
2
|
-
import { PlannedWorkflowNode, WorkflowExecutionPlan } from "../../
|
|
2
|
+
import { PlannedWorkflowNode, WorkflowExecutionPlan } from "../../planning/compile.js";
|
|
3
3
|
import { HookResult } from "../../hooks.js";
|
|
4
4
|
import { AgentResult, RunnerExecutionOptions, RunnerLaunchPlan } from "../../runner.js";
|
|
5
5
|
import { RuntimeActorDescriptor, RuntimeObservabilityEvent } from "../actor-ids.js";
|
|
@@ -44,6 +44,13 @@ interface RuntimeNodeResult {
|
|
|
44
44
|
output: string;
|
|
45
45
|
status: "failed" | "passed";
|
|
46
46
|
}
|
|
47
|
+
/**
|
|
48
|
+
* Agent-output boundary, layer 4 of 4 (PIPE-74 B3). The terminal, structured
|
|
49
|
+
* form of agent output: the layer-3 RuntimeNormalizedOutput text
|
|
50
|
+
* (src/runtime/opencode-adapter.ts) parsed into a typed `output` value and
|
|
51
|
+
* validated against the node's declared schema. This is what downstream nodes
|
|
52
|
+
* and gates consume.
|
|
53
|
+
*/
|
|
47
54
|
interface RuntimeStructuredOutput {
|
|
48
55
|
attempt: number;
|
|
49
56
|
format: "json" | "json_schema" | "jsonl";
|
|
@@ -228,6 +235,14 @@ type PipelineRuntimeEvent = {
|
|
|
228
235
|
type: "workflow.finish";
|
|
229
236
|
workflowId: string;
|
|
230
237
|
});
|
|
238
|
+
/**
|
|
239
|
+
* Middle layer of the runtime-options stack (PIPE-74 B3): everything needed to
|
|
240
|
+
* run one pipeline workflow from config — the config/entrypoint to run, the
|
|
241
|
+
* task, reporting, and the executor (whose per-call controls come from the
|
|
242
|
+
* layer below, {@link RunnerExecutionOptions} in src/runner.ts). Extended by
|
|
243
|
+
* ScheduledWorkflowTaskRuntimeOptions (src/pipeline-runtime.ts) for the
|
|
244
|
+
* single-node, schedule-driven execution path.
|
|
245
|
+
*/
|
|
231
246
|
interface PipelineRuntimeOptions {
|
|
232
247
|
config?: PipelineConfig;
|
|
233
248
|
entrypoint?: string;
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
+
import { uniqueGeneratedId } from "../../strings.js";
|
|
2
|
+
import { dependentsByNeed, hasReachableDependent } from "../../planning/graph.js";
|
|
3
|
+
import { isCoverageNode, isImplementationNode } from "../scheduling-roles.js";
|
|
1
4
|
//#region src/schedule/passes/coverage.ts
|
|
2
5
|
const DEFAULT_GENERATED_COVERAGE_PROFILE_PREFERENCE = [
|
|
3
6
|
"moka-verifier",
|
|
4
7
|
"moka-acceptance-reviewer",
|
|
5
8
|
"moka-thermo-nuclear-reviewer"
|
|
6
9
|
];
|
|
7
|
-
const GENERATED_ID_INVALID_CHARS_RE = /[^a-z0-9]+/g;
|
|
8
|
-
const GENERATED_ID_TRIM_HYPHENS_RE = /^-+|-+$/g;
|
|
9
|
-
const STARTS_WITH_ALPHA_RE = /^[a-z]/;
|
|
10
10
|
function addGeneratedImplementationCoverage(config, artifact) {
|
|
11
11
|
const coverageProfileId = generatedCoverageProfileId(config);
|
|
12
12
|
if (!coverageProfileId) return artifact;
|
|
@@ -26,8 +26,8 @@ function addNodeScopeImplementationCoverage(config, nodes, coverageProfileId) {
|
|
|
26
26
|
...node,
|
|
27
27
|
nodes: addNodeScopeImplementationCoverage(config, node.nodes, coverageProfileId)
|
|
28
28
|
} : node);
|
|
29
|
-
const
|
|
30
|
-
const uncovered = scopedNodes.filter((node) => isImplementationNode(config, node)).filter((node) => !hasDownstreamCoverage(config, node.id,
|
|
29
|
+
const index = dependentsByNeed(scopedNodes);
|
|
30
|
+
const uncovered = scopedNodes.filter((node) => isImplementationNode(config, node)).filter((node) => !hasDownstreamCoverage(config, node.id, index));
|
|
31
31
|
if (uncovered.length === 0) return scopedNodes;
|
|
32
32
|
const coverageNodeId = uniqueGeneratedId("generated-coverage", new Set(scopedNodes.map((node) => node.id)), "generated-coverage");
|
|
33
33
|
return [...scopedNodes, {
|
|
@@ -82,52 +82,8 @@ function generatedCoverageGates(nodeId) {
|
|
|
82
82
|
}
|
|
83
83
|
];
|
|
84
84
|
}
|
|
85
|
-
function
|
|
86
|
-
|
|
87
|
-
for (const node of nodes) for (const need of node.needs ?? []) {
|
|
88
|
-
const dependents = dependentsByNeed.get(need) ?? [];
|
|
89
|
-
dependents.push(node);
|
|
90
|
-
dependentsByNeed.set(need, dependents);
|
|
91
|
-
}
|
|
92
|
-
return dependentsByNeed;
|
|
93
|
-
}
|
|
94
|
-
function isImplementationNode(config, node) {
|
|
95
|
-
return hasSchedulingRole(config, node, "implementation");
|
|
96
|
-
}
|
|
97
|
-
function hasDownstreamCoverage(config, nodeId, dependentsByNeed) {
|
|
98
|
-
return hasReachableDependent(nodeId, dependentsByNeed, (node) => hasSchedulingRole(config, node, "coverage"));
|
|
99
|
-
}
|
|
100
|
-
function hasSchedulingRole(config, node, role) {
|
|
101
|
-
if (node.kind !== "agent") return false;
|
|
102
|
-
return config.profiles[node.profile]?.scheduling_roles?.includes(role) ?? false;
|
|
103
|
-
}
|
|
104
|
-
function hasReachableDependent(nodeId, dependentsByNeed, matches) {
|
|
105
|
-
const visited = /* @__PURE__ */ new Set();
|
|
106
|
-
const queue = [...dependentsByNeed.get(nodeId) ?? []];
|
|
107
|
-
while (queue.length > 0) {
|
|
108
|
-
const node = queue.shift();
|
|
109
|
-
if (!node || visited.has(node.id)) continue;
|
|
110
|
-
visited.add(node.id);
|
|
111
|
-
if (matches(node)) return true;
|
|
112
|
-
queue.push(...dependentsByNeed.get(node.id) ?? []);
|
|
113
|
-
}
|
|
114
|
-
return false;
|
|
115
|
-
}
|
|
116
|
-
function uniqueGeneratedId(value, usedIds, fallbackPrefix) {
|
|
117
|
-
const base = generatedId(value, fallbackPrefix);
|
|
118
|
-
let candidate = base;
|
|
119
|
-
let suffix = 2;
|
|
120
|
-
while (usedIds.has(candidate)) {
|
|
121
|
-
candidate = `${base}-${suffix}`;
|
|
122
|
-
suffix += 1;
|
|
123
|
-
}
|
|
124
|
-
usedIds.add(candidate);
|
|
125
|
-
return candidate;
|
|
126
|
-
}
|
|
127
|
-
function generatedId(value, fallbackPrefix) {
|
|
128
|
-
const slug = value.trim().toLowerCase().replaceAll(GENERATED_ID_INVALID_CHARS_RE, "-").replaceAll(GENERATED_ID_TRIM_HYPHENS_RE, "");
|
|
129
|
-
if (STARTS_WITH_ALPHA_RE.test(slug)) return slug;
|
|
130
|
-
return slug ? `${fallbackPrefix}-${slug}` : fallbackPrefix;
|
|
85
|
+
function hasDownstreamCoverage(config, nodeId, index) {
|
|
86
|
+
return hasReachableDependent(nodeId, index, (node) => isCoverageNode(config, node));
|
|
131
87
|
}
|
|
132
88
|
//#endregion
|
|
133
89
|
export { addGeneratedImplementationCoverage };
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
+
import { uniqueGeneratedId } from "../../strings.js";
|
|
2
|
+
import { flattenNodes } from "../../planning/graph.js";
|
|
1
3
|
//#region src/schedule/passes/ids.ts
|
|
2
|
-
const GENERATED_ID_INVALID_CHARS_RE = /[^a-z0-9]+/g;
|
|
3
|
-
const GENERATED_ID_TRIM_HYPHENS_RE = /^-+|-+$/g;
|
|
4
|
-
const STARTS_WITH_ALPHA_RE = /^[a-z]/;
|
|
5
4
|
function canonicalizeGeneratedScheduleIds(artifact) {
|
|
6
5
|
return {
|
|
7
6
|
...artifact,
|
|
@@ -11,7 +10,7 @@ function canonicalizeGeneratedScheduleIds(artifact) {
|
|
|
11
10
|
function canonicalizeWorkflowNodeIds(workflow) {
|
|
12
11
|
const nodeIdMap = /* @__PURE__ */ new Map();
|
|
13
12
|
const usedNodeIds = /* @__PURE__ */ new Set();
|
|
14
|
-
for (const node of workflow.nodes
|
|
13
|
+
for (const node of flattenNodes(workflow.nodes, (node) => node.kind === "parallel" ? node.nodes : void 0)) nodeIdMap.set(node.id, uniqueGeneratedId(node.id, usedNodeIds, "node"));
|
|
15
14
|
return {
|
|
16
15
|
...workflow,
|
|
17
16
|
nodes: workflow.nodes.map((node) => rewriteGeneratedWorkflowNodeIds(node, nodeIdMap))
|
|
@@ -28,24 +27,5 @@ function rewriteGeneratedWorkflowNodeIds(node, nodeIdMap) {
|
|
|
28
27
|
nodes: rewritten.nodes.map((child) => rewriteGeneratedWorkflowNodeIds(child, nodeIdMap))
|
|
29
28
|
} : rewritten;
|
|
30
29
|
}
|
|
31
|
-
function uniqueGeneratedId(value, usedIds, fallbackPrefix) {
|
|
32
|
-
const base = generatedId(value, fallbackPrefix);
|
|
33
|
-
let candidate = base;
|
|
34
|
-
let suffix = 2;
|
|
35
|
-
while (usedIds.has(candidate)) {
|
|
36
|
-
candidate = `${base}-${suffix}`;
|
|
37
|
-
suffix += 1;
|
|
38
|
-
}
|
|
39
|
-
usedIds.add(candidate);
|
|
40
|
-
return candidate;
|
|
41
|
-
}
|
|
42
|
-
function generatedId(value, fallbackPrefix) {
|
|
43
|
-
const slug = value.trim().toLowerCase().replaceAll(GENERATED_ID_INVALID_CHARS_RE, "-").replaceAll(GENERATED_ID_TRIM_HYPHENS_RE, "");
|
|
44
|
-
if (STARTS_WITH_ALPHA_RE.test(slug)) return slug;
|
|
45
|
-
return slug ? `${fallbackPrefix}-${slug}` : fallbackPrefix;
|
|
46
|
-
}
|
|
47
|
-
function flattenWorkflowNode(node) {
|
|
48
|
-
return node.kind === "parallel" ? [node, ...node.nodes.flatMap(flattenWorkflowNode)] : [node];
|
|
49
|
-
}
|
|
50
30
|
//#endregion
|
|
51
31
|
export { canonicalizeGeneratedScheduleIds };
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
//#region src/schedule/scheduling-roles.ts
|
|
2
|
+
/**
|
|
3
|
+
* Whether an agent node's profile declares the given scheduling role. Non-agent
|
|
4
|
+
* nodes never carry scheduling roles. This is the single source of truth for the
|
|
5
|
+
* implementation/coverage role policy used by schedule generation and
|
|
6
|
+
* validation.
|
|
7
|
+
*/
|
|
8
|
+
function hasSchedulingRole(config, node, role) {
|
|
9
|
+
if (node.kind !== "agent") return false;
|
|
10
|
+
return config.profiles[node.profile]?.scheduling_roles?.includes(role) ?? false;
|
|
11
|
+
}
|
|
12
|
+
function isImplementationNode(config, node) {
|
|
13
|
+
return hasSchedulingRole(config, node, "implementation");
|
|
14
|
+
}
|
|
15
|
+
function isCoverageNode(config, node) {
|
|
16
|
+
return hasSchedulingRole(config, node, "coverage");
|
|
17
|
+
}
|
|
18
|
+
//#endregion
|
|
19
|
+
export { isCoverageNode, isImplementationNode };
|
package/dist/strings.js
CHANGED
|
@@ -4,5 +4,34 @@ function uniqueStrings(values, options = {}) {
|
|
|
4
4
|
const unique = [...new Set(input)];
|
|
5
5
|
return options.sort ? unique.sort() : unique;
|
|
6
6
|
}
|
|
7
|
+
const GENERATED_ID_INVALID_CHARS_RE = /[^a-z0-9]+/g;
|
|
8
|
+
const GENERATED_ID_TRIM_HYPHENS_RE = /^-+|-+$/g;
|
|
9
|
+
const STARTS_WITH_ALPHA_RE = /^[a-z]/;
|
|
10
|
+
/**
|
|
11
|
+
* Slugify an arbitrary string into a workflow-safe id (lowercase, hyphenated).
|
|
12
|
+
* When the slug does not begin with a letter it is prefixed with
|
|
13
|
+
* `fallbackPrefix` so the result is always a valid node/workflow id.
|
|
14
|
+
*/
|
|
15
|
+
function generatedId(value, fallbackPrefix) {
|
|
16
|
+
const slug = value.trim().toLowerCase().replaceAll(GENERATED_ID_INVALID_CHARS_RE, "-").replaceAll(GENERATED_ID_TRIM_HYPHENS_RE, "");
|
|
17
|
+
if (STARTS_WITH_ALPHA_RE.test(slug)) return slug;
|
|
18
|
+
return slug ? `${fallbackPrefix}-${slug}` : fallbackPrefix;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Slugify `value` to a generated id (see {@link generatedId}) and disambiguate
|
|
22
|
+
* against `usedIds` by appending an incrementing numeric suffix. Mutates
|
|
23
|
+
* `usedIds` to reserve the chosen id.
|
|
24
|
+
*/
|
|
25
|
+
function uniqueGeneratedId(value, usedIds, fallbackPrefix) {
|
|
26
|
+
const base = generatedId(value, fallbackPrefix);
|
|
27
|
+
let candidate = base;
|
|
28
|
+
let suffix = 2;
|
|
29
|
+
while (usedIds.has(candidate)) {
|
|
30
|
+
candidate = `${base}-${suffix}`;
|
|
31
|
+
suffix += 1;
|
|
32
|
+
}
|
|
33
|
+
usedIds.add(candidate);
|
|
34
|
+
return candidate;
|
|
35
|
+
}
|
|
7
36
|
//#endregion
|
|
8
|
-
export { uniqueStrings };
|
|
37
|
+
export { uniqueGeneratedId, uniqueStrings };
|
|
@@ -320,6 +320,38 @@ the execution surface while Claude Code orchestrates. Unsupported runner or host
|
|
|
320
320
|
mappings fail closed instead of doing instruction-only translation or generic
|
|
321
321
|
worker substitution.
|
|
322
322
|
|
|
323
|
+
## Planning: Compile And Generate
|
|
324
|
+
|
|
325
|
+
Two distinct planning strategies turn config into an executable plan, and the
|
|
326
|
+
module layout names the distinction (`src/planning/`):
|
|
327
|
+
|
|
328
|
+
- `planning/compile.ts` — **deterministic DAG compilation**. `compileWorkflowPlan`
|
|
329
|
+
validates a workflow's nodes (duplicate ids, missing dependencies, group
|
|
330
|
+
references, cycles), normalizes group dependencies, and produces the
|
|
331
|
+
`WorkflowExecutionPlan` (graph, topological order, parallel batches). This is
|
|
332
|
+
the engine's mandatory front door: every execution path runs through it,
|
|
333
|
+
including AI-generated schedules. Exposed as the `./planner` package subpath.
|
|
334
|
+
- `planning/generate.ts` — **optional AI decomposition**. `generateScheduleArtifact`
|
|
335
|
+
drives a planner profile to decompose a task into a schedule artifact, then
|
|
336
|
+
normalizes it through auditable passes (coverage → models → ids → references)
|
|
337
|
+
and finally feeds it back into `compileScheduleArtifact`, which merges the
|
|
338
|
+
generated workflows into config and calls `compileWorkflowPlan`. So generate
|
|
339
|
+
output is always an input to compile — never a substitute for it. Exposed as
|
|
340
|
+
the `./schedule` package subpath.
|
|
341
|
+
|
|
342
|
+
Both layers share one policy-free traversal model, `planning/graph.ts`
|
|
343
|
+
(`flattenNodes`, `dependentsByNeed`, `hasReachableDependent`, `findNode`,
|
|
344
|
+
`findDependencyCycles`). Each caller supplies its own predicates and owns its
|
|
345
|
+
error vocabulary, so config validation, deterministic compile, and AI-schedule
|
|
346
|
+
validation reason over the same graph structure without sharing layer-specific
|
|
347
|
+
policy. Schedule-specific support (artifact re-export barrel, baseline, prompts,
|
|
348
|
+
the normalization passes, backlog context, scheduling-role policy) stays under
|
|
349
|
+
`src/schedule/` and imports the artifact + generate API from `planning/generate`.
|
|
350
|
+
|
|
351
|
+
The toposort uses `@dagrejs/graphlib` for the graph model but an iterative
|
|
352
|
+
traversal for the topological sort, because graphlib's recursive topsort can
|
|
353
|
+
overflow the call stack on deep generated workflow chains.
|
|
354
|
+
|
|
323
355
|
## Troubleshooting
|
|
324
356
|
|
|
325
357
|
- Missing host resources: run `moka install-commands`; `moka run` loads the
|
package/docs/operator-guide.md
CHANGED
|
@@ -100,8 +100,8 @@ moka doctor --cluster momokaya-pipeline --kube-context momokaya
|
|
|
100
100
|
`moka doctor --cluster` adds value-free runner-job preflight checks for the
|
|
101
101
|
selected namespace, defaulting to `momokaya-pipeline`. It checks that expected
|
|
102
102
|
Secret objects exist by name, ExternalSecrets and `ClusterSecretStore/openbao`
|
|
103
|
-
report Ready status, the runner ServiceAccount has workflow RBAC,
|
|
104
|
-
|
|
103
|
+
report Ready status, the runner ServiceAccount has workflow RBAC, and Argo
|
|
104
|
+
Workflow prerequisites are reachable. It does
|
|
105
105
|
not read, print, decode, diff, or validate Secret values.
|
|
106
106
|
|
|
107
107
|
OpenBao and External Secrets Operator remain infrastructure-owned
|
|
@@ -160,7 +160,6 @@ momokaya:
|
|
|
160
160
|
githubAuthSecretName: <github-auth-secret-name>
|
|
161
161
|
imagePullSecretName: <image-pull-secret-name>
|
|
162
162
|
opencodeAuthSecretName: <opencode-auth-secret-name>
|
|
163
|
-
queueName: <local-queue-name>
|
|
164
163
|
serviceAccountName: <runner-service-account-name>
|
|
165
164
|
```
|
|
166
165
|
|
|
@@ -82,10 +82,9 @@ keep the console package dependency, console expected version, and runner image
|
|
|
82
82
|
label version aligned. A future breaking payload change must increment the
|
|
83
83
|
contract version and ship a compatibility plan.
|
|
84
84
|
|
|
85
|
-
Console-created Jobs are labeled with `
|
|
86
|
-
`pipeline.oisin.dev/
|
|
87
|
-
`pipeline.oisin.dev/
|
|
88
|
-
`pipeline.oisin.dev/requested-by`.
|
|
85
|
+
Console-created Jobs are labeled with `pipeline.oisin.dev/project`,
|
|
86
|
+
`pipeline.oisin.dev/run-id`, `pipeline.oisin.dev/source`,
|
|
87
|
+
`pipeline.oisin.dev/task`, plus optional `pipeline.oisin.dev/requested-by`.
|
|
89
88
|
|
|
90
89
|
## Event Batches
|
|
91
90
|
|