@oisincoveney/pipeline 1.23.2 → 1.24.0

package/README.md

@@ -116,20 +116,22 @@ target repository.
 ## Pipeline Console Runner Image
 
 `oisin-pipeline` is also the runner package/image used by `pipeline-console`.
-The console owns Kubernetes Job creation, run listing, cancellation, event
-storage, Kueue discovery, and UI rendering. This package owns the in-container
-`runner-job` command: payload validation, existing runtime invocation, event
-translation, authenticated event posting, signal cancellation, and final event
-flushing.
-
-The console starts the image with the payload as a mounted ConfigMap file and
-the event auth token as a mounted Secret file. The runner reads the payload
-from `--payload-file` and reads the event auth token from the file path
-specified in `events.authTokenFile`. No environment variables are used for
-payload or auth token delivery. The payload contract is documented in
+The control plane owns Argo Workflow submission, run listing, cancellation,
+event storage, Kueue discovery, and UI rendering. This package owns the
+in-container `runner-command` command used by Argo Workflow DAG tasks: payload
+validation, direct argv execution, event translation, authenticated event
+posting, signal cancellation, and final event flushing.
+
+Argo starts the image with payload, schedule, and per-task descriptor files
+mounted from ConfigMaps, plus the event auth token mounted from a Secret. The
+runner reads `--payload-file`, `--schedule-file`, and `/etc/pipeline/task.json`;
+the payload contains the workflow id and the task descriptor contains the node
+id. The event auth token still comes from the file path specified in
+`events.authTokenFile`; payload JSON, task identity, and auth token material are
+not delivered through environment variables. The payload contract is documented in
 [`docs/pipeline-console-runner-contract.md`](docs/pipeline-console-runner-contract.md).
 The executable contract is exported from
-`@oisincoveney/pipeline/runner-job-contract` for payload construction,
+`@oisincoveney/pipeline/runner-command-contract` for payload construction,
 validation, contract-version checks, and JSON Schema generation.
 Use `PIPELINE_TARGET_PATH=/path/to/worktree` when the checked-out target repo is
 mounted somewhere other than the process working directory.
@@ -225,25 +227,18 @@ workflow. See `docs/config-architecture.md` for the host support matrix.
 
 ### Structural Parallelism
 
-Workflows can compose other workflows with fixed YAML structure. A
-`kind: workflow` node invokes another named workflow; without `worktree_root` it
-runs in the current worktree, and with `worktree_root` it runs in an isolated
-git worktree. A `kind: parallel` node contains a fixed set of child nodes that
-run concurrently after dependencies pass. This is structural parallelism, not
-dynamic fanout: agents may route work to tracks, but the branch topology stays
-auditable in YAML.
+Workflows and generated schedules can express fixed parallel structure. A
+`kind: parallel` node contains a fixed set of child nodes that run concurrently
+after dependencies pass. A `kind: group` node groups existing nodes behind a
+single dependency target. This is structural parallelism, not dynamic fanout:
+agents may route work to tracks, but the branch topology stays auditable in YAML
+or in the generated schedule artifact.
 
-The built-in `epic` entrypoint uses those primitives:
+The scheduler emits explicit root DAGs for epics:
 
 ```yaml
-entrypoints:
-  epic:
-    workflow: epic-drain
-    description: Route an epic's tickets into specialist tracks, run them in parallel, then thermo-nuclear review.
-
 workflows:
-  epic-drain:
-    description: Research, route, parallel-implement tracks in isolated worktrees, integrate, thermo-nuclear review.
+  root:
     nodes:
       - id: research
         kind: agent
@@ -257,39 +252,26 @@ workflows:
         needs: [plan]
         nodes:
           - id: test
-            kind: workflow
-            workflow: default
-            worktree_root: .pipeline/runs/${runId}/test
+            kind: agent
+            profile: pipeline-code-writer
           - id: frontend
-            kind: workflow
-            workflow: default
-            worktree_root: .pipeline/runs/${runId}/frontend
+            kind: agent
+            profile: pipeline-code-writer
           - id: backend
-            kind: workflow
-            workflow: default
-            worktree_root: .pipeline/runs/${runId}/backend
+            kind: agent
+            profile: pipeline-code-writer
           - id: k8s
-            kind: workflow
-            workflow: infra
-            worktree_root: .pipeline/runs/${runId}/k8s
-      - id: merge
-        kind: builtin
-        builtin: drain-merge
-        needs: [implement]
-      - id: review
+            kind: agent
+            profile: pipeline-code-writer
+      - id: verify
         kind: agent
-        profile: pipeline-thermo-nuclear-reviewer
-        needs: [merge]
-        gates:
-          - { id: review-verdict, kind: verdict, target: stdout }
+        profile: pipeline-verifier
+        needs: [implement]
 ```
 
-Use `.pipeline/runs/${runId}/<track>` for isolated track worktrees; the default
-`.gitignore` excludes `.pipeline/runs/`. The `drain-merge` builtin consumes the
-parallel output, skips non-passing or non-worktree children, verifies mergeable
-branches share a base SHA, and merges passing branches into an integration
-branch in declaration order. It reports merge conflicts; it does not resolve
-them automatically.
+For Argo execution, Git refs carry node state across DAG tasks. Runner tasks
+derive state refs from `payload.run.id`, `payload.workflow.id`, and the mounted
+task descriptor node id.
 
 Default profile skills and generated host resources are installed by
 `pipe init`. Runtime MCP projection and host-specific isolation policy live in
@@ -367,12 +349,6 @@ runners:
 - Parallel DAG batches run concurrently after dependencies and gates pass.
 - `kind: parallel` child sets are fixed in YAML; routing agents decide which
   work belongs in each declared track, not how many tracks exist.
-- `kind: workflow` nodes invoke named workflows and can run in isolated
-  worktrees when `worktree_root` is set.
-- Worktree roots support `${runId}` and `${nodeId}` templates and should live
-  under `.pipeline/runs/` for generated run artifacts.
-- `drain-merge` merges passing worktree branches in declaration order and
-  reports conflicts for manual resolution.
 - Workflow execution can cap parallelism and enable fail-fast batch stopping.
 - Nodes can declare bounded retries, retry reasons, backoff, and execution
   timeouts.
@@ -418,11 +394,10 @@ Runner Job producers can import the shared payload contract:
 
 ```ts
 import {
-  RUNNER_JOB_CONTRACT_VERSION,
-  buildRunnerJobPayload,
-  parseRunnerJobPayload,
-  runnerJobPayloadJsonSchema,
-} from "@oisincoveney/pipeline/runner-job-contract";
+  buildRunnerCommandPayload,
+  parseRunnerCommandPayload,
+  runnerCommandPayloadSchema,
+} from "@oisincoveney/pipeline/runner-command-contract";
 ```
 
 ## Verification
@@ -437,4 +412,6 @@ bun run build:cli
 ```
 
 
-Runner Kubernetes Jobs can be generated with `buildRunnerJobK8sManifest` from `@oisincoveney/pipeline/runner-job-k8s`.
+Argo Workflows can be rendered with `buildRunnerArgoWorkflowManifest` from
+`@oisincoveney/pipeline/argo-workflow` and submitted with
+`submitRunnerArgoWorkflow` from `@oisincoveney/pipeline/argo-submit`.

package/dist/argo-graph.js

@@ -0,0 +1,94 @@
+import { z } from "zod";
+//#region src/argo-graph.ts
+const EXECUTABLE_NODE_KINDS = [
+	"agent",
+	"builtin",
+	"command"
+];
+const argoExecutableTaskSchema = z.object({
+	dependencies: z.array(z.string().min(1)),
+	nodeId: z.string().min(1),
+	taskName: z.string().min(1),
+	templateName: z.string().min(1)
+}).strict();
+const argoExecutionGraphSchema = z.object({
+	terminalNodeIds: z.array(z.string().min(1)),
+	tasks: z.array(argoExecutableTaskSchema).min(1),
+	terminalTaskNames: z.array(z.string().min(1)),
+	workflowId: z.string().min(1)
+}).strict();
+function compileArgoExecutionGraph(plan) {
+	const compiler = new ArgoGraphCompiler(plan);
+	return argoExecutionGraphSchema.parse(compiler.compile());
+}
+var ArgoGraphCompiler = class {
+	nodeById = /* @__PURE__ */ new Map();
+	plan;
+	tasks = [];
+	constructor(plan) {
+		this.plan = plan;
+		this.indexNodes(plan.topologicalOrder);
+	}
+	compile() {
+		this.compileNodes(this.plan.topologicalOrder, []);
+		return {
+			terminalNodeIds: this.terminalTasks().map((task) => task.nodeId),
+			tasks: this.tasks,
+			terminalTaskNames: this.terminalTasks().map((task) => task.taskName),
+			workflowId: this.plan.workflowId
+		};
+	}
+	indexNodes(nodes) {
+		for (const node of nodes) {
+			if (this.nodeById.has(node.id)) throw new Error(`Argo schedule contains duplicate node id '${node.id}'`);
+			this.nodeById.set(node.id, node);
+			this.indexNodes(node.children ?? []);
+		}
+	}
+	compileNodes(nodes, inheritedNeeds) {
+		for (const node of nodes) {
+			if (isExecutableNode(node)) {
+				const dependencies = this.resolveDependencyTaskNames([...inheritedNeeds, ...node.needs]);
+				const task = argoExecutableTaskSchema.parse({
+					dependencies,
+					nodeId: node.id,
+					taskName: argoTaskName(node.id),
+					templateName: argoTemplateName(node.id)
+				});
+				this.tasks.push(task);
+				continue;
+			}
+			if (node.kind === "group") continue;
+			if (node.kind === "parallel") this.compileNodes(node.children ?? [], [...inheritedNeeds, ...node.needs]);
+		}
+	}
+	resolveDependencyTaskNames(nodeIds) {
+		return unique(nodeIds.flatMap((nodeId) => this.resolveDependencyNodeIds(nodeId).map((id) => argoTaskName(id))));
+	}
+	resolveDependencyNodeIds(nodeId) {
+		const node = this.nodeById.get(nodeId);
+		if (!node) return [];
+		if (isExecutableNode(node)) return [node.id];
+		if (node.kind === "group") return unique([...node.nodes ?? [], ...node.needs].flatMap((id) => this.resolveDependencyNodeIds(id)));
+		if (node.kind === "parallel") return unique((node.children ?? []).flatMap((child) => this.resolveDependencyNodeIds(child.id)));
+		return [];
+	}
+	terminalTasks() {
+		const dependedOn = new Set(this.tasks.flatMap((task) => task.dependencies));
+		return this.tasks.filter((task) => !dependedOn.has(task.taskName));
+	}
+};
+function isExecutableNode(node) {
+	return EXECUTABLE_NODE_KINDS.includes(node.kind);
+}
+function argoTaskName(nodeId) {
+	return `node-${nodeId}`;
+}
+function argoTemplateName(nodeId) {
+	return `task-${nodeId}`;
+}
+function unique(values) {
+	return [...new Set(values)];
+}
+//#endregion
+export { compileArgoExecutionGraph };

package/dist/argo-submit.d.ts

@@ -0,0 +1,61 @@
+import { PipelineConfig } from "./config.js";
+import { z } from "zod";
+import { CoreV1Api, CustomObjectsApi, KubeConfig } from "@kubernetes/client-node";
+
+//#region src/argo-submit.d.ts
+declare const submitRunnerArgoWorkflowOptionsSchema: z.ZodObject<{
+  codexAuthSecretName: z.ZodOptional<z.ZodString>;
+  eventAuthSecretKey: z.ZodOptional<z.ZodString>;
+  eventAuthSecretName: z.ZodOptional<z.ZodString>;
+  generateName: z.ZodOptional<z.ZodString>;
+  githubAuthSecretName: z.ZodOptional<z.ZodString>;
+  image: z.ZodOptional<z.ZodString>;
+  imagePullPolicy: z.ZodOptional<z.ZodEnum<{
+    Always: "Always";
+    IfNotPresent: "IfNotPresent";
+    Never: "Never";
+  }>>;
+  imagePullSecretName: z.ZodOptional<z.ZodString>;
+  kubeconfigPath: z.ZodOptional<z.ZodString>;
+  name: z.ZodOptional<z.ZodString>;
+  namespace: z.ZodDefault<z.ZodString>;
+  opencodeAuthSecretName: z.ZodOptional<z.ZodString>;
+  orchestrator: z.ZodDefault<z.ZodEnum<{
+    codex: "codex";
+    opencode: "opencode";
+  }>>;
+  payloadJson: z.ZodString;
+  queueName: z.ZodOptional<z.ZodString>;
+  scheduleYaml: z.ZodString;
+  serviceAccountName: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+declare const submitRunnerArgoWorkflowResultSchema: z.ZodObject<{
+  namespace: z.ZodString;
+  payloadConfigMapName: z.ZodString;
+  scheduleConfigMapName: z.ZodString;
+  taskDescriptorConfigMapName: z.ZodString;
+  workflowName: z.ZodString;
+  workflowUid: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+declare const commandScheduleOptionsSchema: z.ZodObject<{
+  command: z.ZodArray<z.ZodString>;
+  generatedAt: z.ZodDefault<z.ZodDate>;
+  scheduleId: z.ZodOptional<z.ZodString>;
+  task: z.ZodString;
+}, z.core.$strict>;
+type SubmitRunnerArgoWorkflowOptions = z.input<typeof submitRunnerArgoWorkflowOptionsSchema> & {
+  config: PipelineConfig;
+};
+type SubmitRunnerArgoWorkflowResult = z.infer<typeof submitRunnerArgoWorkflowResultSchema>;
+type CommandScheduleOptions = z.input<typeof commandScheduleOptionsSchema>;
+type CoreApi = Pick<CoreV1Api, "createNamespacedConfigMap">;
+type WorkflowApi = Pick<CustomObjectsApi, "createNamespacedCustomObject" | "getClusterCustomObject">;
+interface SubmitRunnerArgoWorkflowDependencies {
+  coreApi?: CoreApi;
+  kubeConfig?: KubeConfig;
+  workflowApi?: WorkflowApi;
+}
+declare function submitRunnerArgoWorkflow(rawOptions: SubmitRunnerArgoWorkflowOptions, dependencies?: SubmitRunnerArgoWorkflowDependencies): Promise<SubmitRunnerArgoWorkflowResult>;
+declare function buildCommandScheduleYaml(rawOptions: CommandScheduleOptions): string;
+//#endregion
+export { CommandScheduleOptions, SubmitRunnerArgoWorkflowDependencies, SubmitRunnerArgoWorkflowOptions, SubmitRunnerArgoWorkflowResult, buildCommandScheduleYaml, submitRunnerArgoWorkflow };

package/dist/argo-submit.js

@@ -0,0 +1,206 @@
+import { compileArgoExecutionGraph } from "./argo-graph.js";
+import { buildRunnerTaskDescriptor } from "./runner-command/task-descriptor.js";
+import { buildRunnerArgoWorkflowManifest, runnerArgoWorkflowManifestSchema } from "./argo-workflow.js";
+import { parseRunnerCommandPayload, runnerCommandPayloadSchema } from "./runner-command-contract.js";
+import { compileScheduleArtifact, parseScheduleArtifact } from "./schedule-planner.js";
+import { stringify } from "yaml";
+import { z } from "zod";
+import { randomBytes } from "node:crypto";
+import { CoreV1Api, CustomObjectsApi, KubeConfig } from "@kubernetes/client-node";
+//#region src/argo-submit.ts
+const scheduleIdSchema = z.string().regex(/^[a-z][a-z0-9-]*$/);
+const configMapSchema = z.object({
+	apiVersion: z.literal("v1"),
+	data: z.record(z.string().min(1), z.string()),
+	kind: z.literal("ConfigMap"),
+	metadata: z.object({
+		labels: z.record(z.string().min(1), z.string().min(1)).optional(),
+		name: z.string().min(1),
+		namespace: z.string().min(1)
+	}).strict()
+}).strict();
+const submitRunnerArgoWorkflowOptionsSchema = z.object({
+	codexAuthSecretName: z.string().min(1).optional(),
+	eventAuthSecretKey: z.string().min(1).optional(),
+	eventAuthSecretName: z.string().min(1).optional(),
+	generateName: z.string().min(1).optional(),
+	githubAuthSecretName: z.string().min(1).optional(),
+	image: z.string().min(1).optional(),
+	imagePullPolicy: z.enum([
+		"Always",
+		"IfNotPresent",
+		"Never"
+	]).optional(),
+	imagePullSecretName: z.string().min(1).optional(),
+	kubeconfigPath: z.string().min(1).optional(),
+	name: z.string().min(1).optional(),
+	namespace: z.string().min(1).default("momokaya-pipeline"),
+	opencodeAuthSecretName: z.string().min(1).optional(),
+	orchestrator: z.enum(["codex", "opencode"]).default("opencode"),
+	payloadJson: z.string().min(1),
+	queueName: z.string().min(1).optional(),
+	scheduleYaml: z.string().min(1),
+	serviceAccountName: z.string().min(1).optional()
+}).strict().refine((options) => options.name !== void 0 || options.generateName !== void 0, { message: "Argo submit options must declare name or generateName" });
+const submitRunnerArgoWorkflowResultSchema = z.object({
+	namespace: z.string().min(1),
+	payloadConfigMapName: z.string().min(1),
+	scheduleConfigMapName: z.string().min(1),
+	taskDescriptorConfigMapName: z.string().min(1),
+	workflowName: z.string().min(1),
+	workflowUid: z.string().min(1).optional()
+}).strict();
+const commandScheduleOptionsSchema = z.object({
+	command: z.array(z.string().min(1)).min(1),
+	generatedAt: z.date().default(() => /* @__PURE__ */ new Date()),
+	scheduleId: scheduleIdSchema.optional(),
+	task: z.string().min(1)
+}).strict();
+async function submitRunnerArgoWorkflow(rawOptions, dependencies = {}) {
+	const { config, ...schemaOptions } = rawOptions;
+	const options = submitRunnerArgoWorkflowOptionsSchema.parse(schemaOptions);
+	const payload = runnerCommandPayloadSchema.parse(parseRunnerCommandPayload(options.payloadJson));
+	const compiled = compileScheduleArtifact(config, parseScheduleArtifact(options.scheduleYaml, "schedule.yaml"));
+	const payloadConfigMapName = `pipeline-payload-${randomBytes(6).toString("hex")}`;
+	const scheduleArtifactConfigMapName = `pipeline-schedule-${randomBytes(6).toString("hex")}`;
+	const taskDescriptorConfigMapName = `pipeline-task-descriptors-${randomBytes(6).toString("hex")}`;
+	if (payload.workflow.id !== compiled.workflowId) throw new Error(`Runner payload workflow '${payload.workflow.id}' does not match schedule workflow '${compiled.workflowId}'`);
+	const graph = compileArgoExecutionGraph(compiled.plan);
+	const labels = {
+		"pipeline.oisin.dev/project": payload.run.project,
+		"pipeline.oisin.dev/run-id": payload.run.id,
+		"pipeline.oisin.dev/source": "argo-workflow",
+		"pipeline.oisin.dev/workflow": compiled.workflowId
+	};
+	const workflow = buildRunnerArgoWorkflowManifest({
+		codexAuthSecretName: options.codexAuthSecretName,
+		eventAuthSecretKey: options.eventAuthSecretKey,
+		eventAuthSecretName: options.eventAuthSecretName,
+		generateName: options.generateName,
+		githubAuthSecretName: options.githubAuthSecretName,
+		image: options.image,
+		imagePullPolicy: options.imagePullPolicy,
+		imagePullSecretName: options.imagePullSecretName,
+		labels,
+		name: options.name,
+		namespace: options.namespace,
+		opencodeAuthSecretName: options.opencodeAuthSecretName,
+		orchestrator: options.orchestrator,
+		payloadConfigMapName,
+		plan: compiled.plan,
+		queueName: options.queueName,
+		scheduleConfigMapName: scheduleArtifactConfigMapName,
+		serviceAccountName: options.serviceAccountName,
+		taskDescriptorConfigMapName
+	});
+	const { coreApi, workflowApi } = apiClients(options, dependencies);
+	await assertArgoWorkflowCrdAvailable(workflowApi);
+	await coreApi.createNamespacedConfigMap({
+		body: configMapSchema.parse({
+			apiVersion: "v1",
+			data: { "payload.json": options.payloadJson },
+			kind: "ConfigMap",
+			metadata: {
+				labels,
+				name: payloadConfigMapName,
+				namespace: options.namespace
+			}
+		}),
+		namespace: options.namespace
+	});
+	await coreApi.createNamespacedConfigMap({
+		body: configMapSchema.parse({
+			apiVersion: "v1",
+			data: Object.fromEntries(graph.tasks.map((task) => [`${task.taskName}.json`, `${JSON.stringify(buildRunnerTaskDescriptor(task.nodeId))}\n`])),
+			kind: "ConfigMap",
+			metadata: {
+				labels,
+				name: taskDescriptorConfigMapName,
+				namespace: options.namespace
+			}
+		}),
+		namespace: options.namespace
+	});
+	await coreApi.createNamespacedConfigMap({
+		body: configMapSchema.parse({
+			apiVersion: "v1",
+			data: { "schedule.yaml": options.scheduleYaml },
+			kind: "ConfigMap",
+			metadata: {
+				labels,
+				name: scheduleArtifactConfigMapName,
+				namespace: options.namespace
+			}
+		}),
+		namespace: options.namespace
+	});
+	const response = await workflowApi.createNamespacedCustomObject({
+		body: runnerArgoWorkflowManifestSchema.parse(workflow),
+		group: "argoproj.io",
+		namespace: options.namespace,
+		plural: "workflows",
+		version: "v1alpha1"
+	});
+	const created = z.object({ metadata: z.object({
+		name: z.string().min(1).optional(),
+		uid: z.string().min(1).optional()
+	}).passthrough() }).passthrough().parse(response);
+	return submitRunnerArgoWorkflowResultSchema.parse({
+		namespace: options.namespace,
+		payloadConfigMapName,
+		scheduleConfigMapName: scheduleArtifactConfigMapName,
+		taskDescriptorConfigMapName,
+		workflowName: created.metadata.name ?? workflow.metadata.name,
+		workflowUid: created.metadata.uid
+	});
+}
+async function assertArgoWorkflowCrdAvailable(workflowApi) {
+	try {
+		await workflowApi.getClusterCustomObject({
+			group: "apiextensions.k8s.io",
+			name: "workflows.argoproj.io",
+			plural: "customresourcedefinitions",
+			version: "v1"
+		});
+	} catch (error) {
+		if (isKubernetesNotFound(error)) throw new Error("Argo Workflows is not installed in the target cluster: missing CustomResourceDefinition workflows.argoproj.io");
+		throw error;
+	}
+}
+function isKubernetesNotFound(error) {
+	if (!(error instanceof Error)) return false;
+	return "code" in error && error.code === 404 || error.message.includes("404") || error.message.includes("NotFound");
+}
+function buildCommandScheduleYaml(rawOptions) {
+	const options = commandScheduleOptionsSchema.parse(rawOptions);
+	const scheduleId = options.scheduleId ?? `custom-${randomBytes(8).toString("hex")}`;
+	return stringify({
+		generated_at: options.generatedAt.toISOString(),
+		kind: "pipeline-schedule",
+		root_workflow: "root",
+		schedule_id: scheduleId,
+		source_entrypoint: "custom",
+		task: options.task,
+		version: 1,
+		workflows: { root: { nodes: [{
+			command: options.command,
+			id: "command",
+			kind: "command"
+		}] } }
+	});
+}
+function apiClients(options, dependencies) {
+	if (dependencies.coreApi && dependencies.workflowApi) return {
+		coreApi: dependencies.coreApi,
+		workflowApi: dependencies.workflowApi
+	};
+	const kubeConfig = dependencies.kubeConfig ?? new KubeConfig();
+	if (!dependencies.kubeConfig) if (options.kubeconfigPath) kubeConfig.loadFromFile(options.kubeconfigPath);
+	else kubeConfig.loadFromDefault();
+	return {
+		coreApi: dependencies.coreApi ?? kubeConfig.makeApiClient(CoreV1Api),
+		workflowApi: dependencies.workflowApi ?? kubeConfig.makeApiClient(CustomObjectsApi)
+	};
+}
+//#endregion
+export { buildCommandScheduleYaml, submitRunnerArgoWorkflow };