@oisincoveney/pipeline 1.28.1 → 1.29.0

package/README.md

@@ -223,7 +223,7 @@ single dependency target. Agents may route work to tracks, but the branch
 topology stays auditable in YAML or in the generated schedule artifact.
 
 See [`docs/config-architecture.md`](docs/config-architecture.md) for the config
-shape and [`docs/xstate-runtime-actor-model.md`](docs/xstate-runtime-actor-model.md)
+shape and [`docs/runtime-actor-model.md`](docs/runtime-actor-model.md)
 for the runtime actor model.
 
 ## Generated Host Resources

package/dist/argo-graph.js

@@ -1,3 +1,4 @@
+import { uniqueStrings } from "./strings.js";
 import { z } from "zod";
 //#region src/argo-graph.ts
 const EXECUTABLE_NODE_KINDS = [
@@ -63,14 +64,14 @@ var ArgoGraphCompiler = class {
 		}
 	}
 	resolveDependencyTaskNames(nodeIds) {
-		return unique(nodeIds.flatMap((nodeId) => this.resolveDependencyNodeIds(nodeId).map((id) => argoTaskName(id))));
+		return uniqueStrings(nodeIds.flatMap((nodeId) => this.resolveDependencyNodeIds(nodeId).map((id) => argoTaskName(id))));
 	}
 	resolveDependencyNodeIds(nodeId) {
 		const node = this.nodeById.get(nodeId);
 		if (!node) return [];
 		if (isExecutableNode(node)) return [node.id];
-		if (node.kind === "group") return unique([...node.nodes ?? [], ...node.needs].flatMap((id) => this.resolveDependencyNodeIds(id)));
-		if (node.kind === "parallel") return unique((node.children ?? []).flatMap((child) => this.resolveDependencyNodeIds(child.id)));
+		if (node.kind === "group") return uniqueStrings([...node.nodes ?? [], ...node.needs].flatMap((id) => this.resolveDependencyNodeIds(id)));
+		if (node.kind === "parallel") return uniqueStrings((node.children ?? []).flatMap((child) => this.resolveDependencyNodeIds(child.id)));
 		return [];
 	}
 	terminalTasks() {
@@ -87,8 +88,5 @@ function argoTaskName(nodeId) {
 function argoTemplateName(nodeId) {
 	return `task-${nodeId}`;
 }
-function unique(values) {
-	return [...new Set(values)];
-}
 //#endregion
 export { compileArgoExecutionGraph };

package/dist/argo-submit.d.ts

@@ -1,4 +1,4 @@
-import { PipelineConfig } from "./config.js";
+import { PipelineConfig } from "./config/schemas.js";
 import { workflowSubmitResultSchema } from "./workflow-submit-contract.js";
 import { z } from "zod";
 import { CoreV1Api, CustomObjectsApi, KubeConfig } from "@kubernetes/client-node";

package/dist/argo-submit.js

@@ -2,7 +2,8 @@ import { compileArgoExecutionGraph } from "./argo-graph.js";
 import { buildRunnerTaskDescriptor } from "./runner-command/task-descriptor.js";
 import { buildRunnerArgoWorkflowManifest, runnerArgoWorkflowManifestSchema } from "./argo-workflow.js";
 import { parseRunnerCommandPayload, runnerCommandPayloadSchema } from "./runner-command-contract.js";
-import { compileScheduleArtifact, parseScheduleArtifact } from "./schedule-planner.js";
+import { compileScheduleArtifact, parseScheduleArtifact } from "./schedule/planner.js";
+import "./schedule-planner.js";
 import { workflowSubmitResultSchema } from "./workflow-submit-contract.js";
 import { stringify } from "yaml";
 import { z } from "zod";

package/dist/argo-workflow.d.ts

@@ -81,6 +81,16 @@ declare const runnerArgoWorkflowManifestSchema: z.ZodObject<{
         }, z.core.$strict>>;
       }, z.core.$strict>>;
       name: z.ZodString;
+      retryStrategy: z.ZodOptional<z.ZodObject<{
+        expression: z.ZodOptional<z.ZodString>;
+        limit: z.ZodOptional<z.ZodString>;
+        retryPolicy: z.ZodEnum<{
+          Always: "Always";
+          OnError: "OnError";
+          OnFailure: "OnFailure";
+          OnTransientError: "OnTransientError";
+        }>;
+      }, z.core.$strict>>;
     }, z.core.$strict>>;
     ttlStrategy: z.ZodOptional<z.ZodObject<{
       secondsAfterCompletion: z.ZodOptional<z.ZodNumber>;

package/dist/argo-workflow.js

@@ -8,9 +8,15 @@ const ARGO_WORKFLOW_KIND = "Workflow";
 const RUNNER_WORKFLOW_IMAGE = "ghcr.io/oisin-ee/pipeline-runner:latest";
 const RUNNER_WORKFLOW_SERVICE_ACCOUNT = "pipeline-runner";
 const RUNNER_WORKFLOW_ENTRYPOINT = "pipeline";
+const RUNNER_WORKFLOW_START_TASK = "workflow-start";
 const RUNNER_WORKFLOW_PAYLOAD_PATH = "/etc/pipeline/payload.json";
 const RUNNER_WORKFLOW_SCHEDULE_PATH = "/etc/pipeline/schedule.yaml";
 const RUNNER_GIT_CREDENTIALS_PATH = "/etc/pipeline/git-credentials";
+const RUNNER_STARTUP_RETRY_STRATEGY = {
+	expression: "asInt(lastRetry.exitCode) == 70",
+	limit: "3",
+	retryPolicy: "OnFailure"
+};
 const RUNNER_OPENCODE_ENV = [{
 	name: "CODEX_AUTH_PER_PROJECT_ACCOUNTS",
 	value: "0"
@@ -54,6 +60,16 @@ const argoWorkflowArtifactSchema = z.object({
 	name: z.string().min(1),
 	path: z.string().min(1).optional()
 }).strict();
+const argoWorkflowRetryStrategySchema = z.object({
+	expression: z.string().min(1).optional(),
+	limit: z.string().min(1).optional(),
+	retryPolicy: z.enum([
+		"Always",
+		"OnError",
+		"OnFailure",
+		"OnTransientError"
+	])
+}).strict();
 const argoWorkflowTemplateSchema = z.object({
 	container: z.object({
 		args: z.array(z.string().min(1)).min(1),
@@ -83,7 +99,8 @@ const argoWorkflowTemplateSchema = z.object({
 		parameters: z.array(z.object({ name: z.string().min(1) }).strict()).optional()
 	}).strict().optional(),
 	outputs: z.object({ artifacts: z.array(argoWorkflowArtifactSchema) }).strict().optional(),
-	name: z.string().min(1)
+	name: z.string().min(1),
+	retryStrategy: argoWorkflowRetryStrategySchema.optional()
 }).strict().refine((template) => template.container !== void 0 || template.dag !== void 0, { message: "Workflow templates must declare container or dag" });
 const runnerArgoWorkflowManifestSchema = z.object({
 	apiVersion: z.literal(ARGO_WORKFLOW_API_VERSION),
@@ -179,13 +196,17 @@ function buildRunnerArgoWorkflowManifest(rawOptions) {
 			serviceAccountName: options.serviceAccountName,
 			templates: [
 				{
-					dag: { tasks: graph.tasks.map((task) => ({
-						...task.dependencies.length > 0 ? { dependencies: task.dependencies } : {},
+					dag: { tasks: [{
+						name: RUNNER_WORKFLOW_START_TASK,
+						template: RUNNER_WORKFLOW_START_TASK
+					}, ...graph.tasks.map((task) => ({
+						dependencies: [RUNNER_WORKFLOW_START_TASK, ...task.dependencies],
 						name: task.taskName,
 						template: task.templateName
-					})) },
+					}))] },
 					name: RUNNER_WORKFLOW_ENTRYPOINT
 				},
+				runnerLifecycleTemplate(options, volumeMounts),
 				...graph.tasks.map((task) => runnerCommandTemplate(task, options, volumeMounts)),
 				runnerFinalizerTemplate(options, volumeMounts)
 			],
@@ -333,6 +354,30 @@ function runnerWorkflowStorage(options, tasks) {
 		volumes: z.array(argoWorkflowVolumeSchema).parse(volumes)
 	};
 }
+function runnerLifecycleTemplate(options, volumeMounts) {
+	return {
+		container: {
+			args: [
+				"runner-lifecycle",
+				"--phase",
+				"workflow.start",
+				"--payload-file",
+				RUNNER_WORKFLOW_PAYLOAD_PATH,
+				"--schedule-file",
+				RUNNER_WORKFLOW_SCHEDULE_PATH
+			],
+			command: ["moka"],
+			env: [...RUNNER_OPENCODE_ENV],
+			image: options.image,
+			imagePullPolicy: options.imagePullPolicy,
+			name: "runner",
+			...options.resources ? { resources: options.resources } : {},
+			volumeMounts
+		},
+		name: RUNNER_WORKFLOW_START_TASK,
+		retryStrategy: { ...RUNNER_STARTUP_RETRY_STRATEGY }
+	};
+}
 function runnerCommandTemplate(task, options, volumeMounts) {
 	const taskVolumeMount = {
 		mountPath: DEFAULT_RUNNER_TASK_DESCRIPTOR_PATH,
@@ -357,7 +402,8 @@ function runnerCommandTemplate(task, options, volumeMounts) {
 			...options.resources ? { resources: options.resources } : {},
 			volumeMounts: [...volumeMounts, taskVolumeMount]
 		},
-		name: task.templateName
+		name: task.templateName,
+		retryStrategy: { ...RUNNER_STARTUP_RETRY_STRATEGY }
 	};
 }
 function runnerFinalizerTemplate(options, volumeMounts) {

package/dist/cli/format.js

@@ -0,0 +1,137 @@
+//#region src/cli/format.ts
+const LINE_RE = /\r?\n/;
+function createTerminalRuntimeReporter(write = (message) => console.log(message)) {
+	const state = { attempts: /* @__PURE__ */ new Map() };
+	return (event) => {
+		write(formatRuntimeProgressMessage(event, state));
+	};
+}
+function formatRuntimeProgressMessage(event, state = { attempts: /* @__PURE__ */ new Map() }) {
+	return formatWorkflowProgress(event, state) ?? formatAgentProgress(event, state) ?? formatCheckProgress(event, state) ?? formatObservabilityProgress(event) ?? formatRepairProgress(event);
+}
+function formatWorkflowProgress(event, state) {
+	switch (event.type) {
+		case "workflow.planned": return `Pipeline planned: ${event.workflowId} (${event.nodes.map((node) => node.id).join(" -> ")})`;
+		case "workflow.start": return `Pipeline starting: ${event.workflowId} (${event.nodeIds.join(" -> ")})`;
+		case "node.start":
+			state.attempts.set(event.nodeId, event.attempt);
+			return [
+				`Node starting: ${event.nodeId}`,
+				event.runnerId ? `runner=${event.runnerId}` : "",
+				event.profile ? `profile=${event.profile}` : "",
+				`attempt=${event.attempt}`
+			].filter(Boolean).join(" ");
+		case "node.finish":
+			state.attempts.set(event.nodeId, event.attempt);
+			return `Node finished: ${event.nodeId} ${event.status} exit=${event.exitCode}`;
+		case "node.output.recorded": return [
+			`Node output: ${event.nodeId}`,
+			`attempt=${event.attempt}`,
+			`format=${event.format}`,
+			formatRuntimeEventOutput(event.output)
+		].filter(Boolean).join(" ");
+		case "workflow.finish": return `Pipeline finished: ${event.workflowId} ${event.outcome}`;
+		default: return null;
+	}
+}
+function formatAgentProgress(event, state) {
+	switch (event.type) {
+		case "agent.start":
+			state.attempts.set(event.nodeId, event.attempt);
+			return `Agent starting: ${event.nodeId} runner=${event.runnerId ?? "unknown"} attempt=${event.attempt}`;
+		case "agent.finish":
+			state.attempts.set(event.nodeId, event.attempt);
+			return `Agent finished: ${event.nodeId} runner=${event.runnerId ?? "unknown"} exit=${event.exitCode}`;
+		case "hook.start": return `Hook starting: ${event.hookId} event=${event.event}${event.nodeId ? ` node=${event.nodeId}` : ""}`;
+		case "hook.finish": return `Hook ${event.passed ? "passed" : "failed"}: ${event.hookId}${event.reason ? ` (${event.reason})` : ""}`;
+		case "hook.result": return `Hook result: ${event.hookId} ${event.status}${event.summary ? ` (${event.summary})` : ""}`;
+		default: return null;
+	}
+}
+function formatCheckProgress(event, state) {
+	switch (event.type) {
+		case "gate.start": return `Gate starting: ${event.nodeId}/${event.gateId}${formatKnownAttempt(state, event.nodeId)}`;
+		case "gate.finish": return [
+			`Gate ${event.passed ? "passed" : "failed"}: ${event.nodeId}/${event.gateId}${formatKnownAttempt(state, event.nodeId)}`,
+			event.reason ? `reason=${event.reason}` : "",
+			...(event.evidence ?? []).map((item) => `evidence=${item}`)
+		].filter(Boolean).join(" ");
+		case "artifact.check.start": return `Artifact check starting: ${event.nodeId}/${event.path}`;
+		case "artifact.check.finish": return `Artifact check ${event.passed ? "passed" : "failed"}: ${event.nodeId}/${event.path}${event.reason ? ` (${event.reason})` : ""}`;
+		default: return null;
+	}
+}
+function formatKnownAttempt(state, nodeId) {
+	const attempt = state.attempts.get(nodeId);
+	return attempt === void 0 ? "" : ` attempt=${attempt}`;
+}
+function formatRuntimeEventOutput(output) {
+	if (typeof output === "string") return output.trimEnd();
+	return JSON.stringify(output);
+}
+function formatRepairProgress(event) {
+	switch (event.type) {
+		case "output.repair": return `Output repair ${event.passed ? "passed" : "failed"}: ${event.nodeId} attempt=${event.attempt}${event.reason ? ` (${event.reason})` : ""}`;
+		default: throw new Error(`Unhandled runtime event: ${event.type}`);
+	}
+}
+function formatObservabilityProgress(event) {
+	switch (event.type) {
+		case "runtime.observability": return `Runtime observed: ${event.name} - ${event.summary}`;
+		default: return null;
+	}
+}
+function formatRuntimeResult(result) {
+	const lines = [
+		`Pipeline complete: ${result.outcome}`,
+		`Workflow: ${result.plan.workflowId}`,
+		`Nodes: ${result.nodes.map((node) => `${node.nodeId}:${node.status}`).join(", ")}`,
+		`Agent boundaries: ${result.agentInvocations.length}`
+	];
+	const outputs = result.nodes.filter((node) => node.output.trim());
+	if (outputs.length > 0) {
+		lines.push("Node outputs:");
+		for (const node of outputs) appendIndentedSection(lines, node.nodeId, [node.output]);
+	}
+	return lines.join("\n");
+}
+function formatRuntimeFailure(result) {
+	const lines = ["Pipeline failed."];
+	for (const failure of result.failureDetails) {
+		lines.push(failure.nodeId ? `- ${failure.nodeId}: ${failure.reason}` : `- ${failure.reason}`);
+		appendIndentedSection(lines, "Evidence", failure.evidence);
+		const node = failure.nodeId ? result.nodes.find((item) => item.nodeId === failure.nodeId) : void 0;
+		if (node) {
+			lines.push(`  Node: status=${node.status} attempts=${node.attempts} exit=${node.exitCode}`);
+			appendIndentedSection(lines, "Node evidence", node.evidence);
+			appendIndentedSection(lines, "Node output", [node.output]);
+		}
+	}
+	if (result.gates.length > 0) {
+		lines.push("Gates:");
+		for (const gate of result.gates) {
+			lines.push(`  - ${gate.nodeId}/${gate.gateId}: ${gate.passed ? "PASS" : "FAIL"}${gate.reason ? ` (${gate.reason})` : ""}`);
+			appendIndentedSection(lines, "Gate evidence", gate.evidence);
+		}
+	}
+	return lines.join("\n");
+}
+function formatDoctorResult(result) {
+	return [`Doctor: ${result.passed ? "PASS" : "FAIL"}`, ...result.checks.map((check) => `- ${check.passed ? "PASS" : "FAIL"} ${check.name}: ${check.detail}`)].join("\n");
+}
+function appendIndentedSection(lines, label, values) {
+	const text = values.filter(Boolean).join("\n").trim();
+	if (!text) return;
+	lines.push(`  ${label}:`);
+	lines.push(indent(truncateMiddle(text, 4e3), "    "));
+}
+function indent(text, prefix) {
+	return text.split(LINE_RE).map((line) => `${prefix}${line}`).join("\n");
+}
+function truncateMiddle(text, maxLength) {
+	if (text.length <= maxLength) return text;
+	const keep = Math.floor((maxLength - 32) / 2);
+	return `${text.slice(0, keep)}\n... truncated ...\n${text.slice(-keep)}`;
+}
+//#endregion
+export { createTerminalRuntimeReporter, formatDoctorResult, formatRuntimeFailure, formatRuntimeResult };

package/dist/cli/program.d.ts

@@ -0,0 +1,34 @@
+import { runPipelineFromConfig } from "../pipeline-runtime.js";
+import { Command } from "commander";
+
+//#region src/cli/program.d.ts
+interface ExecuteOptions {
+  entrypoint?: string;
+  pipelineRunner?: typeof runPipelineFromConfig;
+  schedule?: string;
+  workflow?: string;
+}
+/**
+ * Config-driven `execute` entrypoint. Package-owned defaults are the source of
+ * truth; repo-local pipeline files are ignored by runtime loading.
+ */
+declare function execute(description: string, options?: ExecuteOptions): Promise<void>;
+declare function quick(description: string, options?: Omit<ExecuteOptions, "entrypoint">): Promise<void>;
+interface DoctorCheck {
+  detail: string;
+  name: string;
+  passed: boolean;
+}
+interface DoctorResult {
+  checks: DoctorCheck[];
+  passed: boolean;
+}
+interface DoctorFlags {
+  cluster?: boolean | string;
+  kubeContext?: string;
+}
+declare function createCliProgram(): Command;
+declare function runCli(argv: string[]): Promise<void>;
+declare function runDoctor(cwd: string, options?: DoctorFlags): Promise<DoctorResult>;
+//#endregion
+export { createCliProgram, execute, quick, runCli, runDoctor };