@oisincoveney/pipeline 3.18.0 → 3.19.1

package/README.md

@@ -1,8 +1,9 @@
 # @oisincoveney/pipeline
 
 Config-driven multi-agent pipeline runner for repository work. The installed
-package owns the runtime defaults; target repositories use `.pipeline/runs/` for
-generated schedules and run artifacts, not as the source of runtime config.
+package owns the runtime defaults. Default Moka runs persist generated
+schedules, run manifests, events, node status, and node results in the configured
+Moka Postgres database, not in the target repository.
 
 The published command is `moka`.
 
@@ -94,8 +95,8 @@ moka explain-plan
 `moka run "<task>"` is the primary command surface.
 
 It runs package-owned workflow config from the current worktree. Scheduled
-entrypoints generate a schedule artifact under `.pipeline/runs/<runId>/` and run
-the compiled schedule through the runtime.
+entrypoints generate schedule YAML in memory, persist it on the DB-owned run
+manifest, and run the compiled schedule through the runtime.
 
 Canonical commands:
 
@@ -114,7 +115,7 @@ Canonical commands:
 ```shell
 moka run "Implement PIPE-123 user-facing behavior"
 moka run --target local --effort normal "Implement a standard local change"
-moka run --schedule .pipeline/runs/<runId>/schedule.yaml "Implement PIPE-123"
+moka run --schedule ./approved-schedule.yaml "Implement PIPE-123"
 moka run --workflow inspect "Report the app structure and available checks. Do not modify files."
 moka run --effort quick "Implement a focused fix"
 moka run --effort normal "Implement a standard fix"
@@ -153,18 +154,24 @@ Read-only ticket commands are `moka ticket graph check`, `moka ticket sequence`,
 `moka ticket start` without `--dry-run`: they use Backlog CLI task creation and
 editing or invoke `moka run` for the selected ticket.
 
-Local run artifacts live under `.pipeline/runs/<runId>/`:
+Local run-control state lives in the Moka DB selected by `momokaya.db.url` in
+`~/.config/moka/config.yaml`. The run manifest stores the generated schedule as
+`manifest.schedule`; node results are stored by `(runId, nodeId)` in the durable
+node record table. Use the run-control commands to inspect or export this state:
 
-```text
-.pipeline/runs/<runId>/
-  schedule.yaml
-  manifest.json
-  status.json
-  events.ndjson
-  nodes/<node-id>/
-  artifacts/
+```shell
+moka runs
+moka status <run-id>
+moka logs <run-id> [node-id]
+moka export <run-id> --sanitize
 ```
 
+`moka next node <run-id>` reads the persisted `manifest.schedule` from the Moka
+DB and emits the next ready node envelope. `moka submit-result <run-id>
+<node-id> --json '<RuntimeNodeResult>'` writes the node result back to the same
+DB state. `moka resume <run-id>` reconstructs the graph from the persisted
+schedule; it does not need a repo-local generated schedule file.
+
 Use `moka export <run-id> --sanitize` before sharing a run. The sanitized export
 keeps portable evidence and omits prompt text, session body content, secrets,
 tokens, and credentials.

package/defaults/pipeline.yaml

@@ -100,22 +100,22 @@ scheduler:
         backlog-intake:
           category: intake
           profile: moka-researcher
-          models: [openai/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
+          models: [broker/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
           reasoning_effort: medium
         red-tests:
           category: red
           profile: moka-test-writer
-          models: [openai/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
+          models: [broker/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
           reasoning_effort: high
         green-implementation:
           category: green
           profile: moka-code-writer
-          models: [opencode-go/qwen3.7-max, openai/gpt-5.5, kimi-for-coding/k2p6]
+          models: [opencode-go/qwen3.7-max, broker/gpt-5.5, kimi-for-coding/k2p6]
           reasoning_effort: high
         verification:
           category: verification
           profile: moka-verifier
-          models: [openai/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
+          models: [broker/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
           reasoning_effort: medium
     execute:
       required_categories: [intake, research, red, green, mechanical, acceptance, verification, learn]
@@ -123,42 +123,42 @@ scheduler:
         backlog-intake:
           category: intake
           profile: moka-researcher
-          models: [openai/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
+          models: [broker/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
           reasoning_effort: medium
         research:
           category: research
           profile: moka-researcher
-          models: [openai/gpt-5.5, kimi-for-coding/k2p6]
+          models: [broker/gpt-5.5, kimi-for-coding/k2p6]
           reasoning_effort: medium
         red-tests:
           category: red
           profile: moka-test-writer
-          models: [openai/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
+          models: [broker/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
           reasoning_effort: high
         green-backend:
           category: green
           profile: moka-code-writer
-          models: [opencode-go/qwen3.7-max, openai/gpt-5.5, kimi-for-coding/k2p6]
+          models: [opencode-go/qwen3.7-max, broker/gpt-5.5, kimi-for-coding/k2p6]
           reasoning_effort: high
         green-frontend:
           category: green
           profile: moka-code-writer
-          models: [openai/gpt-5.5, kimi-for-coding/k2p6, opencode-go/qwen3.7-max]
+          models: [broker/gpt-5.5, kimi-for-coding/k2p6, opencode-go/qwen3.7-max]
           reasoning_effort: high
         acceptance-review:
           category: acceptance
           profile: moka-acceptance-reviewer
-          models: [openai/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
+          models: [broker/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
           reasoning_effort: medium
         verification:
           category: verification
           profile: moka-verifier
-          models: [openai/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
+          models: [broker/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
           reasoning_effort: medium
         learn:
           category: learn
           profile: moka-learner
-          models: [openai/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
+          models: [broker/gpt-5.5, kimi-for-coding/kimi-k2-thinking]
           reasoning_effort: low
 schedules:
   quick-schedule:

package/defaults/profiles.yaml

@@ -90,7 +90,7 @@ skills:
 profiles:
   moka-orchestrator:
     runner: opencode
-    model: openai/gpt-5.5
+    model: broker/gpt-5.5
     reasoning_effort: xhigh
     description: Orchestrate the configured pipeline and enforce gates.
     instructions: { inline: "Orchestrate through the canonical local `moka run` supervisor. For compatibility slash commands, run the `moka run` command and flags shown in the command body. Treat execution as CLI/supervised runtime, not OpenCode-native Task execution. Enforce only package-configured gates." }
@@ -129,7 +129,7 @@ profiles:
       repair: { enabled: true, max_attempts: 1 }
   moka-inspector:
     runner: opencode
-    model: openai/gpt-5.5
+    model: broker/gpt-5.5
     reasoning_effort: low
     description: Inspect the repository without modifying files.
     instructions: { inline: "Inspect the repository without modifying files." }
@@ -140,7 +140,7 @@ profiles:
     network: { mode: inherit }
   moka-schedule-planner:
     runner: opencode
-    model: openai/gpt-5.5
+    model: broker/gpt-5.5
     reasoning_effort: xhigh
     description: Refine a baseline schedule into a specialized approved-plan artifact.
     instructions: { inline: "Generate exactly one workflow named root as an explicit schedule graph. Return YAML only." }
@@ -224,7 +224,7 @@ profiles:
       repair: { enabled: true, max_attempts: 1 }
   moka-learner:
     runner: opencode
-    model: openai/gpt-5.5
+    model: broker/gpt-5.5
     reasoning_effort: low
     description: Store durable lessons from the completed run.
     instructions: { inline: "Store durable lessons from the completed run when useful." }

package/dist/argo-submit.d.ts

@@ -34,8 +34,41 @@ declare const submitRunnerArgoWorkflowOptionsSchema: z.ZodObject<{
   name: z.ZodOptional<z.ZodString>;
   namespace: z.ZodString;
   payloadJson: z.ZodString;
+  serviceAccountName: z.ZodOptional<z.ZodString>;
   scheduleYaml: z.ZodString;
+}, z.core.$strict>;
+declare const submitDynamicRunnerArgoWorkflowOptionsSchema: z.ZodObject<{
+  brokerAuth: z.ZodObject<{
+    secretKey: z.ZodDefault<z.ZodString>;
+    secretName: z.ZodString;
+    url: z.ZodDefault<z.ZodString>;
+  }, z.core.$strict>;
+  dbAuth: z.ZodOptional<z.ZodObject<{
+    secretKey: z.ZodDefault<z.ZodString>;
+    secretName: z.ZodString;
+  }, z.core.$strict>>;
+  eventAuthSecretKey: z.ZodOptional<z.ZodString>;
+  eventAuthSecretName: z.ZodOptional<z.ZodString>;
+  generateName: z.ZodOptional<z.ZodString>;
+  gitCredentialsSecretName: z.ZodOptional<z.ZodString>;
+  githubAuthSecretName: z.ZodOptional<z.ZodString>;
+  image: z.ZodOptional<z.ZodString>;
+  imagePullPolicy: z.ZodOptional<z.ZodEnum<{
+    Always: "Always";
+    IfNotPresent: "IfNotPresent";
+    Never: "Never";
+  }>>;
+  imagePullSecretName: z.ZodOptional<z.ZodString>;
+  kubeconfigPath: z.ZodOptional<z.ZodString>;
+  mcpGatewayAuth: z.ZodOptional<z.ZodObject<{
+    secretKey: z.ZodDefault<z.ZodString>;
+    secretName: z.ZodString;
+  }, z.core.$strict>>;
+  name: z.ZodOptional<z.ZodString>;
+  namespace: z.ZodString;
+  payloadJson: z.ZodString;
   serviceAccountName: z.ZodOptional<z.ZodString>;
+  workflowId: z.ZodString;
 }, z.core.$strict>;
 declare const commandScheduleOptionsSchema: z.ZodObject<{
   command: z.ZodArray<z.ZodString>;
@@ -53,7 +86,11 @@ interface SubmitRunnerArgoWorkflowDependencies {
   kubeConfig?: KubernetesArgoIoDependencies["kubeConfig"];
   workflowApi?: WorkflowApi;
 }
+type SubmitDynamicRunnerArgoWorkflowOptions = z.input<typeof submitDynamicRunnerArgoWorkflowOptionsSchema> & {
+  config: PipelineConfig;
+};
 declare function submitRunnerArgoWorkflow(rawOptions: SubmitRunnerArgoWorkflowOptions, dependencies?: SubmitRunnerArgoWorkflowDependencies): Promise<SubmitRunnerArgoWorkflowResult>;
+declare function submitDynamicRunnerArgoWorkflow(rawOptions: SubmitDynamicRunnerArgoWorkflowOptions, dependencies?: SubmitRunnerArgoWorkflowDependencies): Promise<SubmitRunnerArgoWorkflowResult>;
 declare function buildCommandScheduleYaml(rawOptions: CommandScheduleOptions): string;
 //#endregion
-export { CommandScheduleOptions, SubmitRunnerArgoWorkflowDependencies, SubmitRunnerArgoWorkflowOptions, SubmitRunnerArgoWorkflowResult, buildCommandScheduleYaml, submitRunnerArgoWorkflow };
+export { CommandScheduleOptions, SubmitDynamicRunnerArgoWorkflowOptions, SubmitRunnerArgoWorkflowDependencies, SubmitRunnerArgoWorkflowOptions, SubmitRunnerArgoWorkflowResult, buildCommandScheduleYaml, submitDynamicRunnerArgoWorkflow, submitRunnerArgoWorkflow };

package/dist/argo-submit.js

@@ -4,7 +4,7 @@ import { brokerAuthOptionSchema } from "./credentials/broker.js";
 import { compileScheduleArtifact, parseScheduleArtifact } from "./planning/generate.js";
 import { parseRunnerCommandPayload, runnerCommandPayloadSchema } from "./runner-command-contract.js";
 import { buildRunnerTaskDescriptor } from "./runner-command/task-descriptor.js";
-import { buildRunnerArgoWorkflowManifest, runnerArgoWorkflowManifestSchema } from "./argo-workflow.js";
+import { buildDynamicRunnerArgoWorkflowManifest, buildRunnerArgoWorkflowManifest, runnerArgoWorkflowManifestSchema } from "./argo-workflow.js";
 import { normalizeRunnerRepositoryForSubmit } from "./git-remote-url.js";
 import { KubernetesArgoService, KubernetesArgoServiceLive } from "./runtime/services/kubernetes-argo-service.js";
 import { workflowSubmitResultSchema } from "./workflow-submit-contract.js";
@@ -24,7 +24,11 @@ const configMapSchema = z.object({
 		namespace: z.string().min(1)
 	}).strict()
 }).strict();
-const submitRunnerArgoWorkflowOptionsSchema = z.object({
+const createdWorkflowSchema = z.object({ metadata: z.object({
+	name: z.string().min(1).optional(),
+	uid: z.string().min(1).optional()
+}).passthrough() }).passthrough();
+const submitRunnerArgoWorkflowBaseOptionShape = {
 	brokerAuth: brokerAuthOptionSchema,
 	dbAuth: dbAuthOptionSchema.optional(),
 	eventAuthSecretKey: z.string().min(1).optional(),
@@ -44,18 +48,31 @@ const submitRunnerArgoWorkflowOptionsSchema = z.object({
 	name: z.string().min(1).optional(),
 	namespace: z.string().min(1),
 	payloadJson: z.string().min(1),
-	scheduleYaml: z.string().min(1),
 	serviceAccountName: z.string().min(1).optional()
-}).strict().refine((options) => options.name !== void 0 || options.generateName !== void 0, { message: "Argo submit options must declare name or generateName" });
+};
+const submitRunnerArgoWorkflowOptionsSchema = z.object({
+	...submitRunnerArgoWorkflowBaseOptionShape,
+	scheduleYaml: z.string().min(1)
+}).strict().refine(hasWorkflowName, { message: "Argo submit options must declare name or generateName" });
+const submitDynamicRunnerArgoWorkflowOptionsSchema = z.object({
+	...submitRunnerArgoWorkflowBaseOptionShape,
+	workflowId: z.string().min(1)
+}).strict().refine(hasWorkflowName, { message: "Argo submit options must declare name or generateName" });
 const commandScheduleOptionsSchema = z.object({
 	command: z.array(z.string().min(1)).min(1),
 	generatedAt: z.date().default(() => /* @__PURE__ */ new Date()),
 	scheduleId: scheduleIdSchema.optional(),
 	task: z.string().min(1)
 }).strict();
+function hasWorkflowName(options) {
+	return options.name !== void 0 || options.generateName !== void 0;
+}
 function submitRunnerArgoWorkflow(rawOptions, dependencies = {}) {
 	return Effect.runPromise(Effect.provide(Effect.suspend(() => submitRunnerArgoWorkflowEffect(rawOptions, dependencies)), KubernetesArgoServiceLive));
 }
+function submitDynamicRunnerArgoWorkflow(rawOptions, dependencies = {}) {
+	return Effect.runPromise(Effect.provide(Effect.suspend(() => submitDynamicRunnerArgoWorkflowEffect(rawOptions, dependencies)), KubernetesArgoServiceLive));
+}
 function submitRunnerArgoWorkflowEffect(rawOptions, dependencies) {
 	const { config, ...schemaOptions } = rawOptions;
 	const options = submitRunnerArgoWorkflowOptionsSchema.parse(schemaOptions);
@@ -149,26 +166,94 @@ function submitRunnerArgoWorkflowEffect(rawOptions, dependencies) {
 			namespace: options.namespace,
 			options
 		});
-		const response = yield* service.createWorkflow({
+		return workflowSubmitResult(yield* service.createWorkflow({
 			body: runnerArgoWorkflowManifestSchema.parse(workflow),
 			dependencies,
 			namespace: options.namespace,
 			options
-		});
-		const created = z.object({ metadata: z.object({
-			name: z.string().min(1).optional(),
-			uid: z.string().min(1).optional()
-		}).passthrough() }).passthrough().parse(response);
-		return workflowSubmitResultSchema.parse({
+		}), workflow, {
 			namespace: options.namespace,
 			payloadConfigMapName,
 			scheduleConfigMapName: scheduleArtifactConfigMapName,
-			taskDescriptorConfigMapName,
-			workflowName: created.metadata.name ?? workflow.metadata.name,
-			workflowUid: created.metadata.uid
+			taskDescriptorConfigMapName
 		});
 	});
 }
+function submitDynamicRunnerArgoWorkflowEffect(rawOptions, dependencies) {
+	const { config: _config, ...schemaOptions } = rawOptions;
+	const options = submitDynamicRunnerArgoWorkflowOptionsSchema.parse(schemaOptions);
+	const { payload, payloadJson } = normalizeRunnerPayloadForSubmit({
+		payload: runnerCommandPayloadSchema.parse(parseRunnerCommandPayload(options.payloadJson)),
+		payloadJson: options.payloadJson
+	});
+	if (payload.workflow.id !== options.workflowId) throw new Error(`Runner payload workflow '${payload.workflow.id}' does not match dynamic workflow '${options.workflowId}'`);
+	const payloadConfigMapName = `pipeline-payload-${randomBytes(6).toString("hex")}`;
+	const labels = {
+		"pipeline.oisin.dev/project": payload.run.project,
+		"pipeline.oisin.dev/run-id": payload.run.id,
+		"pipeline.oisin.dev/source": "argo-workflow",
+		"pipeline.oisin.dev/workflow": options.workflowId
+	};
+	const workflow = buildDynamicRunnerArgoWorkflowManifest({
+		annotations: payload.task.kind === "ticket" ? {
+			"pipeline.oisin.dev/ticket-id": payload.task.id,
+			"pipeline.oisin.dev/ticket-project": payload.run.project,
+			"pipeline.oisin.dev/ticket-title": payload.task.title
+		} : {},
+		brokerAuth: options.brokerAuth,
+		dbAuth: options.dbAuth,
+		eventAuthSecretKey: options.eventAuthSecretKey,
+		eventAuthSecretName: options.eventAuthSecretName,
+		generateName: options.generateName,
+		gitCredentialsSecretName: options.gitCredentialsSecretName,
+		githubAuthSecretName: options.githubAuthSecretName,
+		image: options.image,
+		imagePullPolicy: options.imagePullPolicy,
+		imagePullSecretName: options.imagePullSecretName,
+		mcpGatewayAuth: options.mcpGatewayAuth,
+		labels,
+		name: options.name,
+		namespace: options.namespace,
+		payloadConfigMapName,
+		serviceAccountName: options.serviceAccountName,
+		workflowId: options.workflowId
+	});
+	return Effect.gen(function* () {
+		const service = yield* KubernetesArgoService;
+		yield* service.createConfigMap({
+			body: configMapSchema.parse({
+				apiVersion: "v1",
+				data: { "payload.json": payloadJson },
+				kind: "ConfigMap",
+				metadata: {
+					labels,
+					name: payloadConfigMapName,
+					namespace: options.namespace
+				}
+			}),
+			dependencies,
+			namespace: options.namespace,
+			options
+		});
+		return workflowSubmitResult(yield* service.createWorkflow({
+			body: runnerArgoWorkflowManifestSchema.parse(workflow),
+			dependencies,
+			namespace: options.namespace,
+			options
+		}), workflow, {
+			namespace: options.namespace,
+			payloadConfigMapName
+		});
+	});
+}
+function workflowSubmitResult(response, workflow, base) {
+	const created = createdWorkflowSchema.parse(response);
+	return workflowSubmitResultSchema.parse({
+		...base,
+		workflowName: created.metadata.name ?? workflow.metadata.name,
+		workflowUid: created.metadata.uid
+	});
+}
 function buildCommandScheduleYaml(rawOptions) {
 	const options = commandScheduleOptionsSchema.parse(rawOptions);
 	const scheduleId = options.scheduleId ?? `custom-${randomBytes(8).toString("hex")}`;
@@ -209,4 +294,4 @@ function compileSubmitArgoGraph(compiled) {
 	});
 }
 //#endregion
-export { buildCommandScheduleYaml, submitRunnerArgoWorkflow };
+export { buildCommandScheduleYaml, submitDynamicRunnerArgoWorkflow, submitRunnerArgoWorkflow };

package/dist/argo-workflow.d.ts

@@ -1,5 +1,5 @@
 import { WorkflowExecutionPlan } from "./planning/compile.js";
-import { buildRunnerArgoWorkflowOptionsSchema } from "./remote/argo/model.js";
+import { buildDynamicRunnerArgoWorkflowOptionsSchema, buildRunnerArgoWorkflowOptionsSchema } from "./remote/argo/model.js";
 import { z } from "zod";
 
 //#region src/argo-workflow.d.ts
@@ -66,10 +66,16 @@ declare const runnerArgoWorkflowManifestSchema: z.ZodObject<{
               name: z.ZodString;
               path: z.ZodOptional<z.ZodString>;
             }, z.core.$strict>>>;
+            parameters: z.ZodOptional<z.ZodArray<z.ZodObject<{
+              name: z.ZodString;
+              value: z.ZodString;
+            }, z.core.$strict>>>;
           }, z.core.$strict>>;
           dependencies: z.ZodOptional<z.ZodArray<z.ZodString>>;
           name: z.ZodString;
           template: z.ZodString;
+          when: z.ZodOptional<z.ZodString>;
+          withParam: z.ZodOptional<z.ZodString>;
         }, z.core.$strict>>;
       }, z.core.$strict>>;
       inputs: z.ZodOptional<z.ZodObject<{
@@ -83,12 +89,30 @@ declare const runnerArgoWorkflowManifestSchema: z.ZodObject<{
         }, z.core.$strict>>>;
       }, z.core.$strict>>;
       outputs: z.ZodOptional<z.ZodObject<{
-        artifacts: z.ZodArray<z.ZodObject<{
+        artifacts: z.ZodOptional<z.ZodArray<z.ZodObject<{
           from: z.ZodOptional<z.ZodString>;
           name: z.ZodString;
           path: z.ZodOptional<z.ZodString>;
-        }, z.core.$strict>>;
+        }, z.core.$strict>>>;
+        parameters: z.ZodOptional<z.ZodArray<z.ZodObject<{
+          name: z.ZodString;
+          valueFrom: z.ZodObject<{
+            path: z.ZodString;
+          }, z.core.$strict>;
+        }, z.core.$strict>>>;
       }, z.core.$strict>>;
+      steps: z.ZodOptional<z.ZodArray<z.ZodArray<z.ZodObject<{
+        arguments: z.ZodOptional<z.ZodObject<{
+          parameters: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            value: z.ZodString;
+          }, z.core.$strict>>>;
+        }, z.core.$strict>>;
+        name: z.ZodString;
+        template: z.ZodString;
+        when: z.ZodOptional<z.ZodString>;
+        withParam: z.ZodOptional<z.ZodString>;
+      }, z.core.$strict>>>>;
       activeDeadlineSeconds: z.ZodOptional<z.ZodNumber>;
       name: z.ZodString;
       retryStrategy: z.ZodOptional<z.ZodObject<{
@@ -132,7 +156,9 @@ type ArgoWorkflowManifest = z.infer<typeof runnerArgoWorkflowManifestSchema>;
 type BuildRunnerArgoWorkflowOptions = z.input<typeof buildRunnerArgoWorkflowOptionsSchema> & {
   plan: WorkflowExecutionPlan;
 };
+type BuildDynamicRunnerArgoWorkflowOptions = z.input<typeof buildDynamicRunnerArgoWorkflowOptionsSchema>;
 declare function buildRunnerArgoWorkflowManifest(rawOptions: BuildRunnerArgoWorkflowOptions): ArgoWorkflowManifest;
+declare function buildDynamicRunnerArgoWorkflowManifest(rawOptions: BuildDynamicRunnerArgoWorkflowOptions): ArgoWorkflowManifest;
 declare function stringifyRunnerArgoWorkflow(workflow: ArgoWorkflowManifest): string;
 //#endregion
-export { ArgoWorkflowManifest, BuildRunnerArgoWorkflowOptions, buildRunnerArgoWorkflowManifest, runnerArgoWorkflowManifestSchema, stringifyRunnerArgoWorkflow };
+export { ArgoWorkflowManifest, BuildDynamicRunnerArgoWorkflowOptions, BuildRunnerArgoWorkflowOptions, buildDynamicRunnerArgoWorkflowManifest, buildRunnerArgoWorkflowManifest, runnerArgoWorkflowManifestSchema, stringifyRunnerArgoWorkflow };

package/dist/argo-workflow.js

@@ -1,8 +1,8 @@
 import { compileArgoExecutionGraph } from "./argo-graph.js";
 import { RUNNER_WORKFLOW_ENTRYPOINT, RUNNER_WORKFLOW_START_TASK } from "./remote/argo/policy.js";
-import { buildRunnerArgoWorkflowOptionsSchema, createRunnerArgoWorkflowManifestSchema } from "./remote/argo/model.js";
-import { runnerWorkflowStorage } from "./remote/argo/storage.js";
-import { runnerCommandTemplate, runnerFinalizerTemplate, runnerLifecycleTemplate } from "./remote/argo/templates.js";
+import { buildDynamicRunnerArgoWorkflowOptionsSchema, buildRunnerArgoWorkflowOptionsSchema, createRunnerArgoWorkflowManifestSchema } from "./remote/argo/model.js";
+import { dynamicRunnerWorkflowStorage, runnerWorkflowStorage } from "./remote/argo/storage.js";
+import { READY_NODE_IDS_PARAMETER, dynamicPreScheduleTemplate, dynamicReadyWaveSelectorTemplate, dynamicRunnerCommandTemplate, dynamicRunnerFinalizerTemplate, runnerCommandTemplate, runnerFinalizerTemplate, runnerLifecycleTemplate } from "./remote/argo/templates.js";
 import { stringify } from "yaml";
 //#region src/argo-workflow.ts
 const runnerArgoWorkflowManifestSchema = createRunnerArgoWorkflowManifestSchema();
@@ -17,6 +17,16 @@ function buildRunnerArgoWorkflowManifest(rawOptions) {
 		spec: workflowSpec(options, graph.tasks, storage)
 	});
 }
+function buildDynamicRunnerArgoWorkflowManifest(rawOptions) {
+	const options = buildDynamicRunnerArgoWorkflowOptionsSchema.parse(rawOptions);
+	const storage = dynamicRunnerWorkflowStorage(options);
+	return runnerArgoWorkflowManifestSchema.parse({
+		apiVersion: "argoproj.io/v1alpha1",
+		kind: "Workflow",
+		metadata: dynamicWorkflowMetadata(options),
+		spec: dynamicWorkflowSpec(options, storage)
+	});
+}
 function stringifyRunnerArgoWorkflow(workflow) {
 	return stringify(runnerArgoWorkflowManifestSchema.parse(workflow));
 }
@@ -40,6 +50,19 @@ function workflowMetadata(options) {
 		namespace: options.namespace
 	};
 }
+function dynamicWorkflowMetadata(options) {
+	return {
+		annotations: compactRecord(options.annotations),
+		...options.name ? { name: options.name } : {},
+		...options.generateName ? { generateName: options.generateName } : {},
+		labels: compactRecord({
+			"pipeline.oisin.dev/source": "argo-workflow",
+			"pipeline.oisin.dev/workflow": options.workflowId,
+			...options.labels
+		}),
+		namespace: options.namespace
+	};
+}
 function workflowSpec(options, tasks, storage) {
 	return {
 		...options.activeDeadlineSeconds ? { activeDeadlineSeconds: options.activeDeadlineSeconds } : {},
@@ -52,6 +75,18 @@ function workflowSpec(options, tasks, storage) {
 		volumes: storage.volumes
 	};
 }
+function dynamicWorkflowSpec(options, storage) {
+	return {
+		...options.activeDeadlineSeconds ? { activeDeadlineSeconds: options.activeDeadlineSeconds } : {},
+		entrypoint: RUNNER_WORKFLOW_ENTRYPOINT,
+		...options.imagePullSecretName ? { imagePullSecrets: [{ name: options.imagePullSecretName }] } : {},
+		serviceAccountName: options.serviceAccountName,
+		onExit: "pipeline-finalizer",
+		templates: dynamicWorkflowTemplates(options, storage.volumeMounts),
+		...options.ttlStrategy ? { ttlStrategy: options.ttlStrategy } : {},
+		volumes: storage.volumes
+	};
+}
 function workflowTemplates(options, tasks, volumeMounts) {
 	return [
 		workflowDagTemplate(tasks),
@@ -60,6 +95,18 @@ function workflowTemplates(options, tasks, volumeMounts) {
 		runnerFinalizerTemplate(options, volumeMounts)
 	];
 }
+function dynamicWorkflowTemplates(options, volumeMounts) {
+	return [
+		dynamicWorkflowEntrypointTemplate(),
+		dynamicDrainTemplate(),
+		dynamicPreScheduleTemplate("pre-research", options, volumeMounts),
+		dynamicPreScheduleTemplate("pre-planning", options, volumeMounts),
+		dynamicPreScheduleTemplate("generate-schedule", options, volumeMounts),
+		dynamicReadyWaveSelectorTemplate(options, volumeMounts),
+		dynamicRunnerCommandTemplate(options, volumeMounts),
+		dynamicRunnerFinalizerTemplate(options, volumeMounts)
+	];
+}
 function workflowDagTemplate(tasks) {
 	return {
 		dag: { tasks: [{
@@ -73,8 +120,58 @@ function workflowDagTemplate(tasks) {
 		name: RUNNER_WORKFLOW_ENTRYPOINT
 	};
 }
+function dynamicWorkflowEntrypointTemplate() {
+	return {
+		name: RUNNER_WORKFLOW_ENTRYPOINT,
+		steps: [
+			[{
+				name: "pre-research",
+				template: "pre-research"
+			}],
+			[{
+				name: "pre-planning",
+				template: "pre-planning"
+			}],
+			[{
+				name: "generate-schedule",
+				template: "generate-schedule"
+			}],
+			[{
+				name: "drain-ready-waves",
+				template: "drain-ready-waves"
+			}]
+		]
+	};
+}
+function dynamicDrainTemplate() {
+	const readyExpression = `{{steps.select-ready-wave.outputs.parameters.${READY_NODE_IDS_PARAMETER}}} != []`;
+	return {
+		name: "drain-ready-waves",
+		steps: [
+			[{
+				name: "select-ready-wave",
+				template: "select-ready-wave"
+			}],
+			[{
+				arguments: { parameters: [{
+					name: "node-id",
+					value: "{{item}}"
+				}] },
+				name: "run-ready-node",
+				template: "runner-command",
+				when: readyExpression,
+				withParam: `{{steps.select-ready-wave.outputs.parameters.${READY_NODE_IDS_PARAMETER}}}`
+			}],
+			[{
+				name: "drain-next-wave",
+				template: "drain-ready-waves",
+				when: readyExpression
+			}]
+		]
+	};
+}
 function compactRecord(input) {
 	return Object.fromEntries(Object.entries(input).filter((entry) => entry[1] !== void 0));
 }
 //#endregion
-export { buildRunnerArgoWorkflowManifest, runnerArgoWorkflowManifestSchema, stringifyRunnerArgoWorkflow };
+export { buildDynamicRunnerArgoWorkflowManifest, buildRunnerArgoWorkflowManifest, runnerArgoWorkflowManifestSchema, stringifyRunnerArgoWorkflow };

package/dist/commands/runner-command-command.js

@@ -1,6 +1,8 @@
 import { runRunnerCommand } from "../runner-command/run.js";
 import { runRunnerFinalize } from "../runner-command/finalize.js";
 import { runRunnerLifecycle } from "../runner-command/lifecycle.js";
+import { runPreSchedulePhase } from "../runner-command/pre-schedule.js";
+import { runSelectReadyWave } from "../runner-command/select-ready-wave.js";
 import { Context, Effect, Layer } from "effect";
 //#region src/commands/runner-command-command.ts
 var RunnerCommandService = class extends Context.Service()("RunnerCommandService") {};
@@ -13,9 +15,17 @@ const RunnerCommandServiceLive = Layer.succeed(RunnerCommandService, {
 		catch: (error) => error,
 		try: () => runRunnerLifecycle(options)
 	}),
+	preSchedule: (options) => Effect.tryPromise({
+		catch: (error) => error,
+		try: () => runPreSchedulePhase(options)
+	}),
 	run: (options) => Effect.tryPromise({
 		catch: (error) => error,
 		try: () => runRunnerCommand(options)
+	}),
+	selectReadyWave: (options) => Effect.tryPromise({
+		catch: (error) => error,
+		try: () => runSelectReadyWave(options)
 	})
 });
 const setProcessExitCode = (exitCode) => Effect.sync(() => {
@@ -30,11 +40,19 @@ const runRunnerLifecycleEffect = (options) => Effect.gen(function* () {
 const runRunnerFinalizeEffect = (options) => Effect.gen(function* () {
 	yield* setProcessExitCode(yield* (yield* RunnerCommandService).finalize(options));
 });
+const runPreScheduleEffect = (options) => Effect.gen(function* () {
+	yield* setProcessExitCode(yield* (yield* RunnerCommandService).preSchedule(options));
+});
+const runSelectReadyWaveEffect = (options) => Effect.gen(function* () {
+	yield* setProcessExitCode(yield* (yield* RunnerCommandService).selectReadyWave(options));
+});
 const runRunnerProgram = (program) => Effect.runPromise(Effect.provide(program, RunnerCommandServiceLive));
 function registerRunnerCommandCommand(program) {
-	program.command("runner-command").description("Run one scheduled Argo Workflow task").requiredOption("--payload-file <path>", "Path to the runner payload JSON").requiredOption("--schedule-file <path>", "Path to the schedule artifact YAML").action((options) => runRunnerProgram(runRunnerCommandEffect(options)));
+	program.command("runner-command").description("Run one scheduled Argo Workflow task").requiredOption("--payload-file <path>", "Path to the runner payload JSON").option("--node-id <id>", "Node id to execute without a task descriptor").option("--schedule-file <path>", "Path to the schedule artifact YAML").option("--schedule-source <source>", "Schedule source: file or db").action((options) => runRunnerProgram(runRunnerCommandEffect(options)));
 	program.command("runner-lifecycle").description("Run one Argo Workflow lifecycle phase").requiredOption("--phase <phase>", "Lifecycle phase to run").requiredOption("--payload-file <path>", "Path to the runner payload JSON").requiredOption("--schedule-file <path>", "Path to the schedule artifact YAML").action((options) => runRunnerProgram(runRunnerLifecycleEffect(options)));
-	program.command("runner-finalize").description("Finalize one Argo Workflow run").requiredOption("--payload-file <path>", "Path to the runner payload JSON").requiredOption("--schedule-file <path>", "Path to the schedule artifact YAML").requiredOption("--argo-status <status>", "Argo Workflow status").action((options) => runRunnerProgram(runRunnerFinalizeEffect(options)));
+	program.command("runner-pre-schedule").description("Run one dynamic pre-schedule phase").requiredOption("--phase <phase>", "pre-research, pre-planning, or generate-schedule").requiredOption("--payload-file <path>", "Path to the runner payload JSON").action((options) => runRunnerProgram(runPreScheduleEffect(options)));
+	program.command("runner-finalize").description("Finalize one Argo Workflow run").requiredOption("--payload-file <path>", "Path to the runner payload JSON").option("--schedule-file <path>", "Path to the schedule artifact YAML").option("--schedule-source <source>", "Schedule source: file or db").requiredOption("--argo-status <status>", "Argo Workflow status").action((options) => runRunnerProgram(runRunnerFinalizeEffect(options)));
+	program.command("runner-select-ready-wave").description("Select DB-ready nodes for the next dynamic Argo wave").requiredOption("--payload-file <path>", "Path to the runner payload JSON").requiredOption("--output-file <path>", "Path where the ready node id JSON array is written").action((options) => runRunnerProgram(runSelectReadyWaveEffect(options)));
 }
 //#endregion
 export { registerRunnerCommandCommand };