@oisincoveney/pipeline 4.0.3 → 4.1.0

package/dist/coordinator/composition.js

@@ -1,20 +1,25 @@
+import { loadMokaDbUrl } from "../moka-global-config.js";
 import { reconcileAllRuns } from "../remote/jobs/reconcile.js";
 import { createRunSpecRegistry, startRunSpecServer } from "../remote/jobs/run-spec-server.js";
 import { runJobFleetWatchLoop } from "../remote/jobs/watch.js";
 import { submitParsedMoka } from "../remote/submit/service.js";
 import { withRunControlStore } from "../run-control/command-context.js";
+import { resolveExecutionStore } from "../run-control/postgres/postgres-execution-store.js";
 import { kubernetesJobFleetReadApi } from "../runtime/services/kubernetes-jobs-read-service.js";
 import { kubernetesJobFleetWatchApi } from "../runtime/services/kubernetes-jobs-watch-service.js";
 import { loadCoordinatorJobsConfig } from "./config.js";
 import { createRunScopedOwnerReference } from "./run-owner.js";
-import { submitWaveJobs } from "./wave-advance.js";
+import { makeSubmitWaveJobs } from "./wave-advance.js";
+import { advanceReconciledWaves, makeWaveTickRegistry } from "./wave-tick-registry.js";
 import * as DateTime from "effect/DateTime";
 import * as Effect from "effect/Effect";
 import * as ManagedRuntime from "effect/ManagedRuntime";
 import * as Layer from "effect/Layer";
+import * as Exit from "effect/Exit";
 import * as Cause from "effect/Cause";
 import { randomBytes } from "node:crypto";
 import * as Fiber from "effect/Fiber";
+import * as Scope from "effect/Scope";
 //#region src/coordinator/composition.ts
 /**
 * ENG-57.1/57.2/57.3: the coordinator's composition root (doc-1 decision 3,
@@ -80,27 +85,58 @@ const defaultOnWatchLoopFailure = (error) => {
 * drives the startup pass, every event-triggered pass, and every
 * post-reconnect pass, so dynamic and static Jobs runs finalize through one
 * run-id-keyed path (AC#4). */
-const buildWatchLoopEffect = (dependencies, config, store) => runJobFleetWatchLoop({
-	...dependencies.watchLoop,
-	namespace: config.namespace,
-	reconcile: reconcileAllRuns({
-		fleet: dependencies.jobFleetReadApi ?? kubernetesJobFleetReadApi({}),
+const buildWatchLoopEffect = (dependencies, config, store, executionStore, registry) => {
+	const fleet = dependencies.jobFleetReadApi ?? kubernetesJobFleetReadApi({});
+	const now = () => DateTime.formatIso(DateTime.nowUnsafe());
+	return runJobFleetWatchLoop({
+		...dependencies.watchLoop,
 		namespace: config.namespace,
-		now: () => DateTime.formatIso(DateTime.nowUnsafe()),
-		store
-	}).pipe(Effect.tap((reports) => Effect.sync(() => dependencies.onReconcile?.(reports))), Effect.asVoid),
-	watch: dependencies.jobFleetWatchApi ?? kubernetesJobFleetWatchApi({})
-});
+		reconcile: reconcileAllRuns({
+			fleet,
+			namespace: config.namespace,
+			now,
+			store,
+			submitted: (runId) => executionStore.listSubmitted({ runId })
+		}).pipe(Effect.tap((reports) => Effect.sync(() => dependencies.onReconcile?.(reports))), Effect.flatMap((reports) => advanceReconciledWaves({
+			executionStore,
+			fleet,
+			now,
+			registry
+		}, reports)), Effect.asVoid),
+		watch: dependencies.jobFleetWatchApi ?? kubernetesJobFleetWatchApi({})
+	});
+};
 /** ENG-57.3: starts the watch loop as its own fiber over the coordinator's
 * lifetime. Resolves the store via the injected test override, or the
 * production `withRunControlStore` seam (Postgres-backed, scoped so
 * interrupting the returned fiber also releases the store's connection). A
 * permanent failure is reported via `onWatchLoopFailure`, never left silent. */
-const startWatchLoop = (dependencies, config) => {
+const startWatchLoop = (dependencies, config, executionStore, registry) => {
 	const onWatchLoopFailure = dependencies.onWatchLoopFailure ?? defaultOnWatchLoopFailure;
-	const watchLoopEffect = dependencies.runControlStore === void 0 ? withRunControlStore((store) => buildWatchLoopEffect(dependencies, config, store)) : buildWatchLoopEffect(dependencies, config, dependencies.runControlStore);
+	const watchLoopEffect = dependencies.runControlStore === void 0 ? withRunControlStore((store) => buildWatchLoopEffect(dependencies, config, store, executionStore, registry)) : buildWatchLoopEffect(dependencies, config, dependencies.runControlStore, executionStore, registry);
 	return watchLoopRuntime.runFork(watchLoopEffect.pipe(Effect.tapCause((cause) => Effect.sync(() => onWatchLoopFailure(Cause.squash(cause))))));
 };
+/**
+* ENG-57.10: acquire the ONE durable {@link ExecutionStore} the coordinator
+* shares between `submit` and the watch loop, for the coordinator's whole
+* lifetime. A test override is used as-is (the test owns its release). Otherwise
+* the Postgres store is bound inside a coordinator-held {@link Scope} via
+* `resolveExecutionStore`'s `acquireRelease`, so `close()` releases the connection
+* exactly once (`Scope.close`) — never leaked, even under watch-fiber interrupt.
+*/
+const acquireSharedExecutionStore = async (dependencies) => {
+	if (dependencies.executionStore !== void 0) return {
+		release: async () => Promise.resolve(),
+		store: dependencies.executionStore
+	};
+	const scope = await watchLoopRuntime.runPromise(Scope.make());
+	return {
+		release: async () => {
+			await watchLoopRuntime.runPromise(Scope.close(scope, Exit.void));
+		},
+		store: await watchLoopRuntime.runPromise(Effect.provideService(resolveExecutionStore(loadMokaDbUrl()), Scope.Scope, scope))
+	};
+};
 const resolveRunSpecBind = (dependencies, env) => {
 	const host = dependencies.runSpecBind?.host ?? env[RUN_SPEC_HOST_ENV];
 	const port = dependencies.runSpecBind?.port ?? parseRunSpecPortEnv(env);
@@ -132,7 +168,9 @@ const createCoordinator = async (dependencies = {}) => {
 		registry
 	}, resolveRunSpecBind(dependencies, env));
 	const runSpecBaseUrl = dependencies.runSpecBaseUrl ?? env[RUN_SPEC_BASE_URL_ENV] ?? runSpecServer.baseUrl;
-	const watchLoopFiber = startWatchLoop(dependencies, config);
+	const { release: releaseExecutionStore, store: executionStore } = await acquireSharedExecutionStore(dependencies);
+	const waveRegistry = watchLoopRuntime.runSync(makeWaveTickRegistry());
+	const watchLoopFiber = startWatchLoop(dependencies, config, executionStore, waveRegistry);
 	const submit = async (options, callerDependencies = {}) => {
 		const runId = options.run?.id ?? (dependencies.generateRunId ?? generateFallbackRunId)();
 		const ownerReference = await createRunScopedOwnerReference({
@@ -145,7 +183,10 @@ const createCoordinator = async (dependencies = {}) => {
 			ownerReference,
 			registry,
 			runSpecBaseUrl,
-			submitJobs: submitWaveJobs,
+			submitJobs: makeSubmitWaveJobs({
+				executionStore,
+				registerWaveContext: (context) => watchLoopRuntime.runSync(waveRegistry.register(context))
+			}),
 			...dependencies.jobsDependencies === void 0 ? {} : { jobsDependencies: dependencies.jobsDependencies }
 		};
 		return await submitParsedMoka(resolveSubmitOptions(options, config), {
@@ -157,6 +198,7 @@ const createCoordinator = async (dependencies = {}) => {
 	return {
 		close: async () => {
 			await watchLoopRuntime.runPromise(Fiber.interrupt(watchLoopFiber));
+			await releaseExecutionStore();
 			await runSpecServer.close();
 		},
 		config,

package/dist/coordinator/wave-advance.js

@@ -1,10 +1,10 @@
-import "../schema-boundary.js";
+import { parseStrictWithSchema } from "../schema-boundary.js";
 import { compileScheduleArtifact, parseScheduleArtifact } from "../planning/generate.js";
-import "../remote/jobs/job-outcome.js";
+import { classifyNodeOutcome } from "../remote/jobs/job-outcome.js";
 import { submitRunnerNodeJobs } from "../remote/jobs/submit.js";
 import { parseJobsSubmitResult } from "../remote/submit/jobs-submit-result.js";
 import { defaultBuildNodeRunSpec, requireCompiledSchedule, requireSubmitNamespace, sharedJobOptions } from "../remote/submit/jobs-submission.js";
-import { inMemoryExecutionStore } from "../run-control/execution-store.js";
+import { executionTerminalStatusSchema } from "../run-control/execution-store.js";
 import { planNextWave } from "./wave-scheduler.js";
 import * as DateTime from "effect/DateTime";
 import * as Effect from "effect/Effect";
@@ -15,6 +15,7 @@ import * as Schema from "effect/Schema";
 import * as Arr from "effect/Array";
 import * as HashMap from "effect/HashMap";
 import * as Layer from "effect/Layer";
+import * as HashSet from "effect/HashSet";
 import * as Str from "effect/String";
 //#region src/coordinator/wave-advance.ts
 /**
@@ -79,6 +80,53 @@ const advanceWave = (deps) => Effect.gen(function* body() {
 		})
 	};
 });
+/**
+* The cluster-truth → ExecutionStore bridge. Observes the run's Jobs/pods once
+* and, for every node NOT already terminal in the store, folds a terminal
+* cluster outcome (via the ENG-19 {@link classifyNodeOutcome} classifier, reused
+* not reimplemented) into the ExecutionStore under CAS. A node whose Job is still
+* pending is left untouched. This is what makes a later {@link advanceWave} tick
+* see a predecessor's terminal success and release the next wave.
+*/
+const recordClusterTerminals = (deps) => Effect.gen(function* body() {
+	const observation = yield* deps.fleet.observeRun({
+		namespace: deps.namespace,
+		runId: deps.runId
+	});
+	const submitted = yield* deps.executionStore.listSubmitted({ runId: deps.runId });
+	yield* Effect.forEach(deps.nodes, (node) => deps.executionStore.getExecution({
+		nodeId: node.id,
+		runId: deps.runId
+	}).pipe(Effect.flatMap((existing) => {
+		if (Option$1.isSome(existing)) return Effect.void;
+		const outcome = classifyNodeOutcome(observation.nodes[node.id] ?? { pods: [] }, { submitted: HashSet.has(submitted, node.id) });
+		if (outcome.kind === "pending") return Effect.void;
+		const status = parseStrictWithSchema(executionTerminalStatusSchema, outcome.status);
+		return deps.executionStore.recordTerminal({
+			at: deps.now(),
+			attempt: 1,
+			identity: {
+				generation: FIRST_EXECUTION_GENERATION,
+				nodeId: node.id,
+				runId: deps.runId
+			},
+			status
+		}).pipe(Effect.asVoid);
+	})), { concurrency: 1 });
+});
+/**
+* A full reconcile-driven tick: fold cluster truth into the ExecutionStore, then
+* advance the wave. This is the unit the coordinator's reconcile loop calls per
+* cycle to release later waves as predecessors finish.
+*/
+const runWaveTick = (deps) => recordClusterTerminals({
+	executionStore: deps.executionStore,
+	fleet: deps.fleet,
+	namespace: deps.namespace,
+	nodes: deps.nodes,
+	now: deps.now,
+	runId: deps.runId
+}).pipe(Effect.flatMap(() => advanceWave(deps)));
 /** A wave-submit requested a node id absent from the compiled schedule — a
 * scheduler invariant violation, never expected in normal operation. */
 var WaveSubmitMissingNodeError = class extends Schema.TaggedErrorClass()("WaveSubmitMissingNodeError", { message: Schema.String }) {
@@ -87,15 +135,13 @@ var WaveSubmitMissingNodeError = class extends Schema.TaggedErrorClass()("WaveSu
 	}
 };
 /**
-* The `SubmitMokaJobsTransport.submitJobs` seam (ENG-57.1/57.2): submit a run's
-* FIRST wave as per-node Jobs. Compiles the schedule, then runs a single
-* {@link advanceWave} tick over a fresh (empty) ExecutionStore — which resolves
-* to the root wave (nodes with no predecessors), fanning parallel roots out and
-* NEVER eagerly submitting a downstream node. Later waves are the reconcile
-* driver's concern ({@link runWaveTick}). Replaces ENG-57.1's single-ready-node
-* primitive so static and dynamic (post-schedule) runs share this one scheduler.
+* Compile a submission into its {@link WaveTickContext}: the node graph and the
+* per-wave create-or-observe submit closure. Extracted from the wave transport so
+* the coordinator can register the SAME `{ nodes, submitReadyNodes }` the first
+* tick used and re-tick it from the watch loop — one scheduler, one submit path,
+* static and dynamic alike.
 */
-const submitWaveJobs = async (input) => {
+const compileWaveContext = (input, executionStore) => {
 	const scheduleYaml = requireCompiledSchedule(input.plan);
 	const namespace = requireSubmitNamespace(input.options);
 	const nodes = compileScheduleArtifact(input.plan.config, parseScheduleArtifact(scheduleYaml, "schedule.yaml"), input.worktreePath).plan.topologicalOrder;
@@ -127,17 +173,44 @@ const submitWaveJobs = async (input) => {
 				nodeIds: [...nodeIds]
 			}, jobsDependencies);
 		}
-	});
-	const result = await runWaveEffect(advanceWave({
-		executionStore: inMemoryExecutionStore(),
+	}).pipe(Effect.tap((submitted) => Effect.forEach(submitted, (job) => executionStore.recordSubmitted({
+		nodeId: job.nodeId,
+		runId: input.plan.runId
+	}), {
+		concurrency: 1,
+		discard: true
+	})));
+	return {
+		namespace,
 		nodes,
-		now: () => DateTime.formatIso(DateTime.nowUnsafe()),
 		runId: input.plan.runId,
 		submitReadyNodes
+	};
+};
+/**
+* The `SubmitMokaJobsTransport.submitJobs` seam (ENG-57.1/57.2/57.10): submit a
+* run's FIRST wave as per-node Jobs over the coordinator's SHARED durable
+* {@link ExecutionStore} (never a fresh in-memory one), registering the run's
+* {@link WaveTickContext} so the watch loop re-ticks later waves as predecessors
+* terminalize. Compiles the schedule, then runs a single {@link advanceWave} tick
+* — resolving to the root wave (nodes with no predecessors), fanning parallel
+* roots out and NEVER eagerly submitting a downstream node. Later waves are the
+* reconcile driver's concern ({@link runWaveTick}). One scheduler for static and
+* dynamic (post-schedule) runs alike.
+*/
+const makeSubmitWaveJobs = (deps) => async (input) => {
+	const context = compileWaveContext(input, deps.executionStore);
+	deps.registerWaveContext(context);
+	const result = await runWaveEffect(advanceWave({
+		executionStore: deps.executionStore,
+		nodes: context.nodes,
+		now: () => DateTime.formatIso(DateTime.nowUnsafe()),
+		runId: context.runId,
+		submitReadyNodes: context.submitReadyNodes
 	}));
 	return parseJobsSubmitResult({
 		jobs: [...result.submitted],
-		namespace,
+		namespace: context.namespace,
 		runId: input.plan.runId,
 		waves: [{ nodeIds: [...result.plan.ready] }],
 		workflowName: input.plan.workflowId,
@@ -145,4 +218,4 @@ const submitWaveJobs = async (input) => {
 	});
 };
 //#endregion
-export { WaveSubmitMissingNodeError, advanceWave, submitWaveJobs };
+export { WaveSubmitMissingNodeError, advanceWave, compileWaveContext, makeSubmitWaveJobs, recordClusterTerminals, runWaveTick };

package/dist/coordinator/wave-tick-registry.js

@@ -0,0 +1,56 @@
+import { isTerminalRunStatus } from "../remote/jobs/job-outcome.js";
+import { runWaveTick } from "./wave-advance.js";
+import * as Effect from "effect/Effect";
+import * as Option$1 from "effect/Option";
+import * as HashMap from "effect/HashMap";
+import * as Ref from "effect/Ref";
+//#region src/coordinator/wave-tick-registry.ts
+const makeWaveTickRegistry = () => Ref.make(HashMap.empty()).pipe(Effect.map((ref) => ({
+	get: (runId) => Ref.get(ref).pipe(Effect.map((map) => HashMap.get(map, runId))),
+	register: (context) => Ref.update(ref, (map) => HashMap.set(map, context.runId, context)),
+	remove: (runId) => Ref.update(ref, (map) => HashMap.remove(map, runId))
+})));
+/** A run whose reconcile report shows a terminal run status is finished — drop
+* its context so it never re-ticks; nothing left to advance. */
+const isReportTerminal = (report) => Option$1.match(report.runStatus, {
+	onNone: () => false,
+	onSome: (status) => isTerminalRunStatus(status)
+});
+/**
+* Advance one reconciled run one wave-tick. A run with a terminal status is
+* dropped from the registry. A run with a registered context is folded through
+* {@link runWaveTick} over the SHARED durable ExecutionStore — releasing any wave
+* whose predecessors just terminalized — and dropped once the whole run is
+* complete. A run with no registered context (e.g. not re-driven after a restart)
+* is a no-op: the reconcile loop still finalized it against cluster truth.
+*/
+const advanceReconciledRun = (deps, report) => {
+	if (isReportTerminal(report)) return deps.registry.remove(report.runId);
+	return deps.registry.get(report.runId).pipe(Effect.flatMap(Option$1.match({
+		onNone: () => Effect.void,
+		onSome: (context) => runWaveTick({
+			executionStore: deps.executionStore,
+			fleet: deps.fleet,
+			namespace: context.namespace,
+			nodes: context.nodes,
+			now: deps.now,
+			runId: context.runId,
+			submitReadyNodes: context.submitReadyNodes
+		}).pipe(Effect.flatMap((result) => result.plan.runComplete ? deps.registry.remove(context.runId) : Effect.void))
+	})));
+};
+/**
+* ENG-57.10: the per-reconcile-cycle wave advancement the coordinator watch loop
+* runs after {@link reconcileAllRuns}. For every run in the cycle's reports it
+* runs one {@link runWaveTick} over the ONE shared durable ExecutionStore, so a
+* later wave submits only once its predecessors terminalize — the missing driver
+* that advanced a multi-wave graph past wave 1. Idempotent: re-running a tick with
+* unchanged inputs re-observes existing Jobs (create-or-observe) and re-reads the
+* ExecutionStore CAS, never a duplicate Job.
+*/
+const advanceReconciledWaves = (deps, reports) => Effect.forEach(reports, (report) => advanceReconciledRun(deps, report), {
+	concurrency: 1,
+	discard: true
+});
+//#endregion
+export { advanceReconciledWaves, makeWaveTickRegistry };

package/dist/remote/jobs/job-outcome.js

@@ -42,7 +42,7 @@ const OUTCOME_RULES = [
 		status: "failed"
 	},
 	{
-		matches: ({ job }) => job === void 0,
+		matches: ({ job }, submitted) => job === void 0 && submitted,
 		reason: "job-lost",
 		status: "failed"
 	},
@@ -57,15 +57,24 @@ const OUTCOME_RULES = [
 * `pending` (no terminal signal yet — leave it running). Only ever consulted
 * for nodes the store still holds non-terminal, so a `finalize` result never
 * overwrites a node that already reported its own outcome.
+*
+* ENG-57.10: `submitted` (default `true`) scopes the dead-Job (`job-lost`)
+* sweeper. Whole-run callers submit every node up front and omit it (keeping the
+* always-finalize net). Incremental wave callers pass the run's durable
+* submitted-set membership so a not-yet-submitted future wave stays `pending`
+* instead of being terminalized failed.
 */
-const classifyNodeOutcome = (observation) => Option$1.match(Arr.findFirst(OUTCOME_RULES, (candidate) => candidate.matches(observation)), {
-	onNone: () => ({ kind: "pending" }),
-	onSome: (rule) => ({
-		kind: "finalize",
-		reason: rule.reason,
-		status: rule.status
-	})
-});
+const classifyNodeOutcome = (observation, context = {}) => {
+	const submitted = context.submitted ?? true;
+	return Option$1.match(Arr.findFirst(OUTCOME_RULES, (candidate) => candidate.matches(observation, submitted)), {
+		onNone: () => ({ kind: "pending" }),
+		onSome: (rule) => ({
+			kind: "finalize",
+			reason: rule.reason,
+			status: rule.status
+		})
+	});
+};
 const TERMINAL_NODE_STATUSES = [
 	"passed",
 	"failed",

package/dist/remote/jobs/reconcile.js

@@ -3,6 +3,7 @@ import * as Effect from "effect/Effect";
 import * as Option$1 from "effect/Option";
 import * as R from "effect/Record";
 import * as Arr from "effect/Array";
+import * as HashSet from "effect/HashSet";
 //#region src/remote/jobs/reconcile.ts
 /** A node absent from the cluster observation has no Job present — the dead-Job
 * signal the classifier finalizes to failed. */
@@ -10,12 +11,12 @@ const observationForNode = (observation, nodeId) => Option$1.getOrElse(R.get(obs
 const emit = (deps, event) => Effect.sync(() => {
 	deps.onEvent?.(event);
 });
-const reconcileNode = (deps, runId, observation, nodeId, currentStatus) => {
+const reconcileNode = (deps, runId, observation, nodeId, currentStatus, submitted) => {
 	if (isTerminalNodeStatus(currentStatus)) return Effect.succeed({
 		finalization: Option$1.none(),
 		status: currentStatus
 	});
-	const outcome = classifyNodeOutcome(observationForNode(observation, nodeId));
+	const outcome = classifyNodeOutcome(observationForNode(observation, nodeId), { submitted });
 	if (outcome.kind === "pending") return Effect.succeed({
 		finalization: Option$1.none(),
 		status: currentStatus
@@ -77,7 +78,9 @@ const reconcileRun = (deps, runId) => Effect.gen(function* body() {
 		namespace: deps.namespace,
 		runId
 	});
-	const results = yield* Effect.forEach(R.toEntries(manifest.nodes), ([nodeId, status]) => reconcileNode(deps, runId, observation, nodeId, status), { concurrency: 1 });
+	const submittedSet = deps.submitted === void 0 ? void 0 : yield* deps.submitted(runId);
+	const isSubmitted = (nodeId) => submittedSet === void 0 || HashSet.has(submittedSet, nodeId);
+	const results = yield* Effect.forEach(R.toEntries(manifest.nodes), ([nodeId, status]) => reconcileNode(deps, runId, observation, nodeId, status, isSubmitted(nodeId)), { concurrency: 1 });
 	const finalizedNodes = results.flatMap((result) => Arr.fromOption(result.finalization));
 	const runStatus = aggregateRunStatus(results.map((result) => result.status));
 	yield* finalizeRun(deps, runId, runStatus);

package/dist/run-control/postgres/execution-schema.js

@@ -0,0 +1,43 @@
+import { mokaPostgresSchema } from "../../runtime/durable-store/postgres/schema.js";
+import { integer, primaryKey, text, timestamp } from "drizzle-orm/pg-core";
+//#region src/run-control/postgres/execution-schema.ts
+/**
+* ENG-57.6: Drizzle schema backing the Postgres {@link ExecutionStore}. Exactly
+* one row per `(run_id, node_id)` holds that node's AUTHORITATIVE terminal
+* execution — the `generation`/`attempt` that won the compare-and-set, its
+* terminal `status`, and the server wall time it terminalized.
+*
+* The CAS is enforced at the database layer: the write is a guarded upsert whose
+* `ON CONFLICT ... DO UPDATE` only fires when the incoming `(generation, attempt)`
+* is strictly newer than the stored row, so a late or replayed older execution
+* can never overwrite a newer terminal state even under concurrent writers.
+*
+* Lives in the shared `moka` substrate schema alongside the run-control and
+* durable-store tables. Deliberately carries no foreign key to a run table: an
+* execution can be terminalized independently of the event-sourced run-control
+* row, and the coordinator owns `(run_id, node_id)` as the sole key.
+*/
+const execution = mokaPostgresSchema.table("moka_execution", {
+	attempt: integer("attempt").notNull(),
+	generation: integer("generation").notNull(),
+	nodeId: text("node_id").notNull(),
+	runId: text("run_id").notNull(),
+	status: text("status").$type().notNull(),
+	terminalizedAt: timestamp("terminalized_at", { withTimezone: true }).defaultNow().notNull()
+}, (table) => [primaryKey({ columns: [table.runId, table.nodeId] })]);
+/**
+* ENG-57.10: the durable SUBMITTED-SET the dead-Job sweeper scopes to. One row per
+* `(run_id, node_id)` records that the wave scheduler actually submitted that
+* node's Job. First-writer-wins (`ON CONFLICT DO NOTHING`): a node is submitted
+* once per run and stays submitted. The sweeper only finalizes a node absent from
+* the cluster as `job-lost` when it appears here — so a not-yet-submitted future
+* wave stays pending instead of being terminalized failed, while a submitted node
+* whose Job later vanished still fails.
+*/
+const executionSubmission = mokaPostgresSchema.table("moka_execution_submission", {
+	nodeId: text("node_id").notNull(),
+	runId: text("run_id").notNull(),
+	submittedAt: timestamp("submitted_at", { withTimezone: true }).defaultNow().notNull()
+}, (table) => [primaryKey({ columns: [table.runId, table.nodeId] })]);
+//#endregion
+export { execution, executionSubmission };

package/dist/run-control/postgres/postgres-execution-store.js

@@ -0,0 +1,103 @@
+import { requireMokaDbUrl } from "../../moka-global-config.js";
+import { migratePostgresSubstrate } from "../../runtime/durable-store/postgres/migrate-substrate.js";
+import { openMokaSubstrateClient } from "../../runtime/durable-store/postgres/client.js";
+import { parseStoredExecution, toStoredExecution } from "../execution-store.js";
+import { execution, executionSubmission } from "./execution-schema.js";
+import * as Effect from "effect/Effect";
+import * as Option$1 from "effect/Option";
+import * as HashSet from "effect/HashSet";
+import { drizzle } from "drizzle-orm/postgres-js";
+import { and, eq, lt, or } from "drizzle-orm";
+//#region src/run-control/postgres/postgres-execution-store.ts
+/** Apply the shared Drizzle migrations (delegates to the lock-guarded substrate migrator). */
+const migratePostgresExecutionStore = async (dbUrl) => {
+	await migratePostgresSubstrate(dbUrl);
+};
+const dbEffect = (run) => Effect.tryPromise({
+	catch: (error) => error,
+	try: run
+});
+const rowToStored = (row) => parseStoredExecution({
+	attempt: row.attempt,
+	generation: row.generation,
+	nodeId: row.nodeId,
+	runId: row.runId,
+	status: row.status,
+	terminalizedAt: row.terminalizedAt.toISOString()
+});
+const readExecution = (db, runId, nodeId) => Effect.gen(function* body() {
+	const row = (yield* dbEffect(() => db.select().from(execution).where(and(eq(execution.runId, runId), eq(execution.nodeId, nodeId))).limit(1))).at(0);
+	return row === void 0 ? Option$1.none() : Option$1.some(rowToStored(row));
+});
+const recordTerminal = (db, request) => Effect.gen(function* body() {
+	const incoming = toStoredExecution(request);
+	const values = {
+		attempt: incoming.attempt,
+		generation: incoming.generation,
+		nodeId: incoming.nodeId,
+		runId: incoming.runId,
+		status: incoming.status,
+		terminalizedAt: new Date(incoming.terminalizedAt)
+	};
+	const appliedRow = (yield* dbEffect(() => db.insert(execution).values(values).onConflictDoUpdate({
+		set: {
+			attempt: values.attempt,
+			generation: values.generation,
+			status: values.status,
+			terminalizedAt: values.terminalizedAt
+		},
+		setWhere: or(lt(execution.generation, values.generation), and(eq(execution.generation, values.generation), lt(execution.attempt, values.attempt))),
+		target: [execution.runId, execution.nodeId]
+	}).returning())).at(0);
+	if (appliedRow !== void 0) return {
+		decision: "applied",
+		record: rowToStored(appliedRow)
+	};
+	const current = yield* readExecution(db, incoming.runId, incoming.nodeId);
+	const surviving = Option$1.getOrElse(current, () => incoming);
+	return {
+		decision: surviving.generation === incoming.generation && surviving.attempt === incoming.attempt ? "idempotent" : "rejected-stale",
+		record: surviving
+	};
+});
+const recordSubmitted = (db, runId, nodeId) => dbEffect(() => db.insert(executionSubmission).values({
+	nodeId,
+	runId
+}).onConflictDoNothing()).pipe(Effect.asVoid);
+const listSubmitted = (db, runId) => dbEffect(() => db.select({ nodeId: executionSubmission.nodeId }).from(executionSubmission).where(eq(executionSubmission.runId, runId))).pipe(Effect.map((rows) => HashSet.fromIterable(rows.map((row) => row.nodeId))));
+const postgresExecutionStore = (dbUrl) => {
+	const client = openMokaSubstrateClient(dbUrl);
+	const db = drizzle(client);
+	return {
+		close: async () => {
+			await client.end();
+		},
+		getExecution: (input) => readExecution(db, input.runId, input.nodeId),
+		listSubmitted: (input) => listSubmitted(db, input.runId),
+		recordSubmitted: (input) => recordSubmitted(db, input.runId, input.nodeId),
+		recordTerminal: (request) => recordTerminal(db, request)
+	};
+};
+/**
+* ENG-57.10: the scoped durable {@link ExecutionStore} the coordinator shares
+* between `submit()` and the watch loop. `db.url` presence is the durable-substrate
+* switch (mirroring {@link resolveRunControlStore}): set → the Postgres store,
+* migrated then bound as a scoped resource whose connection is released on scope
+* exit; absent → `db.url-required` before any store is bound. Because the store's
+* connection is acquired via {@link Effect.acquireRelease}, closing the coordinator
+* scope releases it exactly once — no leaked connection even if the watch fiber is
+* interrupted.
+*/
+const resolveExecutionStore = (dbUrl) => requireMokaDbUrl(dbUrl).pipe(Effect.flatMap((requiredDbUrl) => Effect.tryPromise({
+	catch: (error) => error,
+	try: async () => {
+		await migratePostgresExecutionStore(requiredDbUrl);
+	}
+}).pipe(Effect.flatMap(() => Effect.acquireRelease(Effect.sync(() => postgresExecutionStore(requiredDbUrl)), (store) => Effect.tryPromise({
+	catch: (error) => error,
+	try: async () => {
+		await store.close();
+	}
+}).pipe(Effect.orDie))))));
+//#endregion
+export { migratePostgresExecutionStore, postgresExecutionStore, resolveExecutionStore };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@oisincoveney/pipeline",
-  "version": "4.0.3",
+  "version": "4.1.0",
   "description": "Config-driven multi-agent pipeline runner for repository work",
   "keywords": [],
   "license": "ISC",