@ryanfw/prompt-orchestration-pipeline 1.2.7 → 1.2.9

package/src/core/orchestrator.ts

@@ -19,6 +19,7 @@ interface ResolvedDirs {
   pending: string;
   current: string;
   complete: string;
+  rejected: string;
 }
 
 /** Parsed seed file content (fields consumed by orchestrator). */
@@ -91,19 +92,30 @@ interface ChildHandle {
   kill(signal?: number): void;
 }
 
+interface SpawnedRunner {
+  pid: number;
+  kill?: () => void;
+}
+
 /** Seed filename regex. Captures jobId from {jobId}-seed.json. */
 export const SEED_PATTERN = /^([A-Za-z0-9-_]+)-seed\.json$/;
 
 import { join, basename, dirname, resolve } from "node:path";
-import { mkdir, rename } from "node:fs/promises";
+import { mkdir, rename, rm, stat, writeFile } from "node:fs/promises";
 import { watch } from "chokidar";
 import { createLogger } from "./logger";
 import { createTaskFileIO, generateLogName } from "./file-io";
 import { LogEvent, LogFileExtension } from "../config/log-events";
-import { getConfig, getPipelineConfig } from "./config";
+import { getConfig, getOrchestratorConfig, getPipelineConfig } from "./config";
 import { buildReexecArgs } from "../cli/self-reexec";
 import { writeJobStatus } from "./status-writer";
 import { initializeStatusFromArtifacts } from "./status-initializer";
+import {
+  listQueuedSeeds,
+  releaseJobSlot,
+  tryAcquireJobSlot,
+  updateJobSlotPid,
+} from "./job-concurrency";
 
 /**
  * Normalize any path that may already include `pipeline-data` (or subdirs
@@ -125,6 +137,7 @@ export function resolveDirs(dataDir: string): ResolvedDirs {
     pending: join(root, "pending"),
     current: join(root, "current"),
     complete: join(root, "complete"),
+    rejected: join(root, "rejected"),
   };
 }
 
@@ -165,7 +178,8 @@ export async function spawnRunner(
   dirs: ResolvedDirs,
   running: Map<string, ChildHandle>,
   logger: ReturnType<typeof createLogger>,
-  spawnFn: SpawnFn
+  spawnFn: SpawnFn,
+  onExit?: (jobId: string) => void | Promise<void>,
 ): Promise<void> {
   if (!seed.pipeline) {
     throw new Error(`seed.pipeline is required for job ${jobId}`);
@@ -207,63 +221,48 @@ export async function spawnRunner(
 
   running.set(jobId, child);
 
-  void child.exited.then((result) => {
-    running.delete(jobId);
-    logger.log(`job ${jobId} exited`, {
-      code: result.code,
-      signal: result.signal,
-      completionType: result.completionType,
+  void child.exited
+    .then(async (result) => {
+      running.delete(jobId);
+      logger.log(`job ${jobId} exited`, {
+        code: result.code,
+        signal: result.signal,
+        completionType: result.completionType,
+      });
+      if (!onExit) return;
+      try {
+        await onExit(jobId);
+      } catch (err) {
+        logger.error(`child exit cleanup failed for job ${jobId}`, err);
+      }
+    })
+    .catch((err) => {
+      logger.error(`failed while waiting for job ${jobId} exit`, err);
     });
-  });
 }
 
-export async function handleSeedAdd(
-  filePath: string,
-  dirs: ResolvedDirs,
-  running: Map<string, ChildHandle>,
-  logger: ReturnType<typeof createLogger>,
-  opts: OrchestratorOptions
-): Promise<void> {
-  const filename = basename(filePath);
-  const match = filename.match(SEED_PATTERN);
-
-  if (!match) {
-    logger.warn(`ignoring non-seed file: ${filename}`);
-    return;
-  }
-
-  const jobId = match[1]!;
-
-  let raw: string;
-  try {
-    raw = await Bun.file(filePath).text();
-  } catch (err) {
-    logger.warn(`failed to read ${filename}`, err);
-    return;
-  }
+export interface HandleChildExitOptions {
+  dataDir: string;
+  jobId: string;
+  triggerDrain: () => void;
+  lockTimeoutMs?: number;
+}
 
-  let seed: SeedData;
+export async function handleChildExit(opts: HandleChildExitOptions): Promise<void> {
   try {
-    seed = JSON.parse(raw) as SeedData;
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    logger.warn(`invalid JSON in ${filename}: ${message}`);
-    return;
+    await releaseJobSlot(opts.dataDir, opts.jobId, opts.lockTimeoutMs);
+  } finally {
+    opts.triggerDrain();
   }
+}
 
-  if (running.has(jobId)) return;
-
+async function scaffoldJobDir(
+  jobId: string,
+  seed: SeedData,
+  dirs: ResolvedDirs,
+  logger: ReturnType<typeof createLogger>,
+): Promise<void> {
   const jobDir = join(dirs.current, jobId);
-  const seedDest = join(jobDir, "seed.json");
-  if (await Bun.file(seedDest).exists()) return;
-
-  await mkdir(jobDir, { recursive: true });
-  try {
-    await rename(filePath, seedDest);
-  } catch (err) {
-    logger.error(`failed to move seed file for job ${jobId}`, err);
-    throw err;
-  }
   await mkdir(join(jobDir, "tasks"), { recursive: true });
 
   let pipelineTasks: string[] = [];
@@ -322,6 +321,56 @@ export async function handleSeedAdd(
 
   const logName = generateLogName("orchestrator", "init", LogEvent.START, LogFileExtension.JSON);
   await fileIO.writeLog(logName, JSON.stringify(startLog, null, 2), { mode: "replace" });
+}
+
+export async function handleSeedAdd(
+  filePath: string,
+  dirs: ResolvedDirs,
+  running: Map<string, ChildHandle>,
+  logger: ReturnType<typeof createLogger>,
+  opts: OrchestratorOptions
+): Promise<void> {
+  const filename = basename(filePath);
+  const match = filename.match(SEED_PATTERN);
+
+  if (!match) {
+    logger.warn(`ignoring non-seed file: ${filename}`);
+    return;
+  }
+
+  const jobId = match[1]!;
+
+  let raw: string;
+  try {
+    raw = await Bun.file(filePath).text();
+  } catch (err) {
+    logger.warn(`failed to read ${filename}`, err);
+    return;
+  }
+
+  let seed: SeedData;
+  try {
+    seed = JSON.parse(raw) as SeedData;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    logger.warn(`invalid JSON in ${filename}: ${message}`);
+    return;
+  }
+
+  if (running.has(jobId)) return;
+
+  const jobDir = join(dirs.current, jobId);
+  const seedDest = join(jobDir, "seed.json");
+  if (await Bun.file(seedDest).exists()) return;
+
+  await mkdir(jobDir, { recursive: true });
+  try {
+    await rename(filePath, seedDest);
+  } catch (err) {
+    logger.error(`failed to move seed file for job ${jobId}`, err);
+    throw err;
+  }
+  await scaffoldJobDir(jobId, seed, dirs, logger);
 
   const spawnFn = opts.spawn ?? createDefaultSpawn();
   try {
@@ -364,18 +413,284 @@ export async function stopChildren(
   running.clear();
 }
 
+const MAX_PENDING_SEED_FAILURES = 3;
+const pendingSeedFailures = new Map<string, { count: number; signature: string }>();
+
+export interface DrainPendingQueueOptions {
+  dataDir: string;
+  maxConcurrentJobs: number;
+  lockTimeoutMs: number;
+  spawnRunner: (jobId: string, seed: SeedData) => Promise<SpawnedRunner>;
+}
+
+export interface DrainPendingQueueResult {
+  promoted: string[];
+  remaining: number;
+}
+
+async function dirExists(path: string): Promise<boolean> {
+  try {
+    const s = await stat(path);
+    return s.isDirectory();
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
+    throw err;
+  }
+}
+
+async function promoteSeedFile(
+  pendingSeedPath: string,
+  currentJobDir: string,
+): Promise<void> {
+  await mkdir(currentJobDir, { recursive: true });
+  await rename(pendingSeedPath, join(currentJobDir, "seed.json"));
+}
+
+async function restoreSeedFile(currentJobDir: string, pendingSeedPath: string): Promise<void> {
+  const currentSeedPath = join(currentJobDir, "seed.json");
+  try {
+    await rename(currentSeedPath, pendingSeedPath);
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
+  }
+  await rm(currentJobDir, { recursive: true, force: true });
+}
+
+async function rejectSeedFile(
+  dataDir: string,
+  jobId: string,
+  seedPath: string,
+  reason: string,
+  message: string,
+): Promise<void> {
+  const rejectedJobDir = join(dataDir, "rejected", jobId);
+  await mkdir(rejectedJobDir, { recursive: true });
+  try {
+    await rename(seedPath, join(rejectedJobDir, "seed.json"));
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
+  }
+  await writeFile(
+    join(rejectedJobDir, "rejection.json"),
+    JSON.stringify({
+      jobId,
+      reason,
+      message,
+      rejectedAt: new Date().toISOString(),
+    }),
+  );
+  pendingSeedFailures.delete(jobId);
+}
+
+function recordPendingSeedFailure(jobId: string, signature: string): number {
+  const current = pendingSeedFailures.get(jobId);
+  const count = current && current.signature === signature ? current.count + 1 : 1;
+  pendingSeedFailures.set(jobId, { count, signature });
+  return count;
+}
+
+function clearPendingSeedFailure(jobId: string): void {
+  pendingSeedFailures.delete(jobId);
+}
+
+async function readPendingSeed(seedPath: string): Promise<
+  | { ok: true; seed: SeedData }
+  | { ok: false; reason: "missing" | "invalid"; message: string }
+> {
+  let raw: string;
+  try {
+    raw = await Bun.file(seedPath).text();
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return { ok: false, reason: "missing", message };
+    return { ok: false, reason: "invalid", message };
+  }
+  try {
+    return { ok: true, seed: JSON.parse(raw) as SeedData };
+  } catch (err) {
+    return {
+      ok: false,
+      reason: "invalid",
+      message: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
+export async function drainPendingQueue(
+  opts: DrainPendingQueueOptions,
+): Promise<DrainPendingQueueResult> {
+  const { dataDir, maxConcurrentJobs, lockTimeoutMs, spawnRunner } = opts;
+  const logger = createLogger("orchestrator");
+  const queued = await listQueuedSeeds(dataDir);
+  const promoted: string[] = [];
+
+  for (let i = 0; i < queued.length; i++) {
+    const { jobId, seedPath } = queued[i]!;
+
+    const parsedSeed = await readPendingSeed(seedPath);
+    if (!parsedSeed.ok) {
+      logger.warn(`failed to read queued seed ${jobId}-seed.json (${parsedSeed.reason}): ${parsedSeed.message}`);
+      const failures = recordPendingSeedFailure(jobId, `${parsedSeed.reason}:${queued[i]!.queuedAt}:${parsedSeed.message}`);
+      if (failures >= MAX_PENDING_SEED_FAILURES) {
+        try {
+          await rejectSeedFile(dataDir, jobId, seedPath, parsedSeed.reason, parsedSeed.message);
+          logger.warn(`rejected queued seed ${jobId}-seed.json after ${failures} failed reads`);
+        } catch (rejectErr) {
+          logger.error(`failed to reject queued seed ${jobId}-seed.json`, rejectErr);
+        }
+      }
+      continue;
+    }
+
+    const acquired = await tryAcquireJobSlot({
+      dataDir,
+      jobId,
+      maxConcurrentJobs,
+      source: "orchestrator",
+      lockTimeoutMs,
+    });
+    if (!acquired.ok) {
+      if (acquired.reason === "limit_reached") {
+        return { promoted, remaining: (await listQueuedSeeds(dataDir)).length };
+      }
+      logger.warn(`job slot already held for ${jobId}; skipping queued seed`);
+      continue;
+    }
+
+    const currentJobDir = join(dataDir, "current", jobId);
+    if (await dirExists(currentJobDir)) {
+      await releaseJobSlot(dataDir, jobId, lockTimeoutMs);
+      logger.warn(`current job directory already exists for ${jobId}; skipping promotion`);
+      continue;
+    }
+
+    try {
+      await promoteSeedFile(seedPath, currentJobDir);
+    } catch (err) {
+      await releaseJobSlot(dataDir, jobId, lockTimeoutMs);
+      logger.error(`failed to promote seed file for job ${jobId}`, err);
+      continue;
+    }
+
+    let spawned: SpawnedRunner;
+    try {
+      spawned = await spawnRunner(jobId, parsedSeed.seed);
+    } catch (err) {
+      await releaseJobSlot(dataDir, jobId, lockTimeoutMs);
+      try {
+        await restoreSeedFile(currentJobDir, seedPath);
+      } catch (restoreErr) {
+        logger.error(`failed to restore queued seed after spawn failure for job ${jobId}`, restoreErr);
+      }
+      const message = err instanceof Error ? err.message : String(err);
+      const failures = recordPendingSeedFailure(jobId, `spawn:${message}`);
+      if (failures >= MAX_PENDING_SEED_FAILURES) {
+        try {
+          await rejectSeedFile(dataDir, jobId, seedPath, "spawn_failed", message);
+          logger.error(`rejected queued seed ${jobId} after ${failures} spawn failures`, err);
+        } catch (rejectErr) {
+          logger.error(`failed to reject queued seed after spawn failures for job ${jobId}`, rejectErr);
+        }
+      }
+      logger.error(`failed to spawn queued job ${jobId}`, err);
+      continue;
+    }
+
+    try {
+      await updateJobSlotPid(dataDir, jobId, spawned.pid, lockTimeoutMs);
+    } catch (err) {
+      logger.error(`failed to record runner pid for job ${jobId}`, err);
+      spawned.kill?.();
+      try {
+        await releaseJobSlot(dataDir, jobId, lockTimeoutMs);
+      } catch (releaseErr) {
+        logger.error(`failed to release slot after pid update failure for job ${jobId}`, releaseErr);
+      }
+      try {
+        await restoreSeedFile(currentJobDir, seedPath);
+      } catch (restoreErr) {
+        logger.error(`failed to restore queued seed after pid update failure for job ${jobId}`, restoreErr);
+      }
+      const message = err instanceof Error ? err.message : String(err);
+      const failures = recordPendingSeedFailure(jobId, `pid:${message}`);
+      if (failures >= MAX_PENDING_SEED_FAILURES) {
+        try {
+          await rejectSeedFile(dataDir, jobId, seedPath, "pid_update_failed", message);
+        } catch (rejectErr) {
+          logger.error(`failed to reject queued seed after pid update failures for job ${jobId}`, rejectErr);
+        }
+      }
+      continue;
+    }
+    clearPendingSeedFailure(jobId);
+    promoted.push(jobId);
+  }
+
+  return { promoted, remaining: (await listQueuedSeeds(dataDir)).length };
+}
+
 export function startOrchestrator(opts: OrchestratorOptions): Promise<OrchestratorHandle> {
   if (!opts.dataDir) throw new Error("dataDir is required");
 
   const dirs = resolveDirs(opts.dataDir);
   const factory = opts.watcherFactory ?? ((path, options) => watch(path, options) as unknown as Watcher);
   const logger = createLogger("orchestrator");
+  const orchestratorConfig = getOrchestratorConfig();
+  const maxConcurrentJobs = orchestratorConfig.maxConcurrentJobs;
+  const lockTimeoutMs = orchestratorConfig.lockFileTimeout;
 
   return mkdir(dirs.pending, { recursive: true })
     .then(() => mkdir(dirs.current, { recursive: true }))
     .then(() => mkdir(dirs.complete, { recursive: true }))
+    .then(() => mkdir(dirs.rejected, { recursive: true }))
     .then(() => new Promise<OrchestratorHandle>((resolve, reject) => {
       const running = new Map<string, ChildHandle>();
+      const spawnFn = opts.spawn ?? createDefaultSpawn();
+
+      let isDraining = false;
+      let pendingDrain = false;
+      let stopped = false;
+      let activeDrain: Promise<void> | null = null;
+      const triggerDrain = (): void => {
+        if (stopped) return;
+        if (isDraining) {
+          pendingDrain = true;
+          return;
+        }
+        isDraining = true;
+        activeDrain = (async () => {
+          try {
+            do {
+              if (stopped) return;
+              pendingDrain = false;
+              await drainPendingQueue({
+                dataDir: dirs.dataDir,
+                maxConcurrentJobs,
+                lockTimeoutMs,
+                spawnRunner: spawnRunnerForJob,
+              });
+            } while (pendingDrain);
+          } catch (err) {
+            logger.error("drainPendingQueue failed", err);
+          } finally {
+            isDraining = false;
+            activeDrain = null;
+          }
+        })();
+      };
+
+      const onChildExit = (jobId: string): Promise<void> => {
+        if (stopped) return Promise.resolve();
+        return handleChildExit({ dataDir: dirs.dataDir, jobId, triggerDrain, lockTimeoutMs });
+      };
+
+      const spawnRunnerForJob = async (jobId: string, seed: SeedData): Promise<SpawnedRunner> => {
+        await scaffoldJobDir(jobId, seed, dirs, logger);
+        await spawnRunner(jobId, seed, dirs, running, logger, spawnFn, onChildExit);
+        const child = running.get(jobId);
+        if (!child) throw new Error(`spawnRunner did not register child for ${jobId}`);
+        return { pid: child.pid, kill: () => child.kill(15) };
+      };
 
       const watcher = factory(join(dirs.pending, "*.json"), {
         ignoreInitial: false,
@@ -384,16 +699,14 @@ export function startOrchestrator(opts: OrchestratorOptions): Promise<Orchestrat
       });
 
       const stop = async (): Promise<void> => {
+        stopped = true;
         await watcher.close();
+        if (activeDrain) await activeDrain;
         await stopChildren(running, logger);
       };
 
       watcher
-        .on("add", (filePath) => {
-          handleSeedAdd(filePath, dirs, running, logger, opts).catch((err: unknown) => {
-            logger.error(`unhandled error processing seed file ${filePath}`, err);
-          });
-        })
+        .on("add", () => triggerDrain())
         .on("error", reject)
         .on("ready", () => resolve({ stop }));
     }));

package/src/core/pipeline-runner.ts

@@ -2,18 +2,19 @@ import { join, dirname, resolve, basename } from "node:path";
 import { pathToFileURL } from "node:url";
 import { unlink, mkdir, rename, appendFile } from "node:fs/promises";
 import { unlinkSync } from "node:fs";
-import { getPipelineConfig } from "./config";
+import { getConfig, getPipelineConfig } from "./config";
 import { validatePipelineOrThrow } from "./validation";
 import { loadFreshModule } from "./module-loader";
 import { writeJobStatus } from "./status-writer";
 import { decideTransition } from "./lifecycle-policy";
 import { runPipeline } from "./task-runner";
-import type { AuditLogEntry } from "./task-runner";
+import type { AuditLogEntry, PipelineResult } from "./task-runner";
 import { ensureTaskSymlinkBridge } from "./symlink-bridge";
 import { validateTaskSymlinks, repairTaskSymlinks, cleanupTaskSymlinks } from "./symlink-utils";
 import { createTaskFileIO, generateLogName } from "./file-io";
 import { LogEvent, LogFileExtension } from "../config/log-events";
 import { TaskState } from "../config/statuses";
+import { releaseJobSlot } from "./job-concurrency";
 
 // ─── Type definitions ─────────────────────────────────────────────────────────
 
@@ -98,6 +99,7 @@ export interface TaskStatus {
   attempts?: number;
   executionTimeMs?: number;
   refinementAttempts?: number;
+  restartCount?: number;
   error?: NormalizedError;
   failedStage?: string;
   stageLogPath?: string;
@@ -223,21 +225,41 @@ export function cleanupPidFileSync(workDir: string): void {
   }
 }
 
-/** Registers SIGINT, SIGTERM, and process exit handlers to clean up the PID file. */
-export function installSignalHandlers(workDir: string): void {
-  process.on("SIGINT", () => {
-    cleanupPidFileSync(workDir);
-    process.exit();
-  });
-  process.on("SIGTERM", () => {
-    cleanupPidFileSync(workDir);
-    process.exit();
-  });
+/** Registers SIGINT, SIGTERM, SIGHUP, and process exit handlers to release the job slot and clean up the PID file. */
+export function installSignalHandlers(workDir: string, dataDir: string, jobId: string): void {
+  let shuttingDown = false;
+  const handle = async () => {
+    if (shuttingDown) return;
+    shuttingDown = true;
+    try {
+      await releaseJobSlot(dataDir, jobId);
+    } catch (err) {
+      console.error(`failed to release job slot for ${jobId} during shutdown`, err);
+    }
+    try {
+      cleanupPidFileSync(workDir);
+    } catch (err) {
+      console.error(`failed to clean runner pid for ${jobId} during shutdown`, err);
+    } finally {
+      process.exit();
+    }
+  };
+  process.once("SIGINT", handle);
+  process.once("SIGTERM", handle);
+  process.once("SIGHUP", handle);
   process.on("exit", () => {
     cleanupPidFileSync(workDir);
   });
 }
 
+async function releaseJobSlotBestEffort(dataDir: string, jobId: string): Promise<void> {
+  try {
+    await releaseJobSlot(dataDir, jobId);
+  } catch (err) {
+    console.error(`failed to release job slot for ${jobId}`, err);
+  }
+}
+
 // ─── Pipeline loading ─────────────────────────────────────────────────────────
 
 /** Reads, parses, and validates a pipeline.json file. */
@@ -256,14 +278,21 @@ export async function loadTaskRegistry(registryPath: string): Promise<TaskRegist
 
 // ─── Pipeline job entry point ─────────────────────────────────────────────────
 
+const INITIAL_RETRY_DELAY_MS = 2_000;
+const MAX_RETRY_DELAY_MS = 30_000;
+const RETRY_BACKOFF_MULTIPLIER = 2;
+
 /** Runs a pipeline job end-to-end for the given job ID. */
 export async function runPipelineJob(jobId: string): Promise<void> {
   let workDir: string | undefined;
+  const poRoot = resolve(process.env["PO_ROOT"] ?? process.cwd());
+  let dataDir: string | undefined = resolve(poRoot, process.env["PO_DATA_DIR"] ?? "pipeline-data");
   try {
   const config = await resolveJobConfig(jobId);
   workDir = config.workDir;
+  dataDir = resolve(config.poRoot, config.dataDir);
   await writePidFile(config.workDir);
-  installSignalHandlers(config.workDir);
+  installSignalHandlers(config.workDir, dataDir, jobId);
 
   const pipeline = await loadPipeline(config.pipelineJsonPath);
   const taskRegistry = await loadTaskRegistry(config.taskRegistryPath);
@@ -414,8 +443,38 @@ export async function runPipelineJob(jobId: string): Promise<void> {
       },
     };
 
-    // Delegate to task runner
-    const result = await runPipeline(relocatedEntryPath, taskExecutionContext);
+    // Delegate to task runner with bounded retry loop.
+    // Guard against partial test mocks or malformed runtime config.
+    const configuredMaxAttempts = getConfig().taskRunner?.maxAttempts;
+    const maxAttempts = Number.isInteger(configuredMaxAttempts) ? configuredMaxAttempts : 3;
+    const cap = Math.max(1, maxAttempts);
+
+    let result: PipelineResult | undefined;
+    for (let attempt = 1; attempt <= cap; attempt++) {
+      result = await runPipeline(relocatedEntryPath, taskExecutionContext);
+      if (result.ok) break;
+      if (attempt >= cap) break;
+
+      const delay = Math.min(
+        INITIAL_RETRY_DELAY_MS * RETRY_BACKOFF_MULTIPLIER ** (attempt - 1),
+        MAX_RETRY_DELAY_MS,
+      );
+
+      await writeJobStatus(config.workDir, (snapshot) => {
+        const entry = snapshot.tasks[taskName] ?? {};
+        const currentAttempts = typeof entry.attempts === "number" ? entry.attempts : attempt;
+        entry.state = "running";
+        entry.attempts = currentAttempts + 1;
+        entry.restartCount = (entry.restartCount ?? 0) + 1;
+        delete entry.failedStage;
+        delete entry.error;
+        snapshot.tasks[taskName] = entry;
+      });
+
+      await Bun.sleep(delay);
+    }
+
+    if (!result) throw new Error("Retry loop produced no result");
 
     if (result.ok) {
       // Compute execution time from logs
@@ -466,6 +525,7 @@ export async function runPipelineJob(jobId: string): Promise<void> {
         snapshot.tasks[taskName] = raw as typeof snapshot.tasks[string];
       });
 
+      await releaseJobSlotBestEffort(dataDir, jobId);
       process.exit(1);
     }
 
@@ -484,6 +544,7 @@ export async function runPipelineJob(jobId: string): Promise<void> {
     const finalStatus = JSON.parse(finalStatusText) as JobStatus;
     await completeJob(config, finalStatus, pipelineArtifacts);
   }
+  await releaseJobSlotBestEffort(dataDir, jobId);
   } catch (err) {
     const normalized = normalizeError(err);
     console.error(normalized.message);
@@ -512,6 +573,9 @@ export async function runPipelineJob(jobId: string): Promise<void> {
         // Do not mask the original failure if PID cleanup fails
       }
     }
+    if (dataDir !== undefined) {
+      await releaseJobSlotBestEffort(dataDir, jobId);
+    }
     process.exitCode = 1;
     setTimeout(() => process.exit(1), 5000).unref();
     process.exit(1);