@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4

package/src/core/orchestrator.ts

@@ -1,7 +1,7 @@
 /** Options for starting the orchestrator. */
 export interface OrchestratorOptions {
-  /** Root directory for pipeline data. Normalized to canonical pipeline-data/ root. */
-  dataDir: string;
+  /** Workspace root (not the `pipeline-data` subdir); omitted values fall back to `PO_ROOT` at boot. */
+  dataDir?: string;
   /** Injection point for process spawner. Defaults to Bun.spawn. */
   spawn?: SpawnFn;
   /** Injection point for filesystem watcher factory. Defaults to chokidar.watch. */
@@ -76,7 +76,7 @@ type SpawnFn = (cmd: string[], options: {
 }) => ChildHandle;
 
 /** Result of a child process exit, capturing all diagnostic fields. */
-interface ChildExitResult {
+export interface ChildExitResult {
   /** Exit code, or null if terminated by signal. */
   code: number | null;
   /** Signal name (e.g. "SIGTERM"), or null if exited normally. */
@@ -86,7 +86,7 @@ interface ChildExitResult {
 }
 
 /** Minimal child process handle for tracking. */
-interface ChildHandle {
+export interface ChildHandle {
   readonly pid: number;
   /** Resolves with structured exit details when the process terminates. */
   readonly exited: Promise<ChildExitResult>;
@@ -98,10 +98,22 @@ interface SpawnedRunner {
   kill?: () => void;
 }
 
-/** Seed filename regex. Captures jobId from {jobId}-seed.json. */
-export const SEED_PATTERN = /^([A-Za-z0-9-_]+)-seed\.json$/;
+function describeCleanupError(err: unknown): unknown {
+  if (!(err instanceof AggregateError)) return err;
+  return {
+    name: err.name,
+    message: err.message,
+    errors: err.errors.map((cause) => {
+      if (cause instanceof Error) {
+        return { name: cause.name, message: cause.message, stack: cause.stack };
+      }
+      return { message: String(cause) };
+    }),
+  };
+}
 
-import { join, basename, dirname, resolve } from "node:path";
+import { join, dirname, resolve } from "node:path";
+import { readFileSync } from "node:fs";
 import { mkdir, rename, rm, stat, writeFile } from "node:fs/promises";
 import { watch } from "chokidar";
 import { createLogger } from "./logger";
@@ -115,11 +127,15 @@ import { materializeNormalizedPipelineDefinition } from "./pipeline-definition";
 import { discoverHarnesses } from "../harness/index.ts";
 import { applyHarnessDiscovery } from "../harness/discovery.ts";
 import {
+  listActiveJobLeases,
   listQueuedSeeds,
+  pruneStaleJobSlots,
   releaseJobSlot,
   tryAcquireJobSlot,
   updateJobSlotPid,
 } from "./job-concurrency";
+import { isProcessAlive, reconcileInterruptedJob, reconcileOrphanedCurrentJobs } from "./runner-liveness";
+import { resolveWorkspaceRoot } from "../config/paths";
 
 /**
  * Normalize any path that may already include `pipeline-data` (or subdirs
@@ -184,7 +200,7 @@ export async function spawnRunner(
   running: Map<string, ChildHandle>,
   logger: ReturnType<typeof createLogger>,
   spawnFn: SpawnFn,
-  onExit?: (jobId: string) => void | Promise<void>,
+  onExit?: (jobId: string, result: ChildExitResult) => void | Promise<void>,
 ): Promise<void> {
   if (!seed.pipeline) {
     throw new Error(`seed.pipeline is required for job ${jobId}`);
@@ -236,9 +252,9 @@ export async function spawnRunner(
       });
       if (!onExit) return;
       try {
-        await onExit(jobId);
+        await onExit(jobId, result);
       } catch (err) {
-        logger.error(`child exit cleanup failed for job ${jobId}`, err);
+        logger.error(`child exit cleanup failed for job ${jobId}`, describeCleanupError(err));
       }
     })
     .catch((err) => {
@@ -251,11 +267,46 @@ export interface HandleChildExitOptions {
   jobId: string;
   triggerDrain: () => void;
   lockTimeoutMs?: number;
+  code?: number | null;
+  signal?: string | null;
+  releaseSlot?: (dataDir: string, jobId: string, lockTimeoutMs?: number) => Promise<void>;
+  reconcileJob?: typeof reconcileInterruptedJob;
 }
 
 export async function handleChildExit(opts: HandleChildExitOptions): Promise<void> {
+  const lockTimeoutMs = opts.lockTimeoutMs ?? getOrchestratorConfig().lockFileTimeout;
+  const releaseSlot = opts.releaseSlot ?? releaseJobSlot;
+  const reconcileJob = opts.reconcileJob ?? reconcileInterruptedJob;
   try {
-    await releaseJobSlot(opts.dataDir, opts.jobId, opts.lockTimeoutMs);
+    let releaseError: unknown;
+    let reconcileError: unknown;
+
+    try {
+      await releaseSlot(opts.dataDir, opts.jobId, lockTimeoutMs);
+    } catch (err) {
+      releaseError = err;
+    }
+
+    const jobDir = join(opts.dataDir, "current", opts.jobId);
+    try {
+      await reconcileJob({
+        dataDir: opts.dataDir,
+        jobId: opts.jobId,
+        jobDir,
+        source: "child-exit",
+        code: opts.code,
+        signal: opts.signal,
+        lockTimeoutMs,
+      });
+    } catch (err) {
+      reconcileError = err;
+    }
+
+    if (releaseError && reconcileError) {
+      throw new AggregateError([releaseError, reconcileError], "child exit cleanup failed");
+    }
+    if (releaseError) throw releaseError;
+    if (reconcileError) throw reconcileError;
   } finally {
     opts.triggerDrain();
   }
@@ -327,63 +378,6 @@ async function scaffoldJobDir(
   await fileIO.writeLog(logName, JSON.stringify(startLog, null, 2), { mode: "replace" });
 }
 
-export async function handleSeedAdd(
-  filePath: string,
-  dirs: ResolvedDirs,
-  running: Map<string, ChildHandle>,
-  logger: ReturnType<typeof createLogger>,
-  opts: OrchestratorOptions
-): Promise<void> {
-  const filename = basename(filePath);
-  const match = filename.match(SEED_PATTERN);
-
-  if (!match) {
-    logger.warn(`ignoring non-seed file: ${filename}`);
-    return;
-  }
-
-  const jobId = match[1]!;
-
-  let raw: string;
-  try {
-    raw = await Bun.file(filePath).text();
-  } catch (err) {
-    logger.warn(`failed to read ${filename}`, err);
-    return;
-  }
-
-  let seed: SeedData;
-  try {
-    seed = JSON.parse(raw) as SeedData;
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    logger.warn(`invalid JSON in ${filename}: ${message}`);
-    return;
-  }
-
-  if (running.has(jobId)) return;
-
-  const jobDir = join(dirs.current, jobId);
-  const seedDest = join(jobDir, "seed.json");
-  if (await Bun.file(seedDest).exists()) return;
-
-  await mkdir(jobDir, { recursive: true });
-  try {
-    await rename(filePath, seedDest);
-  } catch (err) {
-    logger.error(`failed to move seed file for job ${jobId}`, err);
-    throw err;
-  }
-  await scaffoldJobDir(jobId, seed, dirs, logger);
-
-  const spawnFn = opts.spawn ?? createDefaultSpawn();
-  try {
-    await spawnRunner(jobId, seed, dirs, running, logger, spawnFn);
-  } catch (err) {
-    logger.error(`failed to spawn runner for job ${jobId}`, err);
-  }
-}
-
 export async function stopChildren(
   running: Map<string, ChildHandle>,
   logger: ReturnType<typeof createLogger>
@@ -417,6 +411,90 @@ export async function stopChildren(
   running.clear();
 }
 
+export interface AdoptOrphanedRunnersOptions {
+  dataDir: string;
+  currentDir: string;
+  running: Map<string, ChildHandle>;
+  isAlive?: (pid: number) => boolean;
+}
+
+function synthesizeAdoptedHandle(opts: {
+  pid: number;
+  jobId: string;
+  currentDir: string;
+  isAlive: (pid: number) => boolean;
+}): ChildHandle {
+  const { pid, jobId, currentDir, isAlive } = opts;
+  let exitedPromise: Promise<ChildExitResult> | null = null;
+  return {
+    pid,
+    kill: (signal) => {
+      const runnerPidPath = join(currentDir, jobId, "runner.pid");
+      let raw: string;
+      try {
+        raw = readFileSync(runnerPidPath, "utf-8");
+      } catch (err) {
+        if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
+        return;
+      }
+      const current = Number(raw.trim());
+      if (!Number.isInteger(current) || current !== pid) return;
+      try {
+        if (signal === undefined) process.kill(pid);
+        else process.kill(pid, signal);
+      } catch {
+        // ESRCH: already exited. EPERM: not ours. Either way, no-op.
+      }
+    },
+    get exited(): Promise<ChildExitResult> {
+      if (exitedPromise) return exitedPromise;
+      // Adopted handles are stop handles, not a background supervisor. Reading
+      // `exited` starts lazy polling only during shutdown/stop, and the signal
+      // completion sentinel is intentionally generic because the original exit
+      // code/signal is unavailable for a process we did not spawn.
+      exitedPromise = new Promise<ChildExitResult>((resolve) => {
+        const interval = setInterval(() => {
+          if (!isAlive(pid)) {
+            clearInterval(interval);
+            resolve({ code: null, signal: null, completionType: "signal" });
+          }
+        }, 100);
+        interval.unref();
+      });
+      return exitedPromise;
+    },
+  };
+}
+
+export async function adoptOrphanedRunners(opts: AdoptOrphanedRunnersOptions): Promise<string[]> {
+  const alive = opts.isAlive ?? isProcessAlive;
+  const leases = await listActiveJobLeases(opts.dataDir);
+  const adopted: string[] = [];
+  for (const lease of leases) {
+    const pid = lease.pid;
+    if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) continue;
+    if (!alive(pid)) continue;
+    const runnerPidPath = join(opts.currentDir, lease.jobId, "runner.pid");
+    let raw: string;
+    try {
+      raw = readFileSync(runnerPidPath, "utf-8");
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
+      continue;
+    }
+    const filePid = Number(raw.trim());
+    if (!Number.isInteger(filePid) || filePid !== pid) continue;
+    opts.running.set(lease.jobId, synthesizeAdoptedHandle({
+      pid,
+      jobId: lease.jobId,
+      currentDir: opts.currentDir,
+      isAlive: alive,
+    }));
+    adopted.push(lease.jobId);
+  }
+  return adopted;
+}
+
 const MAX_PENDING_SEED_FAILURES = 3;
 const pendingSeedFailures = new Map<string, { count: number; signature: string }>();
 
@@ -424,6 +502,7 @@ export interface DrainPendingQueueOptions {
   dataDir: string;
   maxConcurrentJobs: number;
   lockTimeoutMs: number;
+  staleLeaseTimeoutMs: number;
   spawnRunner: (jobId: string, seed: SeedData) => Promise<SpawnedRunner>;
 }
 
@@ -540,7 +619,7 @@ async function readPendingSeed(seedPath: string): Promise<
 export async function drainPendingQueue(
   opts: DrainPendingQueueOptions,
 ): Promise<DrainPendingQueueResult> {
-  const { dataDir, maxConcurrentJobs, lockTimeoutMs, spawnRunner } = opts;
+  const { dataDir, maxConcurrentJobs, lockTimeoutMs, staleLeaseTimeoutMs, spawnRunner } = opts;
   const logger = createLogger("orchestrator");
   const queued = await listQueuedSeeds(dataDir);
   const promoted: string[] = [];
@@ -569,6 +648,7 @@ export async function drainPendingQueue(
       maxConcurrentJobs,
       source: "orchestrator",
       lockTimeoutMs,
+      staleLeaseTimeoutMs,
     });
     if (!acquired.ok) {
       if (acquired.reason === "limit_reached") {
@@ -670,14 +750,17 @@ async function harnessPreflight(logger: ReturnType<typeof createLogger>): Promis
 }
 
 export function startOrchestrator(opts: OrchestratorOptions): Promise<OrchestratorHandle> {
-  if (!opts.dataDir) throw new Error("dataDir is required");
+  const root = resolveWorkspaceRoot(opts.dataDir);
+  getConfig();
 
-  const dirs = resolveDirs(opts.dataDir);
+  const dirs = resolveDirs(root);
   const factory = opts.watcherFactory ?? ((path, options) => watch(path, options) as unknown as Watcher);
   const logger = createLogger("orchestrator");
   const orchestratorConfig = getOrchestratorConfig();
   const maxConcurrentJobs = orchestratorConfig.maxConcurrentJobs;
   const lockTimeoutMs = orchestratorConfig.lockFileTimeout;
+  const staleLeaseTimeoutMs = orchestratorConfig.staleLeaseTimeoutMs;
+  const running = new Map<string, ChildHandle>();
 
   return mkdir(dirs.pending, { recursive: true })
     .then(() => mkdir(dirs.current, { recursive: true }))
@@ -685,8 +768,43 @@ export function startOrchestrator(opts: OrchestratorOptions): Promise<Orchestrat
     .then(() => mkdir(dirs.rejected, { recursive: true }))
     .then(() => mkdir(dirs.staging, { recursive: true }))
     .then(() => harnessPreflight(logger))
+    .then(async () => {
+      try {
+        const pruned = await pruneStaleJobSlots(dirs.dataDir, lockTimeoutMs, staleLeaseTimeoutMs);
+        logger.log("boot prune complete", { pruned });
+      } catch (err) {
+        logger.warn(
+          `boot prune failed; continuing: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+      try {
+        const summary = await reconcileOrphanedCurrentJobs({
+          dataDir: dirs.dataDir,
+          currentDir: dirs.current,
+          lockTimeoutMs,
+          staleLeaseTimeoutMs,
+          staleAfterMs: getOrchestratorConfig().staleRunningGraceMs,
+        });
+        logger.log("boot reconciliation complete", summary);
+      } catch (err) {
+        logger.warn(
+          `boot reconciliation failed; continuing: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+      try {
+        const adopted = await adoptOrphanedRunners({
+          dataDir: dirs.dataDir,
+          currentDir: dirs.current,
+          running,
+        });
+        if (adopted.length > 0) logger.log("adopted orphaned runners", { adopted });
+      } catch (err) {
+        logger.warn(
+          `runner re-adoption failed; continuing: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+    })
     .then(() => new Promise<OrchestratorHandle>((resolve, reject) => {
-      const running = new Map<string, ChildHandle>();
       const spawnFn = opts.spawn ?? createDefaultSpawn();
 
       let isDraining = false;
@@ -709,6 +827,7 @@ export function startOrchestrator(opts: OrchestratorOptions): Promise<Orchestrat
                 dataDir: dirs.dataDir,
                 maxConcurrentJobs,
                 lockTimeoutMs,
+                staleLeaseTimeoutMs,
                 spawnRunner: spawnRunnerForJob,
               });
             } while (pendingDrain);
@@ -721,9 +840,16 @@ export function startOrchestrator(opts: OrchestratorOptions): Promise<Orchestrat
         })();
       };
 
-      const onChildExit = (jobId: string): Promise<void> => {
+      const onChildExit = (jobId: string, result: ChildExitResult): Promise<void> => {
         if (stopped) return Promise.resolve();
-        return handleChildExit({ dataDir: dirs.dataDir, jobId, triggerDrain, lockTimeoutMs });
+        return handleChildExit({
+          dataDir: dirs.dataDir,
+          jobId,
+          triggerDrain,
+          lockTimeoutMs,
+          code: result.code,
+          signal: result.signal,
+        });
       };
 
       const spawnRunnerForJob = async (jobId: string, seed: SeedData): Promise<SpawnedRunner> => {