@ryanfw/prompt-orchestration-pipeline 1.3.1 → 1.3.3

package/src/core/file-io.ts

@@ -1,9 +1,10 @@
 import { mkdir, rename, appendFile } from "node:fs/promises";
-import { existsSync, mkdirSync, readFileSync, writeFileSync, appendFileSync, renameSync } from "node:fs";
+import { existsSync, mkdirSync, writeFileSync, appendFileSync, renameSync } from "node:fs";
 import { dirname, join, basename } from "node:path";
 import { Database } from "bun:sqlite";
 import { LogFileExtension, isValidLogEvent, isValidLogFileExtension } from "../config/log-events";
-import { writeJobStatus } from "./status-writer";
+import { writeJobStatus, flushJobStatus } from "./status-writer";
+import { createJobLogger } from "./logger";
 import { executeBatch, validateBatchOptions } from "./batch-runner";
 import type { BatchOptions, BatchResult } from "./batch-runner";
 
@@ -169,34 +170,6 @@ export async function trackFile(
   });
 }
 
-/**
- * @internal Synchronous status writer for writeLogSync and getDB.
- * Reads tasks-status.json, applies updater, writes back as pretty-printed JSON.
- * Falls back to a minimal default snapshot on missing or invalid JSON.
- * No temp-file-rename, no async queue participation.
- */
-export function writeJobStatusSync(
-  jobDir: string,
-  updater: (snapshot: Record<string, unknown>) => void,
-): void {
-  const statusPath = join(jobDir, "tasks-status.json");
-  let snapshot: Record<string, unknown>;
-  try {
-    const raw = readFileSync(statusPath, "utf-8");
-    snapshot = JSON.parse(raw) as Record<string, unknown>;
-  } catch {
-    snapshot = {
-      id: basename(jobDir),
-      state: "pending",
-      tasks: {},
-      files: { artifacts: [], logs: [], tmp: [] },
-    };
-  }
-  updater(snapshot);
-  mkdirSync(jobDir, { recursive: true });
-  writeFileSync(statusPath, JSON.stringify(snapshot, null, 2));
-}
-
 export function createTaskFileIO(config: TaskFileIOConfig): TaskFileIO {
   const { workDir, taskName, statusPath, getStage } = config;
   const trackTaskFilesEnabled = config.trackTaskFiles ?? true;
@@ -234,28 +207,8 @@ export function createTaskFileIO(config: TaskFileIOConfig): TaskFileIO {
     const db = new Database(dbPath);
     db.exec("PRAGMA journal_mode = WAL;");
 
-    writeJobStatusSync(jobDir, (snapshot: Record<string, unknown>) => {
-      const files = (snapshot["files"] ?? {}) as FilesRecord;
-      snapshot["files"] = files;
-      const artifactsList = (files.artifacts ?? []) as string[];
-      files.artifacts = artifactsList;
-      if (!artifactsList.includes("run.db")) {
-        artifactsList.push("run.db");
-      }
-
-      if (trackTaskFilesEnabled) {
-        const tasks = (snapshot["tasks"] ?? {}) as Record<string, TaskRecord>;
-        snapshot["tasks"] = tasks;
-        const task = (tasks[taskName] ?? {}) as TaskRecord;
-        tasks[taskName] = task;
-        const taskFiles = (task.files ?? {}) as FilesRecord;
-        task.files = taskFiles;
-        const taskArtifactsList = (taskFiles.artifacts ?? []) as string[];
-        taskFiles.artifacts = taskArtifactsList;
-        if (!taskArtifactsList.includes("run.db")) {
-          taskArtifactsList.push("run.db");
-        }
-      }
+    void trackFile(jobDir, "artifacts", "run.db", taskName, trackTaskFilesEnabled).catch((err) => {
+      createJobLogger("file-io", basename(jobDir)).warn("getDB: manifest tracking failed", err);
     });
 
     return db;
@@ -314,28 +267,8 @@ export function createTaskFileIO(config: TaskFileIOConfig): TaskFileIO {
         renameSync(tmpPath, filePath);
       }
 
-      writeJobStatusSync(jobDir, (snapshot: Record<string, unknown>) => {
-        const files = (snapshot["files"] ?? {}) as FilesRecord;
-        snapshot["files"] = files;
-        const logsList = (files.logs ?? []) as string[];
-        files.logs = logsList;
-        if (!logsList.includes(name)) {
-          logsList.push(name);
-        }
-
-        if (trackTaskFilesEnabled) {
-          const tasks = (snapshot["tasks"] ?? {}) as Record<string, TaskRecord>;
-          snapshot["tasks"] = tasks;
-          const task = (tasks[taskName] ?? {}) as TaskRecord;
-          tasks[taskName] = task;
-          const taskFiles = (task.files ?? {}) as FilesRecord;
-          task.files = taskFiles;
-          const taskLogsList = (taskFiles.logs ?? []) as string[];
-          taskFiles.logs = taskLogsList;
-          if (!taskLogsList.includes(name)) {
-            taskLogsList.push(name);
-          }
-        }
+      void trackFile(jobDir, "logs", name, taskName, trackTaskFilesEnabled).catch((err) => {
+        createJobLogger("file-io", basename(jobDir)).warn("writeLogSync: manifest tracking failed", err);
       });
     },
 
@@ -348,6 +281,7 @@ export function createTaskFileIO(config: TaskFileIOConfig): TaskFileIO {
         return await executeBatch(db, options);
       } finally {
         db.close();
+        await flushJobStatus(jobDir);
       }
     },
   };

package/src/core/job-concurrency.ts

@@ -252,7 +252,7 @@ function delay(ms: number): Promise<void> {
 }
 
 // mkdir with recursive:false is atomic — exactly one caller wins the create when racing.
-async function withRuntimeLock<T>(
+export async function withRuntimeLock<T>(
   dataDir: string,
   lockTimeoutMs: number,
   fn: () => Promise<T>,
@@ -347,6 +347,11 @@ async function readLease(slotPath: string): Promise<JobSlotLease | null> {
   }
 }
 
+export async function readJobSlotLease(dataDir: string, jobId: string): Promise<JobSlotLease | null> {
+  const { runningJobsDir } = getConcurrencyRuntimePaths(dataDir);
+  return readLease(join(runningJobsDir, `${jobId}.json`));
+}
+
 async function writeLeaseAtomic(slotPath: string, lease: JobSlotLease): Promise<void> {
   const tmpPath = `${slotPath}.tmp.${randomBytes(8).toString("hex")}`;
   await writeFile(tmpPath, JSON.stringify(lease));

package/src/core/job-view.ts

@@ -0,0 +1,329 @@
+import {
+  JOB_STATUS_SYNONYMS,
+  VALID_JOB_STATUSES,
+  deriveJobStatusFromTasks,
+  jobStateToStatus,
+  normalizeJobStatus,
+  normalizeTaskState,
+} from "../config/statuses.ts";
+import type { JobStatusValue, TaskStateValue } from "../config/statuses.ts";
+import type { GateInfo } from "./status-writer.ts";
+
+export type RetryError = { message: string; name?: string; stack?: string } | string;
+
+export interface JobViewError {
+  code: string;
+  message: string;
+  path?: string;
+  name?: string;
+  stack?: string;
+}
+
+export interface JobViewTask {
+  state: TaskStateValue;
+  name: string;
+  files: { artifacts: string[]; logs: string[]; tmp: string[] };
+  startedAt?: string | null;
+  endedAt?: string | null;
+  attempts?: number;
+  restartCount?: number;
+  retrying?: boolean;
+  nextRetryAt?: string;
+  lastRetryError?: RetryError | null;
+  executionTimeMs?: number;
+  refinementAttempts?: number;
+  stageLogPath?: string;
+  errorContext?: unknown;
+  currentStage?: string;
+  failedStage?: string;
+  artifacts?: unknown;
+  tokenUsage?: unknown[];
+  error?: { message: string; [key: string]: unknown } | null;
+  skipReason?: string;
+  skippedBy?: string;
+  controlApplied?: boolean;
+}
+
+export interface JobView {
+  id: string;
+  jobId: string;
+  name: string;
+  title: string;
+  status: JobStatusValue;
+  progress: number;
+  createdAt: string | null;
+  updatedAt: string | null;
+  lastUpdated?: string;
+  location: string | null;
+  tasks: Record<string, JobViewTask>;
+  files: Record<string, unknown>;
+  costs: Record<string, unknown>;
+  gate?: GateInfo | null;
+  pipeline?: string;
+  pipelineLabel?: string;
+  pipelineConfig?: Record<string, unknown>;
+  current?: unknown;
+  currentStage?: unknown;
+  warnings?: string[];
+  readable?: boolean;
+  error?: JobViewError;
+  errorContext?: unknown;
+}
+
+interface TaskFilesView {
+  artifacts: string[];
+  logs: string[];
+  tmp: string[];
+}
+
+const EMPTY_TASK_FILES: TaskFilesView = { artifacts: [], logs: [], tmp: [] };
+const DEFAULT_JOB_ERROR_CODE = "JOB_ERROR";
+
+const warnedFallbackKeys = new Set<string>();
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isRecognizedStatusValue(value: unknown): value is string {
+  if (typeof value !== "string") return false;
+  const normalized = value.toLowerCase().trim();
+  if (Object.prototype.hasOwnProperty.call(JOB_STATUS_SYNONYMS, normalized)) return true;
+  return VALID_JOB_STATUSES.has(normalized);
+}
+
+function toStringOrNull(value: unknown): string | null {
+  return typeof value === "string" ? value : null;
+}
+
+function toStringArray(value: unknown): string[] {
+  if (!Array.isArray(value)) return [];
+  return value.filter((entry): entry is string => typeof entry === "string");
+}
+
+function toFiniteNumber(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+function toTaskFiles(value: unknown): TaskFilesView {
+  if (!isRecord(value)) return { ...EMPTY_TASK_FILES };
+  return {
+    artifacts: toStringArray(value["artifacts"]),
+    logs: toStringArray(value["logs"]),
+    tmp: toStringArray(value["tmp"]),
+  };
+}
+
+function toTaskError(value: unknown): JobViewTask["error"] {
+  if (value === null) return null;
+  if (typeof value === "string") return { message: value };
+  if (isRecord(value)) return value as { message: string; [key: string]: unknown };
+  return undefined;
+}
+
+function toGateInfo(value: unknown): GateInfo | null | undefined {
+  if (value === null) return null;
+  if (!isRecord(value)) return undefined;
+  const rawArtifacts = value["artifacts"];
+  const artifacts = Array.isArray(rawArtifacts)
+    ? rawArtifacts.filter((entry): entry is string => typeof entry === "string")
+    : undefined;
+  return {
+    afterTask: typeof value["afterTask"] === "string" ? value["afterTask"] : "",
+    message: typeof value["message"] === "string" ? value["message"] : "",
+    requestedAt: typeof value["requestedAt"] === "string" ? value["requestedAt"] : "",
+    ...(artifacts && artifacts.length > 0 ? { artifacts } : {}),
+  };
+}
+
+function toJobViewTask(name: string, value: unknown): JobViewTask {
+  const task = isRecord(value) ? value : {};
+  return {
+    name,
+    state: normalizeTaskState(task["state"]),
+    files: toTaskFiles(task["files"]),
+    startedAt: toStringOrNull(task["startedAt"]),
+    endedAt: toStringOrNull(task["endedAt"]),
+    attempts: typeof task["attempts"] === "number" ? task["attempts"] : undefined,
+    restartCount: typeof task["restartCount"] === "number" ? task["restartCount"] : undefined,
+    retrying: typeof task["retrying"] === "boolean" ? task["retrying"] : undefined,
+    nextRetryAt: typeof task["nextRetryAt"] === "string" ? task["nextRetryAt"] : undefined,
+    lastRetryError: Object.hasOwn(task, "lastRetryError")
+      ? (task["lastRetryError"] as RetryError | null)
+      : undefined,
+    executionTimeMs: typeof task["executionTimeMs"] === "number"
+      ? task["executionTimeMs"]
+      : undefined,
+    refinementAttempts: typeof task["refinementAttempts"] === "number"
+      ? task["refinementAttempts"]
+      : undefined,
+    stageLogPath: typeof task["stageLogPath"] === "string" ? task["stageLogPath"] : undefined,
+    errorContext: task["errorContext"],
+    currentStage: typeof task["currentStage"] === "string" ? task["currentStage"] : undefined,
+    failedStage: typeof task["failedStage"] === "string" ? task["failedStage"] : undefined,
+    artifacts: task["artifacts"],
+    tokenUsage: Array.isArray(task["tokenUsage"]) ? task["tokenUsage"] : undefined,
+    error: toTaskError(task["error"]),
+    skipReason: typeof task["skipReason"] === "string" ? task["skipReason"] : undefined,
+    skippedBy: typeof task["skippedBy"] === "string" ? task["skippedBy"] : undefined,
+    controlApplied: typeof task["controlApplied"] === "boolean" ? task["controlApplied"] : undefined,
+  };
+}
+
+function toJobViewTasks(value: unknown): Record<string, JobViewTask> {
+  if (Array.isArray(value)) {
+    return Object.fromEntries(
+      value.map((entry, index) => {
+        const record = isRecord(entry) ? entry : {};
+        const explicitName = record["name"];
+        const name = typeof explicitName === "string" && explicitName.length > 0
+          ? explicitName
+          : `task-${index}`;
+        return [name, toJobViewTask(name, record)] as const;
+      }),
+    );
+  }
+  if (isRecord(value)) {
+    return Object.fromEntries(
+      Object.entries(value).map(([name, entry]) => [name, toJobViewTask(name, entry)] as const),
+    );
+  }
+  return {};
+}
+
+function getProgress(rawProgress: unknown, tasks: JobViewTask[]): number {
+  const explicitProgress = toFiniteNumber(rawProgress);
+  if (explicitProgress !== undefined) return explicitProgress;
+  if (tasks.length === 0) return 0;
+
+  const completed = tasks.filter((task) => task.state === "done" || task.state === "skipped").length;
+  return Math.floor((completed / tasks.length) * 100);
+}
+
+function getRawTaskEntries(value: unknown): unknown[] {
+  if (Array.isArray(value)) return value;
+  if (isRecord(value)) return Object.values(value);
+  return [];
+}
+
+function getCosts(record: Record<string, unknown>): Record<string, unknown> {
+  const explicitCosts = record["costs"];
+  if (isRecord(explicitCosts)) return explicitCosts;
+
+  let totalCost = 0;
+  let totalInputTokens = 0;
+  let totalOutputTokens = 0;
+
+  for (const value of getRawTaskEntries(record["tasks"])) {
+    if (!isRecord(value)) continue;
+    const usage = value["tokenUsage"];
+    if (!Array.isArray(usage)) continue;
+
+    for (const entry of usage) {
+      if (!Array.isArray(entry)) continue;
+      totalInputTokens += toFiniteNumber(entry[1]) ?? 0;
+      totalOutputTokens += toFiniteNumber(entry[2]) ?? 0;
+      totalCost += toFiniteNumber(entry[3]) ?? 0;
+    }
+  }
+
+  const totalTokens = totalInputTokens + totalOutputTokens;
+  if (totalTokens === 0 && totalCost === 0) return {};
+  return { totalCost, totalTokens, totalInputTokens, totalOutputTokens };
+}
+
+function resolveStateStatus(
+  fromState: JobStatusValue,
+  tasks: JobViewTask[],
+  location: string,
+): JobStatusValue {
+  if (fromState !== "complete" || location === "complete" || tasks.length === 0) {
+    return fromState;
+  }
+
+  const taskStatus = deriveJobStatusFromTasks(tasks);
+  return taskStatus === "complete" ? fromState : taskStatus;
+}
+
+function toTitle(record: Record<string, unknown>, fallback: string): string {
+  const candidate = record["title"] ?? record["name"];
+  return typeof candidate === "string" && candidate.length > 0 ? candidate : fallback;
+}
+
+function toErrorView(value: unknown): JobViewError | undefined {
+  if (!isRecord(value)) return undefined;
+  if (typeof value["message"] !== "string") return undefined;
+  const code = typeof value["code"] === "string"
+    ? value["code"]
+    : typeof value["name"] === "string"
+      ? value["name"]
+      : DEFAULT_JOB_ERROR_CODE;
+  const result: JobViewError = { code, message: value["message"] };
+  if (typeof value["path"] === "string") result.path = value["path"];
+  if (typeof value["name"] === "string") result.name = value["name"];
+  if (typeof value["stack"] === "string") result.stack = value["stack"];
+  return result;
+}
+
+export function normalizeJobView(raw: unknown, jobId: string, location: string): JobView {
+  const record = isRecord(raw) ? raw : {};
+
+  const tasks = toJobViewTasks(record["tasks"]);
+  const taskList = Object.values(tasks);
+
+  const fromState = jobStateToStatus(record["state"]);
+  let status: JobStatusValue;
+  if (fromState !== null) {
+    status = resolveStateStatus(fromState, taskList, location);
+  } else if (isRecognizedStatusValue(record["status"])) {
+    status = normalizeJobStatus(record["status"]);
+  } else {
+    const warnKey = `${location}:${jobId}`;
+    if (!warnedFallbackKeys.has(warnKey)) {
+      warnedFallbackKeys.add(warnKey);
+      console.warn(
+        `normalizeJobView: no valid state or status for ${warnKey}; deriving job status from tasks.`,
+      );
+    }
+    status = deriveJobStatusFromTasks(taskList);
+  }
+
+  const title = toTitle(record, jobId);
+  const lastUpdated = toStringOrNull(record["lastUpdated"]);
+
+  const result: JobView = {
+    id: jobId,
+    jobId,
+    name: title,
+    title,
+    status,
+    progress: getProgress(record["progress"], taskList),
+    createdAt: toStringOrNull(record["createdAt"]),
+    updatedAt: toStringOrNull(record["updatedAt"]) ?? lastUpdated,
+    location,
+    tasks,
+    files: isRecord(record["files"]) ? record["files"] : {},
+    costs: getCosts(record),
+  };
+
+  if (lastUpdated !== null) result.lastUpdated = lastUpdated;
+  if ("gate" in record) result.gate = toGateInfo(record["gate"]);
+  if (typeof record["pipeline"] === "string") result.pipeline = record["pipeline"];
+  if (typeof record["pipelineLabel"] === "string") result.pipelineLabel = record["pipelineLabel"];
+  if (isRecord(record["pipelineConfig"])) result.pipelineConfig = record["pipelineConfig"];
+  if ("current" in record) result.current = record["current"];
+  if ("currentStage" in record) result.currentStage = record["currentStage"];
+  if (Array.isArray(record["warnings"])) {
+    result.warnings = record["warnings"].filter((entry): entry is string => typeof entry === "string");
+  }
+  if (typeof record["readable"] === "boolean") result.readable = record["readable"];
+  const errorView = toErrorView(record["error"]);
+  if (errorView) result.error = errorView;
+  if ("errorContext" in record) result.errorContext = record["errorContext"];
+
+  return result;
+}
+
+export function __resetWarnedFallbackKeys(): void {
+  warnedFallbackKeys.clear();
+}

package/src/core/orchestrator.ts

@@ -98,6 +98,20 @@ interface SpawnedRunner {
   kill?: () => void;
 }
 
+function describeCleanupError(err: unknown): unknown {
+  if (!(err instanceof AggregateError)) return err;
+  return {
+    name: err.name,
+    message: err.message,
+    errors: err.errors.map((cause) => {
+      if (cause instanceof Error) {
+        return { name: cause.name, message: cause.message, stack: cause.stack };
+      }
+      return { message: String(cause) };
+    }),
+  };
+}
+
 /** Seed filename regex. Captures jobId from {jobId}-seed.json. */
 export const SEED_PATTERN = /^([A-Za-z0-9-_]+)-seed\.json$/;
 
@@ -120,6 +134,7 @@ import {
   tryAcquireJobSlot,
   updateJobSlotPid,
 } from "./job-concurrency";
+import { reconcileInterruptedJob, reconcileOrphanedCurrentJobs } from "./runner-liveness";
 
 /**
  * Normalize any path that may already include `pipeline-data` (or subdirs
@@ -184,7 +199,7 @@ export async function spawnRunner(
   running: Map<string, ChildHandle>,
   logger: ReturnType<typeof createLogger>,
   spawnFn: SpawnFn,
-  onExit?: (jobId: string) => void | Promise<void>,
+  onExit?: (jobId: string, result: ChildExitResult) => void | Promise<void>,
 ): Promise<void> {
   if (!seed.pipeline) {
     throw new Error(`seed.pipeline is required for job ${jobId}`);
@@ -236,9 +251,9 @@ export async function spawnRunner(
       });
       if (!onExit) return;
       try {
-        await onExit(jobId);
+        await onExit(jobId, result);
       } catch (err) {
-        logger.error(`child exit cleanup failed for job ${jobId}`, err);
+        logger.error(`child exit cleanup failed for job ${jobId}`, describeCleanupError(err));
       }
     })
     .catch((err) => {
@@ -251,11 +266,46 @@ export interface HandleChildExitOptions {
   jobId: string;
   triggerDrain: () => void;
   lockTimeoutMs?: number;
+  code?: number | null;
+  signal?: string | null;
+  releaseSlot?: (dataDir: string, jobId: string, lockTimeoutMs?: number) => Promise<void>;
+  reconcileJob?: typeof reconcileInterruptedJob;
 }
 
 export async function handleChildExit(opts: HandleChildExitOptions): Promise<void> {
+  const lockTimeoutMs = opts.lockTimeoutMs ?? getOrchestratorConfig().lockFileTimeout;
+  const releaseSlot = opts.releaseSlot ?? releaseJobSlot;
+  const reconcileJob = opts.reconcileJob ?? reconcileInterruptedJob;
   try {
-    await releaseJobSlot(opts.dataDir, opts.jobId, opts.lockTimeoutMs);
+    let releaseError: unknown;
+    let reconcileError: unknown;
+
+    try {
+      await releaseSlot(opts.dataDir, opts.jobId, lockTimeoutMs);
+    } catch (err) {
+      releaseError = err;
+    }
+
+    const jobDir = join(opts.dataDir, "current", opts.jobId);
+    try {
+      await reconcileJob({
+        dataDir: opts.dataDir,
+        jobId: opts.jobId,
+        jobDir,
+        source: "child-exit",
+        code: opts.code,
+        signal: opts.signal,
+        lockTimeoutMs,
+      });
+    } catch (err) {
+      reconcileError = err;
+    }
+
+    if (releaseError && reconcileError) {
+      throw new AggregateError([releaseError, reconcileError], "child exit cleanup failed");
+    }
+    if (releaseError) throw releaseError;
+    if (reconcileError) throw reconcileError;
   } finally {
     opts.triggerDrain();
   }
@@ -685,6 +735,21 @@ export function startOrchestrator(opts: OrchestratorOptions): Promise<Orchestrat
     .then(() => mkdir(dirs.rejected, { recursive: true }))
     .then(() => mkdir(dirs.staging, { recursive: true }))
     .then(() => harnessPreflight(logger))
+    .then(async () => {
+      try {
+        const summary = await reconcileOrphanedCurrentJobs({
+          dataDir: dirs.dataDir,
+          currentDir: dirs.current,
+          lockTimeoutMs,
+          staleAfterMs: getOrchestratorConfig().staleRunningGraceMs,
+        });
+        logger.log("boot reconciliation complete", summary);
+      } catch (err) {
+        logger.warn(
+          `boot reconciliation failed; continuing: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+    })
     .then(() => new Promise<OrchestratorHandle>((resolve, reject) => {
       const running = new Map<string, ChildHandle>();
       const spawnFn = opts.spawn ?? createDefaultSpawn();
@@ -721,9 +786,16 @@ export function startOrchestrator(opts: OrchestratorOptions): Promise<Orchestrat
         })();
       };
 
-      const onChildExit = (jobId: string): Promise<void> => {
+      const onChildExit = (jobId: string, result: ChildExitResult): Promise<void> => {
         if (stopped) return Promise.resolve();
-        return handleChildExit({ dataDir: dirs.dataDir, jobId, triggerDrain, lockTimeoutMs });
+        return handleChildExit({
+          dataDir: dirs.dataDir,
+          jobId,
+          triggerDrain,
+          lockTimeoutMs,
+          code: result.code,
+          signal: result.signal,
+        });
       };
 
       const spawnRunnerForJob = async (jobId: string, seed: SeedData): Promise<SpawnedRunner> => {

package/src/core/pipeline-runner.ts

@@ -5,7 +5,7 @@ import { unlinkSync } from "node:fs";
 import { getConfig, getPipelineConfig } from "./config";
 import { validatePipelineOrThrow } from "./validation";
 import { loadFreshModule } from "./module-loader";
-import { atomicWrite, writeJobStatus, type GateInfo } from "./status-writer";
+import { atomicWrite, writeJobStatus, flushJobStatus, StatusCorruptError, type GateInfo } from "./status-writer";
 import { decideTransition } from "./lifecycle-policy";
 import { runPipeline } from "./task-runner";
 import type { AuditLogEntry, PipelineResult } from "./task-runner";
@@ -672,8 +672,14 @@ export async function runPipelineJob(jobId: string): Promise<void> {
   while (true) {
     const pipeline = await loadPipeline(config.pipelineJsonPath);
     const pipelineTasks = normalizePipelineTasks(pipeline);
-    const statusText = await Bun.file(config.statusPath).text();
-    const status = JSON.parse(statusText) as { tasks: Record<string, { state?: string }> };
+    let status: { tasks: Record<string, { state?: string }> };
+    try {
+      const statusText = await Bun.file(config.statusPath).text();
+      status = JSON.parse(statusText) as { tasks: Record<string, { state?: string }> };
+    } catch (parseErr: unknown) {
+      if (parseErr instanceof SyntaxError) throw new StatusCorruptError(config.statusPath, { cause: parseErr });
+      throw parseErr;
+    }
 
     let selectedEntry: PipelineTaskEntry | null = null;
     let selectedTaskState: TaskStateValue = TaskState.PENDING;
@@ -1092,6 +1098,49 @@ export async function runPipelineJob(jobId: string): Promise<void> {
   }
   await releaseJobSlotBestEffort(dataDir, jobId);
   } catch (err) {
+    if (err instanceof StatusCorruptError) {
+      console.error(`Status corruption detected for job "${jobId}": ${err.statusPath} — refusing to overwrite`);
+      if (workDir !== undefined) {
+        try {
+          const failureIO = createTaskFileIO({
+            workDir,
+            taskName: "orchestrator",
+            trackTaskFiles: false,
+            getStage: () => "runPipelineJob",
+            statusPath: join(workDir, "tasks-status.json"),
+          });
+          const failureLogName = generateLogName(
+            "orchestrator",
+            "runPipelineJob",
+            LogEvent.FAILURE_DETAILS,
+            LogFileExtension.JSON,
+          );
+          await failureIO.writeLog(
+            failureLogName,
+            JSON.stringify({
+              type: "status_corrupt",
+              jobId,
+              statusPath: err.statusPath,
+              message: err.message,
+            }, null, 2),
+          );
+        } catch {
+          // Do not mask the original error if log-write fails
+        }
+        try {
+          await cleanupPidFile(workDir);
+        } catch {
+          // Do not mask the original failure if PID cleanup fails
+        }
+      }
+      if (dataDir !== undefined) {
+        await releaseJobSlotBestEffort(dataDir, jobId);
+      }
+      process.exitCode = 1;
+      setTimeout(() => process.exit(1), 5000).unref();
+      process.exit(1);
+      return;
+    }
     const normalized = normalizeError(err);
     console.error(normalized.message);
     if (workDir !== undefined) {
@@ -1171,6 +1220,8 @@ export async function completeJob(
   // Create complete/ directory if needed
   await mkdir(config.completeDir, { recursive: true });
 
+  await flushJobStatus(config.workDir);
+
   // Move job directory from current/ to complete/
   await rename(config.workDir, destDir);