@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.3

package/src/core/runner-liveness.ts

@@ -0,0 +1,276 @@
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+
+import { deriveJobStatusFromTasks } from "../config/statuses";
+import { createLogger } from "./logger";
+import { readJobStatusResult, writeJobStatus } from "./status-writer";
+import type { StatusSnapshot } from "./status-writer";
+import { readJobSlotLease, withRuntimeLock } from "./job-concurrency";
+import type { JobSlotLease } from "./job-concurrency";
+
+export const STALE_RUNNING_GRACE_MS = 30_000;
+
+export interface StaleRunningInput {
+  status: StatusSnapshot;
+  pid: number | null;
+  pidAlive: boolean;
+  nowMs: number;
+  staleAfterMs: number;
+}
+
+export type StaleRunningDecision =
+  | { stale: false; reason: "not_running" | "live_pid" | "fresh_status" }
+  | { stale: true; runningTaskId: string | null; ageMs: number };
+
+export type ReconcileSource = "child-exit" | "boot-sweep";
+
+export type ReconcileReason =
+  | "marked-failed"
+  | "not-running"
+  | "no-running-task"
+  | "live-pid"
+  | "live-lease"
+  | "fresh-status"
+  | "unreadable";
+
+export interface ReconcileResult {
+  reconciled: boolean;
+  taskId?: string;
+  reason: ReconcileReason;
+}
+
+export interface ReconcileOptions {
+  dataDir: string;
+  jobId: string;
+  jobDir: string;
+  source: ReconcileSource;
+  lockTimeoutMs: number;
+  code?: number | null;
+  signal?: string | null;
+  requireStaleStatus?: boolean;
+  staleAfterMs?: number;
+  now?: () => number;
+  isAlive?: (pid: number) => boolean;
+}
+
+export interface SweepOptions {
+  dataDir: string;
+  currentDir: string;
+  lockTimeoutMs: number;
+  staleAfterMs?: number;
+  now?: () => number;
+  isAlive?: (pid: number) => boolean;
+}
+
+export interface SweepResult {
+  scanned: number;
+  reconciled: string[];
+}
+
+export async function readRunnerPid(jobDir: string): Promise<number | null> {
+  try {
+    const content = await Bun.file(path.join(jobDir, "runner.pid")).text();
+    const pid = parseInt(content.trim(), 10);
+    return Number.isNaN(pid) ? null : pid;
+  } catch {
+    return null;
+  }
+}
+
+export function isProcessAlive(pid: number): boolean {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return (err as NodeJS.ErrnoException).code === "EPERM";
+  }
+}
+
+function getTaskDerivedStatus(snapshot: StatusSnapshot): string {
+  return deriveJobStatusFromTasks(
+    Object.values(snapshot.tasks).map((task) => ({ state: task.state })),
+  );
+}
+
+export function findRunningTaskId(snapshot: StatusSnapshot): string | null {
+  if (snapshot.current && snapshot.tasks[snapshot.current]?.state === "running") {
+    return snapshot.current;
+  }
+  return Object.keys(snapshot.tasks).find((taskId) => snapshot.tasks[taskId]?.state === "running") ?? null;
+}
+
+export function classifyStaleRunningStatus(input: StaleRunningInput): StaleRunningDecision {
+  if (getTaskDerivedStatus(input.status) !== "running") {
+    return { stale: false, reason: "not_running" };
+  }
+  if (input.pid !== null && input.pidAlive) {
+    return { stale: false, reason: "live_pid" };
+  }
+  const updatedMs = Date.parse(input.status.lastUpdated);
+  if (!Number.isFinite(updatedMs)) {
+    return { stale: true, runningTaskId: findRunningTaskId(input.status), ageMs: 0 };
+  }
+  const ageMs = input.nowMs - updatedMs;
+  if (ageMs < input.staleAfterMs) {
+    return { stale: false, reason: "fresh_status" };
+  }
+  return { stale: true, runningTaskId: findRunningTaskId(input.status), ageMs };
+}
+
+const reconcileLogger = createLogger("runner-liveness");
+
+function isLeaseLiveOrInFlight(
+  lease: JobSlotLease,
+  nowMs: number,
+  lockTimeoutMs: number,
+  isAlive: (pid: number) => boolean,
+): boolean {
+  if (typeof lease.pid === "number" && lease.pid > 0) {
+    return isAlive(lease.pid);
+  }
+  const acquiredMs = Date.parse(lease.acquiredAt);
+  if (!Number.isFinite(acquiredMs)) return false;
+  return nowMs - acquiredMs < lockTimeoutMs;
+}
+
+function isFreshStatus(snapshot: StatusSnapshot, nowMs: number, staleAfterMs: number): boolean {
+  const updatedMs = Date.parse(snapshot.lastUpdated);
+  return Number.isFinite(updatedMs) && nowMs - updatedMs < staleAfterMs;
+}
+
+export async function reconcileInterruptedJob(opts: ReconcileOptions): Promise<ReconcileResult> {
+  const {
+    dataDir,
+    jobId,
+    jobDir,
+    source,
+    lockTimeoutMs,
+    code,
+    signal,
+    requireStaleStatus = false,
+    staleAfterMs = STALE_RUNNING_GRACE_MS,
+    now = Date.now,
+    isAlive = isProcessAlive,
+  } = opts;
+
+  return withRuntimeLock(dataDir, lockTimeoutMs, async () => {
+    const statusResult = await readJobStatusResult(jobDir);
+    if (!statusResult.ok) {
+      return { reconciled: false, reason: "unreadable" } satisfies ReconcileResult;
+    }
+    const snapshot = statusResult.value;
+
+    const derived = getTaskDerivedStatus(snapshot);
+    if (derived !== "running" && snapshot.state !== "running") {
+      return { reconciled: false, reason: "not-running" } satisfies ReconcileResult;
+    }
+
+    const pid = await readRunnerPid(jobDir);
+    if (pid !== null && isAlive(pid)) {
+      return { reconciled: false, reason: "live-pid" } satisfies ReconcileResult;
+    }
+
+    const lease = await readJobSlotLease(dataDir, jobId);
+    if (lease && isLeaseLiveOrInFlight(lease, now(), lockTimeoutMs, isAlive)) {
+      return { reconciled: false, reason: "live-lease" } satisfies ReconcileResult;
+    }
+
+    if (requireStaleStatus && derived === "running") {
+      const decision = classifyStaleRunningStatus({
+        status: snapshot,
+        pid,
+        pidAlive: false,
+        nowMs: now(),
+        staleAfterMs,
+      });
+      if (!decision.stale) {
+        return {
+          reconciled: false,
+          reason: decision.reason === "not_running" ? "not-running" : "fresh-status",
+        } satisfies ReconcileResult;
+      }
+    } else if (requireStaleStatus && snapshot.state === "running") {
+      if (isFreshStatus(snapshot, now(), staleAfterMs)) {
+        return { reconciled: false, reason: "fresh-status" } satisfies ReconcileResult;
+      }
+    }
+
+    const taskId = findRunningTaskId(snapshot);
+    const nowMs = now();
+    await writeJobStatus(jobDir, (s) => {
+      if (taskId !== null) {
+        const task = s.tasks[taskId];
+        if (task) {
+          task.state = "failed";
+          task.endedAt = new Date(nowMs).toISOString();
+          task.error = "runner exited without completing this task";
+          task.errorContext = { kind: "runner-interrupted", code, signal, source };
+        }
+      } else {
+        s.current = null;
+        s.currentStage = null;
+        s.error = {
+          name: "RunnerInterrupted",
+          message: "runner exited while the job was still marked running",
+        };
+        s.errorContext = { kind: "runner-interrupted", code, signal, source };
+      }
+      s.state = "failed";
+    });
+
+    reconcileLogger.log("reconciled interrupted job", { jobId, taskId, pid, code, signal, path: source });
+
+    return taskId === null
+      ? { reconciled: true, reason: "marked-failed" } satisfies ReconcileResult
+      : { reconciled: true, taskId, reason: "marked-failed" } satisfies ReconcileResult;
+  });
+}
+
+function isErrnoException(value: unknown): value is NodeJS.ErrnoException {
+  return value instanceof Error && typeof (value as NodeJS.ErrnoException).code === "string";
+}
+
+export async function reconcileOrphanedCurrentJobs(opts: SweepOptions): Promise<SweepResult> {
+  const { dataDir, currentDir, lockTimeoutMs, staleAfterMs, now, isAlive } = opts;
+
+  let entries: string[];
+  try {
+    const dirents = await readdir(currentDir, { withFileTypes: true });
+    entries = dirents.filter((entry) => entry.isDirectory()).map((entry) => entry.name);
+  } catch (err) {
+    if (isErrnoException(err) && err.code === "ENOENT") {
+      return { scanned: 0, reconciled: [] };
+    }
+    throw err;
+  }
+
+  const reconciled: string[] = [];
+  for (const entry of entries) {
+    const jobId = entry;
+    const jobDir = path.join(currentDir, entry);
+    const result = await reconcileInterruptedJob({
+      dataDir,
+      jobId,
+      jobDir,
+      source: "boot-sweep",
+      requireStaleStatus: true,
+      staleAfterMs,
+      lockTimeoutMs,
+      now,
+      isAlive,
+    });
+    if (result.reconciled) {
+      reconciled.push(jobId);
+    }
+  }
+
+  if (entries.length > 0) {
+    reconcileLogger.log("reconciled orphaned current jobs", {
+      scanned: entries.length,
+      reconciled: reconciled.length,
+    });
+  }
+
+  return { scanned: entries.length, reconciled };
+}

package/src/core/status-writer.ts

@@ -199,18 +199,37 @@ export async function flushJobStatus(jobDir: string): Promise<void> {
 
 const writeQueues = new Map<string, Promise<StatusSnapshot>>();
 
-export async function readJobStatus(jobDir: string): Promise<StatusSnapshot | null> {
+export type StatusReadResult =
+  | { ok: true; value: StatusSnapshot }
+  | { ok: false; reason: "missing" | "corrupt" };
+
+export async function readJobStatusResult(jobDir: string): Promise<StatusReadResult> {
   if (typeof jobDir !== "string" || jobDir.length === 0) {
     throw new Error("jobDir must be a non-empty string");
   }
 
   const statusPath = join(jobDir, STATUS_FILENAME);
   try {
-    const text = await Bun.file(statusPath).text();
-    const raw = JSON.parse(text);
-    return validateStatusSnapshot(raw, jobDir);
+    const raw = JSON.parse(await Bun.file(statusPath).text());
+    return { ok: true, value: validateStatusSnapshot(raw, jobDir) };
+  } catch (err: unknown) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return { ok: false, reason: "missing" };
+    if (err instanceof SyntaxError) return { ok: false, reason: "corrupt" };
+    throw err;
+  }
+}
+
+/** @deprecated conflates missing and corrupt as null; use readJobStatusResult. */
+export async function readJobStatus(jobDir: string): Promise<StatusSnapshot | null> {
+  if (typeof jobDir !== "string" || jobDir.length === 0) {
+    throw new Error("jobDir must be a non-empty string");
+  }
+
+  try {
+    const result = await readJobStatusResult(jobDir);
+    return result.ok ? result.value : null;
   } catch (err) {
-    console.warn(`readJobStatus: could not read ${statusPath}`, err);
+    console.warn(`readJobStatus: could not read status for ${jobDir}`, err);
     return null;
   }
 }

package/src/ui/client/__tests__/job-adapter.test.ts

@@ -227,6 +227,38 @@ describe("job adapter", () => {
     expect(job.__warnings).toEqual(["Unsupported task collection shape"]);
   });
 
+  it("passes readable:false and the full error payload through to the normalized summary", () => {
+    const error = {
+      code: "INVALID_JSON",
+      message: "Status file is unreadable. Repair or delete tasks-status.json for job \"job-1\" to recover.",
+      path: "pipeline-data/current/job-1/tasks-status.json",
+    };
+
+    const job = adaptJobSummary({
+      jobId: "job-1",
+      status: "error",
+      tasks: {},
+      readable: false,
+      error,
+    });
+
+    expect(job.readable).toBe(false);
+    expect(job.error).toEqual(error);
+    expect(job.error?.path).toBe(error.path);
+  });
+
+  it("omits readable and error when the input does not provide them", () => {
+    const job = adaptJobSummary({
+      jobId: "job-1",
+      tasks: { build: { state: "done" } },
+    });
+
+    expect(job.readable).toBeUndefined();
+    expect(Object.hasOwn(job, "readable")).toBe(false);
+    expect(job.error).toBeUndefined();
+    expect(Object.hasOwn(job, "error")).toBe(false);
+  });
+
   it("disables actions for running jobs", () => {
     const job = adaptJobSummary({
       jobId: "job-1",
@@ -270,4 +302,50 @@ describe("job adapter", () => {
 
     expect(deriveAllowedActions(job, ["build", "test"])).toEqual({ start: true, restart: true });
   });
+
+  it("adapts a canonical failed job to status 'failed' and displayCategory 'errors'", () => {
+    const job = adaptJobSummary({
+      jobId: "job-1",
+      status: "failed",
+      tasks: { build: { state: "done" } },
+    });
+
+    expect(job.status).toBe("failed");
+    expect(job.displayCategory).toBe("errors");
+  });
+
+  it("tolerates legacy status 'error' and emits canonical 'failed'", () => {
+    const job = adaptJobSummary({
+      jobId: "job-1",
+      status: "error",
+      tasks: { build: { state: "done" } },
+    });
+
+    expect(job.status).toBe("failed");
+    expect(job.displayCategory).toBe("errors");
+  });
+
+  it("uses 'task-0' and 'task-1' fallback ids for unnamed array entries to match the server fixture", () => {
+    const tasks = normalizeTasks([
+      { state: "done" },
+      { state: "pending" },
+    ]);
+
+    expect(Object.keys(tasks).sort()).toEqual(["task-0", "task-1"]);
+    expect(tasks["task-0"]?.name).toBe("task-0");
+    expect(tasks["task-0"]?.state).toBe("done");
+    expect(tasks["task-1"]?.name).toBe("task-1");
+    expect(tasks["task-1"]?.state).toBe("pending");
+  });
+
+  it("uses explicit task names as object keys when array entries are named", () => {
+    const tasks = normalizeTasks([
+      { name: "build", state: "done" },
+      { name: "test", state: "pending" },
+    ]);
+
+    expect(Object.keys(tasks).sort()).toEqual(["build", "test"]);
+    expect(tasks["build"]?.name).toBe("build");
+    expect(tasks["test"]?.name).toBe("test");
+  });
 });

package/src/ui/client/adapters/job-adapter.ts

@@ -108,6 +108,14 @@ function getWarnings(rawTasks: unknown): string[] {
   return ["Unsupported task collection shape"];
 }
 
+function isErrorShape(value: unknown): value is { code: string; message: string; path?: string } {
+  if (!isRecord(value)) return false;
+  if (typeof value["code"] !== "string") return false;
+  if (typeof value["message"] !== "string") return false;
+  if ("path" in value && typeof value["path"] !== "string") return false;
+  return true;
+}
+
 function normalizeCostsSummary(costs: unknown): CostsSummary {
   if (!isRecord(costs)) return EMPTY_COSTS;
   return {
@@ -202,6 +210,8 @@ function adaptBaseJob(apiJob: Record<string, unknown>): NormalizedJobSummary {
     totalTokens: toNumber(apiJob["totalTokens"], normalizeCostsSummary(apiJob["costsSummary"]).totalTokens),
     displayCategory: getDisplayCategory(status),
     __warnings: getWarnings(rawTasks),
+    ...(typeof apiJob["readable"] === "boolean" ? { readable: apiJob["readable"] } : {}),
+    ...(isErrorShape(apiJob["error"]) ? { error: apiJob["error"] } : {}),
   };
 }
 

package/src/ui/client/types.ts

@@ -155,6 +155,8 @@ export interface NormalizedJobSummary {
   totalTokens?: number;
   displayCategory: string;
   __warnings?: string[];
+  readable?: boolean;
+  error?: { code: string; message: string; path?: string };
 }
 
 export interface TaskCostBreakdown {

package/src/ui/components/JobTable.tsx

@@ -65,6 +65,8 @@ export default function JobTable({
         <tbody className="divide-y divide-gray-200">
           {jobs.map((job) => {
             const validId = Boolean(job.id);
+            const readable = job.readable !== false;
+            const clickable = validId && readable;
             const taskMap = Array.isArray(job.tasks) ? Object.fromEntries(job.tasks.map((task) => [task.name, task] as const)) : job.tasks;
             const currentTaskId =
               typeof job.current === "string" ? job.current : job.current?.taskName;
@@ -77,35 +79,50 @@ export default function JobTable({
             return (
               <tr
                 key={job.jobId}
-                className={validId ? "cursor-pointer transition hover:bg-gray-50" : "cursor-not-allowed opacity-60"}
-                onClick={() => validId && onOpenJob(job.id)}
-                tabIndex={validId ? 0 : -1}
+                className={clickable ? "cursor-pointer transition hover:bg-gray-50" : "cursor-not-allowed opacity-60"}
+                onClick={() => clickable && onOpenJob(job.id)}
+                tabIndex={clickable ? 0 : -1}
                 onKeyDown={(event) => {
-                  if (validId && (event.key === "Enter" || event.key === " ")) onOpenJob(job.id);
+                  if (clickable && (event.key === "Enter" || event.key === " ")) onOpenJob(job.id);
                 }}
               >
                 <td className="px-4 py-4">
                   <div className="font-medium text-[#6d28d9]">{job.name}</div>
                   <div className="text-xs text-gray-500">{job.id}</div>
+                  {!readable && job.error ? <div className="text-xs text-red-600">{job.error.message}</div> : null}
                 </td>
                 <td className="px-4 py-4 text-sm text-gray-700">{job.pipelineLabel ?? job.pipeline ?? pipeline?.name ?? "—"}</td>
                 <td className="px-4 py-4">
-                  <Badge intent={getIntent(job.status)}>{job.status}</Badge>
+                  {readable ? (
+                    <Badge intent={getIntent(job.status)}>{job.status}</Badge>
+                  ) : (
+                    <Badge intent="red">Unreadable</Badge>
+                  )}
                 </td>
                 <td className="px-4 py-4 text-sm text-gray-700">{currentTask?.name ?? currentTaskId ?? "—"}</td>
                 <td className="px-4 py-4">
-                  <div className="flex min-w-[160px] items-center gap-3">
-                    <Progress value={job.progress} variant={job.status === "failed" ? "error" : job.status === "complete" ? "completed" : job.status === "running" ? "running" : "pending"} className="flex-1" />
-                    <span className="text-sm text-gray-600">{job.progress}%</span>
-                  </div>
+                  {readable ? (
+                    <div className="flex min-w-[160px] items-center gap-3">
+                      <Progress value={job.progress} variant={job.status === "failed" ? "error" : job.status === "complete" ? "completed" : job.status === "running" ? "running" : "pending"} className="flex-1" />
+                      <span className="text-sm text-gray-600">{job.progress}%</span>
+                    </div>
+                  ) : (
+                    "—"
+                  )}
                 </td>
                 <td className="px-4 py-4 text-sm text-gray-700">{countCompleted(job)} of {job.taskCount}</td>
                 <td className="px-4 py-4">
-                  <div className="text-sm text-gray-700">{totalCost > 0 ? formatCurrency4(totalCost) : "—"}</div>
-                  {totalTokens > 0 ? <div className="text-xs text-gray-500">{formatTokensCompact(totalTokens)}</div> : null}
+                  {readable ? (
+                    <>
+                      <div className="text-sm text-gray-700">{totalCost > 0 ? formatCurrency4(totalCost) : "—"}</div>
+                      {totalTokens > 0 ? <div className="text-xs text-gray-500">{formatTokensCompact(totalTokens)}</div> : null}
+                    </>
+                  ) : (
+                    "—"
+                  )}
                 </td>
                 <td className="px-4 py-4 text-sm text-gray-700">
-                  {startMs ? <TimerText startMs={startMs} endMs={endMs} granularity="second" /> : "—"}
+                  {readable ? (startMs ? <TimerText startMs={startMs} endMs={endMs} granularity="second" /> : "—") : "—"}
                 </td>
               </tr>
             );

package/src/ui/components/__tests__/JobTable.test.tsx

@@ -1,6 +1,6 @@
 import "./test-dom";
 
-import { render } from "@testing-library/react";
+import { fireEvent, render } from "@testing-library/react";
 import { MemoryRouter } from "react-router-dom";
 import { afterEach, expect, mock, test } from "bun:test";
 
@@ -53,3 +53,56 @@ test("JobTable renders waiting status badge", () => {
 
   expect(view.getByText("waiting")).toBeTruthy();
 });
+
+test("JobTable renders an unreadable job as a non-clickable row with badge and recovery text", () => {
+  const recoveryMessage =
+    'Status file is unreadable (INVALID_JSON). Repair or delete tasks-status.json for job "job-bad" to recover.';
+  const onOpenJob = mock((_jobId: string) => {});
+  const view = render(
+    <MemoryRouter>
+      <JobTable
+        jobs={[
+          makeJob({
+            id: "job-bad",
+            jobId: "job-bad",
+            name: "Bad Job",
+            status: "error",
+            readable: false,
+            error: { code: "INVALID_JSON", message: recoveryMessage },
+          }),
+        ]}
+        onOpenJob={onOpenJob}
+      />
+    </MemoryRouter>,
+  );
+
+  const row = view.container.querySelector("tbody tr");
+  expect(row).toBeTruthy();
+  expect(row?.getAttribute("tabindex")).toBe("-1");
+  expect(row?.className).toContain("cursor-not-allowed");
+
+  expect(view.getByText("Unreadable")).toBeTruthy();
+  expect(view.getByText(recoveryMessage)).toBeTruthy();
+
+  fireEvent.click(row!);
+  fireEvent.keyDown(row!, { key: "Enter" });
+  expect(onOpenJob).not.toHaveBeenCalled();
+});
+
+test("JobTable renders a normal job without readable or error as a clickable row", () => {
+  const onOpenJob = mock((_jobId: string) => {});
+  const view = render(
+    <MemoryRouter>
+      <JobTable jobs={[makeJob()]} onOpenJob={onOpenJob} />
+    </MemoryRouter>,
+  );
+
+  const row = view.container.querySelector("tbody tr");
+  expect(row).toBeTruthy();
+  expect(row?.getAttribute("tabindex")).toBe("0");
+  expect(row?.className).toContain("cursor-pointer");
+
+  fireEvent.click(row!);
+  expect(onOpenJob).toHaveBeenCalledTimes(1);
+  expect(onOpenJob).toHaveBeenCalledWith("job-1");
+});

package/src/ui/components/types.ts

@@ -73,6 +73,8 @@ export interface JobSummary {
   totalTokens?: number;
   displayCategory: DisplayCategory;
   gate?: GateInfo | null;
+  readable?: boolean;
+  error?: { code: string; message: string; path?: string };
 }
 
 export interface GateInfo {