@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4

package/src/core/job-concurrency.ts

@@ -1,8 +1,8 @@
 import { mkdir, readdir, readFile, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { randomBytes } from "node:crypto";
+import { SEED_FILENAME_PATTERN } from "./seed-naming.ts";
 
-const SEED_FILENAME_PATTERN = /^([A-Za-z0-9][A-Za-z0-9-_]*)-seed\.json$/;
 const TEMP_LEASE_PATTERN = /\.json\.tmp\.[a-f0-9]+$/;
 const LOCK_POLL_INTERVAL_MS = 25;
 const DEFAULT_STALE_LEASE_TIMEOUT_MS = 30_000;
@@ -26,7 +26,7 @@ export interface QueuedJobSummary {
 export interface StaleJobSlot {
   jobId: string;
   slotPath: string;
-  reason: "missing_current_job" | "missing_pid" | "dead_pid" | "invalid_json";
+  reason: "missing_current_job" | "missing_pid" | "dead_pid" | "invalid_json" | "stale_runner";
 }
 
 export interface JobConcurrencyStatus {
@@ -50,6 +50,7 @@ export interface AcquireJobSlotOptions {
   source: JobSlotLease["source"];
   pid?: number | null;
   lockTimeoutMs?: number;
+  staleLeaseTimeoutMs?: number;
 }
 
 export interface ConcurrencyRuntimePaths {
@@ -140,11 +141,23 @@ async function removeLeaseFile(slotPath: string): Promise<void> {
   }
 }
 
+async function runnerPidMatches(dataDir: string, jobId: string, expectedPid: number): Promise<boolean> {
+  let raw: string;
+  try {
+    raw = await readFile(join(dataDir, "current", jobId, "runner.pid"), "utf-8");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
+    throw err;
+  }
+  const pid = Number(raw.trim());
+  return Number.isInteger(pid) && pid === expectedPid;
+}
+
 async function classifyLease(
   dataDir: string,
   slotPath: string,
   fileName: string,
-  lockTimeoutMs: number,
+  staleLeaseTimeoutMs: number,
 ): Promise<StaleJobSlot | null> {
   const fallbackJobId = fileName.replace(/\.json$/, "");
   let raw: string;
@@ -162,7 +175,7 @@ async function classifyLease(
   }
   const jobId = typeof parsed.jobId === "string" ? parsed.jobId : fallbackJobId;
   const acquiredMs = Date.parse(parsed.acquiredAt);
-  const leaseAgedOut = !Number.isFinite(acquiredMs) || Date.now() - acquiredMs >= lockTimeoutMs;
+  const leaseAgedOut = !Number.isFinite(acquiredMs) || Date.now() - acquiredMs >= staleLeaseTimeoutMs;
   if (!(await directoryExists(join(dataDir, "current", jobId)))) {
     // Grace window: drainPendingQueue acquires the slot before creating
     // current/<jobId>, so a fresh lease without a current dir is in-flight, not stale.
@@ -178,6 +191,9 @@ async function classifyLease(
   if (!isProcessAlive(parsed.pid)) {
     return { jobId, slotPath, reason: "dead_pid" };
   }
+  if (leaseAgedOut && !(await runnerPidMatches(dataDir, jobId, parsed.pid))) {
+    return { jobId, slotPath, reason: "stale_runner" };
+  }
   return null;
 }
 
@@ -191,7 +207,7 @@ async function listRunningJobFileNames(runningJobsDir: string): Promise<string[]
   }
 }
 
-async function removeStaleTempLeaseFiles(runningJobsDir: string, names: string[], lockTimeoutMs: number): Promise<void> {
+async function removeStaleTempLeaseFiles(runningJobsDir: string, names: string[], staleLeaseTimeoutMs: number): Promise<void> {
   await Promise.all(
     names
       .filter((name) => TEMP_LEASE_PATTERN.test(name))
@@ -199,7 +215,7 @@ async function removeStaleTempLeaseFiles(runningJobsDir: string, names: string[]
         const tempPath = join(runningJobsDir, name);
         try {
           const stats = await stat(tempPath);
-          if (Date.now() - stats.mtimeMs >= lockTimeoutMs) await removeLeaseFile(tempPath);
+          if (Date.now() - stats.mtimeMs >= staleLeaseTimeoutMs) await removeLeaseFile(tempPath);
         } catch (err) {
           if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
         }
@@ -209,19 +225,19 @@ async function removeStaleTempLeaseFiles(runningJobsDir: string, names: string[]
 
 async function collectLeaseState(
   dataDir: string,
-  lockTimeoutMs: number,
+  staleLeaseTimeoutMs: number,
   prune: boolean,
 ): Promise<{ activeJobs: JobSlotLease[]; staleSlots: StaleJobSlot[] }> {
   const { runningJobsDir } = getConcurrencyRuntimePaths(dataDir);
   const names = await listRunningJobFileNames(runningJobsDir);
-  if (prune) await removeStaleTempLeaseFiles(runningJobsDir, names, lockTimeoutMs);
+  if (prune) await removeStaleTempLeaseFiles(runningJobsDir, names, staleLeaseTimeoutMs);
 
   const staleSlots: StaleJobSlot[] = [];
   const stalePaths = new Set<string>();
   await Promise.all(names.map(async (name) => {
     if (!name.endsWith(".json") || TEMP_LEASE_PATTERN.test(name)) return;
     const slotPath = join(runningJobsDir, name);
-    const result = await classifyLease(dataDir, slotPath, name, lockTimeoutMs);
+    const result = await classifyLease(dataDir, slotPath, name, staleLeaseTimeoutMs);
     if (result) {
       staleSlots.push(result);
       stalePaths.add(slotPath);
@@ -240,11 +256,13 @@ async function collectLeaseState(
 export async function pruneStaleJobSlots(
   dataDir: string,
   lockTimeoutMs: number,
+  staleLeaseTimeoutMs?: number,
 ): Promise<StaleJobSlot[]> {
+  const staleAgeMs = staleLeaseTimeoutMs ?? lockTimeoutMs;
   return withRuntimeLock(dataDir, lockTimeoutMs, async () => {
-    const { staleSlots } = await collectLeaseState(dataDir, lockTimeoutMs, true);
+    const { staleSlots } = await collectLeaseState(dataDir, staleAgeMs, true);
     return staleSlots;
-  });
+  }, staleAgeMs);
 }
 
 function delay(ms: number): Promise<void> {
@@ -252,11 +270,13 @@ function delay(ms: number): Promise<void> {
 }
 
 // mkdir with recursive:false is atomic — exactly one caller wins the create when racing.
-async function withRuntimeLock<T>(
+export async function withRuntimeLock<T>(
   dataDir: string,
   lockTimeoutMs: number,
   fn: () => Promise<T>,
+  staleLeaseTimeoutMs?: number,
 ): Promise<T> {
+  const staleAgeMs = staleLeaseTimeoutMs ?? lockTimeoutMs;
   const { lockDir, runtimeDir } = getConcurrencyRuntimePaths(dataDir);
   const ownerPath = join(lockDir, "owner.json");
   await mkdir(runtimeDir, { recursive: true });
@@ -281,7 +301,7 @@ async function withRuntimeLock<T>(
         const raw = await readFile(ownerPath, "utf-8");
         const owner = JSON.parse(raw) as { acquiredAt?: unknown; pid?: unknown };
         const ownerAcquiredMs = typeof owner.acquiredAt === "string" ? Date.parse(owner.acquiredAt) : NaN;
-        const ownerTimedOut = Number.isFinite(ownerAcquiredMs) && Date.now() - ownerAcquiredMs >= lockTimeoutMs;
+        const ownerTimedOut = Number.isFinite(ownerAcquiredMs) && Date.now() - ownerAcquiredMs >= staleAgeMs;
         if (typeof owner.pid === "number" && !isProcessAlive(owner.pid)) {
           await rm(lockDir, { recursive: true, force: true });
           continue;
@@ -293,7 +313,7 @@ async function withRuntimeLock<T>(
       } catch (ownerErr) {
         if ((ownerErr as NodeJS.ErrnoException).code === "ENOENT") {
           const stats = await stat(lockDir);
-          if (Date.now() - stats.mtimeMs >= lockTimeoutMs) {
+          if (Date.now() - stats.mtimeMs >= staleAgeMs) {
             await rm(lockDir, { recursive: true, force: true });
             continue;
           }
@@ -321,10 +341,10 @@ async function withRuntimeLock<T>(
 async function buildStatus(
   dataDir: string,
   maxConcurrentJobs: number,
-  lockTimeoutMs: number,
+  staleLeaseTimeoutMs: number,
   prune: boolean,
 ): Promise<JobConcurrencyStatus> {
-  const { activeJobs, staleSlots } = await collectLeaseState(dataDir, lockTimeoutMs, prune);
+  const { activeJobs, staleSlots } = await collectLeaseState(dataDir, staleLeaseTimeoutMs, prune);
   const queuedJobs = await listQueuedSeeds(dataDir);
   return {
     limit: maxConcurrentJobs,
@@ -347,6 +367,16 @@ async function readLease(slotPath: string): Promise<JobSlotLease | null> {
   }
 }
 
+export async function readJobSlotLease(dataDir: string, jobId: string): Promise<JobSlotLease | null> {
+  const { runningJobsDir } = getConcurrencyRuntimePaths(dataDir);
+  return readLease(join(runningJobsDir, `${jobId}.json`));
+}
+
+export async function listActiveJobLeases(dataDir: string): Promise<JobSlotLease[]> {
+  const { runningJobsDir } = getConcurrencyRuntimePaths(dataDir);
+  return readActiveLeases(runningJobsDir);
+}
+
 async function writeLeaseAtomic(slotPath: string, lease: JobSlotLease): Promise<void> {
   const tmpPath = `${slotPath}.tmp.${randomBytes(8).toString("hex")}`;
   await writeFile(tmpPath, JSON.stringify(lease));
@@ -380,10 +410,11 @@ export async function tryAcquireJobSlot(
 ): Promise<AcquireJobSlotResult> {
   const { dataDir, jobId, maxConcurrentJobs, source } = options;
   const lockTimeoutMs = options.lockTimeoutMs ?? DEFAULT_STALE_LEASE_TIMEOUT_MS;
+  const staleLeaseTimeoutMs = options.staleLeaseTimeoutMs ?? lockTimeoutMs;
   const { runningJobsDir } = getConcurrencyRuntimePaths(dataDir);
   await mkdir(runningJobsDir, { recursive: true });
   return withRuntimeLock(dataDir, lockTimeoutMs, async () => {
-    const status = await buildStatus(dataDir, maxConcurrentJobs, lockTimeoutMs, true);
+    const status = await buildStatus(dataDir, maxConcurrentJobs, staleLeaseTimeoutMs, true);
     const activeJobs = status.activeJobs;
     const slotPath = join(runningJobsDir, `${jobId}.json`);
     if (activeJobs.some((l) => l.jobId === jobId)) {
@@ -401,7 +432,7 @@ export async function tryAcquireJobSlot(
     };
     await writeLeaseAtomic(slotPath, lease);
     return { ok: true, lease };
-  });
+  }, staleLeaseTimeoutMs);
 }
 
 export async function updateJobSlotPid(
@@ -438,9 +469,14 @@ export async function getJobConcurrencyStatus(
   dataDir: string,
   maxConcurrentJobs: number,
   lockTimeoutMs: number,
+  staleLeaseTimeoutMs?: number,
 ): Promise<JobConcurrencyStatus> {
-  return withRuntimeLock(dataDir, lockTimeoutMs, () =>
-    buildStatus(dataDir, maxConcurrentJobs, lockTimeoutMs, true),
+  const staleAgeMs = staleLeaseTimeoutMs ?? lockTimeoutMs;
+  return withRuntimeLock(
+    dataDir,
+    lockTimeoutMs,
+    () => buildStatus(dataDir, maxConcurrentJobs, staleAgeMs, true),
+    staleAgeMs,
   );
 }
 

package/src/core/job-submission.ts

@@ -0,0 +1,133 @@
+import path from "node:path";
+import { mkdir, rename } from "node:fs/promises";
+import {
+  getPendingSeedPath,
+  getPipelineDataDir,
+  resolveWorkspaceRoot,
+} from "../config/paths.ts";
+import { resolveWithin } from "../utils/path-containment.ts";
+import { getPipelineConfig } from "./config.ts";
+
+export interface SubmitSuccessResult {
+  success: true;
+  jobId: string;
+  jobName: string;
+}
+
+export interface SubmitFailureResult {
+  success: false;
+  message: string;
+}
+
+export type SubmitResult = SubmitSuccessResult | SubmitFailureResult;
+
+export interface SubmitSeedOptions {
+  root: string;
+  seedObject: unknown;
+}
+
+export interface SubmitUploadedSeedOptions {
+  root: string;
+  seedObject: unknown;
+  artifacts?: Array<{ filename: string; content: Uint8Array }>;
+}
+
+interface SubmitInternalOptions {
+  root: string;
+  seedObject: unknown;
+  artifacts?: Array<{ filename: string; content: Uint8Array }>;
+}
+
+function validateSeed(seedObject: unknown, rootDir: string): SubmitResult | null {
+  if (typeof seedObject !== "object" || seedObject === null || Array.isArray(seedObject)) {
+    return { success: false, message: "seed must be a JSON object" };
+  }
+  const seed = seedObject as Record<string, unknown>;
+  if (typeof seed["pipeline"] !== "string" || seed["pipeline"].length === 0) {
+    return { success: false, message: "seed.pipeline must be a non-empty string" };
+  }
+  if (seed["name"] !== undefined) {
+    if (typeof seed["name"] !== "string" || seed["name"].length === 0) {
+      return { success: false, message: "seed.name must be a non-empty string if provided" };
+    }
+  }
+  try {
+    getPipelineConfig(seed["pipeline"] as string, rootDir);
+  } catch (err) {
+    return { success: false, message: (err as Error).message };
+  }
+  return null;
+}
+
+async function stageArtifacts(
+  rootDir: string,
+  jobId: string,
+  artifacts: Array<{ filename: string; content: Uint8Array }>,
+): Promise<SubmitFailureResult | null> {
+  const stagingDir = path.join(getPipelineDataDir(rootDir), "staging", jobId);
+  const resolved: string[] = [];
+  for (const artifact of artifacts) {
+    const target = resolveWithin(stagingDir, artifact.filename);
+    if (target === null) {
+      return { success: false, message: `unsafe artifact path: ${artifact.filename}` };
+    }
+    resolved.push(target);
+  }
+  for (let i = 0; i < artifacts.length; i++) {
+    await mkdir(path.dirname(resolved[i]!), { recursive: true });
+    await Bun.write(resolved[i]!, artifacts[i]!.content);
+  }
+  return null;
+}
+
+// Atomic visibility to the watcher, not durability (no fsync).
+// Crash durability is tracked separately as backlog #40.
+async function writePendingSeedAtomic(
+  rootDir: string,
+  jobId: string,
+  seedObject: unknown,
+): Promise<void> {
+  const pendingPath = getPendingSeedPath(rootDir, jobId);
+  // Suffix `.tmp` keeps the temp file outside the orchestrator's `pending/*.json` watch glob,
+  // so the watcher never observes a partial write as a seed.
+  const tmp = `${pendingPath}.${Date.now()}.tmp`;
+  await mkdir(path.dirname(pendingPath), { recursive: true });
+  await Bun.write(tmp, JSON.stringify(seedObject, null, 2));
+  await rename(tmp, pendingPath);
+}
+
+async function submit(opts: SubmitInternalOptions): Promise<SubmitResult> {
+  const rootDir = resolveWorkspaceRoot(opts.root);
+
+  const validationFailure = validateSeed(opts.seedObject, rootDir);
+  if (validationFailure) return validationFailure;
+
+  const seed = opts.seedObject as Record<string, unknown>;
+  const jobId = crypto.randomUUID();
+  const jobName = typeof seed["name"] === "string" ? seed["name"] : jobId;
+
+  if (opts.artifacts && opts.artifacts.length > 0) {
+    const stageFailure = await stageArtifacts(rootDir, jobId, opts.artifacts);
+    if (stageFailure) return stageFailure;
+  }
+
+  // Seed is written last so the orchestrator trigger never appears before
+  // its artifacts are fully staged (spec §4 step 4; cross-phase invariant 2).
+  await writePendingSeedAtomic(rootDir, jobId, opts.seedObject);
+
+  return { success: true, jobId, jobName };
+}
+
+export async function submitSeed(opts: SubmitSeedOptions): Promise<SubmitResult> {
+  return submit({ root: opts.root, seedObject: opts.seedObject });
+}
+
+export async function submitUploadedSeed(
+  opts: SubmitUploadedSeedOptions,
+): Promise<SubmitResult> {
+  return submit({
+    root: opts.root,
+    seedObject: opts.seedObject,
+    artifacts: opts.artifacts,
+  });
+}