@ryanfw/prompt-orchestration-pipeline 1.2.11 → 1.2.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ryanfw/prompt-orchestration-pipeline",
-  "version": "1.2.11",
+  "version": "1.2.12",
   "description": "A Prompt-orchestration pipeline (POP) is a framework for building, running, and experimenting with complex chains of LLM tasks.",
   "type": "module",
   "main": "src/ui/server/index.ts",

package/src/core/__tests__/orchestrator.test.ts

@@ -3,7 +3,13 @@ import { mkdtemp, mkdir, writeFile, utimes, rm, readdir, readFile } from "node:f
 import { existsSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { drainPendingQueue, handleChildExit } from "../orchestrator";
+import {
+  drainPendingQueue,
+  handleChildExit,
+  resolveDirs,
+  promoteSeedFile,
+  restoreSeedFile,
+} from "../orchestrator";
 import {
   getConcurrencyRuntimePaths,
   getJobConcurrencyStatus,
@@ -38,6 +44,81 @@ function fakeSpawnRunner(jobIdToPid: Map<string, number>): (jobId: string) => Pr
   };
 }
 
+describe("resolveDirs", () => {
+  test("returns a staging dir under the pipeline-data root", () => {
+    expect(resolveDirs("/x/pipeline-data").staging).toBe("/x/pipeline-data/staging");
+  });
+});
+
+describe("promoteSeedFile", () => {
+  test("moves staged artifacts into current/files/artifacts (byte-identical) then the seed, removing staging", async () => {
+    const dir = await setupDir("promote-staged-");
+    try {
+      const jobId = "job-promote";
+      const stagingJobDir = join(dir, "staging", jobId);
+      const currentJobDir = join(dir, "current", jobId);
+      const bytes = new Uint8Array(256);
+      for (let i = 0; i < 256; i++) bytes[i] = i;
+      await mkdir(join(stagingJobDir, "a"), { recursive: true });
+      await writeFile(join(stagingJobDir, "top.txt"), "top");
+      await writeFile(join(stagingJobDir, "a", "b.bin"), bytes);
+      const seedPath = await writeSeed(dir, jobId, { pipeline: "p" }, 1700000000);
+
+      await promoteSeedFile(seedPath, currentJobDir, stagingJobDir);
+
+      expect(existsSync(join(currentJobDir, "seed.json"))).toBe(true);
+      expect(await readFile(join(currentJobDir, "files", "artifacts", "top.txt"), "utf-8")).toBe("top");
+      const readBytes = new Uint8Array(await readFile(join(currentJobDir, "files", "artifacts", "a", "b.bin")));
+      expect(Array.from(readBytes)).toEqual(Array.from(bytes));
+      expect(existsSync(stagingJobDir)).toBe(false);
+      expect(existsSync(seedPath)).toBe(false);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("with no staging dir, produces only current/seed.json (no files/artifacts)", async () => {
+    const dir = await setupDir("promote-nostage-");
+    try {
+      const jobId = "job-nostage";
+      const currentJobDir = join(dir, "current", jobId);
+      const stagingJobDir = join(dir, "staging", jobId);
+      const seedPath = await writeSeed(dir, jobId, { pipeline: "p" }, 1700000000);
+
+      await promoteSeedFile(seedPath, currentJobDir, stagingJobDir);
+
+      expect(existsSync(join(currentJobDir, "seed.json"))).toBe(true);
+      expect(existsSync(join(currentJobDir, "files"))).toBe(false);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+});
+
+describe("restoreSeedFile", () => {
+  test("returns artifacts to staging, seed to pending, and removes current/", async () => {
+    const dir = await setupDir("restore-");
+    try {
+      const jobId = "job-restore";
+      const currentJobDir = join(dir, "current", jobId);
+      const stagingJobDir = join(dir, "staging", jobId);
+      const pendingSeedPath = join(dir, "pending", `${jobId}-seed.json`);
+      await mkdir(join(dir, "staging"), { recursive: true });
+      await mkdir(join(currentJobDir, "files", "artifacts"), { recursive: true });
+      await writeFile(join(currentJobDir, "files", "artifacts", "x.bin"), "data");
+      await writeFile(join(currentJobDir, "seed.json"), JSON.stringify({ pipeline: "p" }));
+
+      await restoreSeedFile(currentJobDir, pendingSeedPath, stagingJobDir);
+
+      expect(existsSync(pendingSeedPath)).toBe(true);
+      expect(await readFile(join(stagingJobDir, "x.bin"), "utf-8")).toBe("data");
+      expect(existsSync(currentJobDir)).toBe(false);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+});
+
 describe("drainPendingQueue", () => {
   test("with limit 2 and three seeds, promotes exactly two and one remains in pending", async () => {
     const dir = await setupDir("drain-limit2-");
@@ -88,12 +169,13 @@ describe("drainPendingQueue", () => {
     }
   });
 
-  test("seed whose current/<jobId>/ already exists is not promoted, slot is released, seed remains in pending", async () => {
-    const dir = await setupDir("drain-existing-");
+  test("AC5: seed whose current/<jobId>/seed.json already exists is skipped, slot released, seed remains in pending", async () => {
+    const dir = await setupDir("drain-existing-seed-");
     try {
       await writeSeed(dir, "job-existing", { pipeline: "p" }, 1700000000);
       await writeSeed(dir, "job-fresh", { pipeline: "p" }, 1700000100);
       await mkdir(join(dir, "current", "job-existing"), { recursive: true });
+      await writeFile(join(dir, "current", "job-existing", "seed.json"), JSON.stringify({ pipeline: "p" }));
       const pids = new Map<string, number>();
       const result = await drainPendingQueue({
         dataDir: dir,
@@ -102,6 +184,7 @@ describe("drainPendingQueue", () => {
         spawnRunner: fakeSpawnRunner(pids),
       });
       expect(result.promoted).toEqual(["job-fresh"]);
+      expect(pids.has("job-existing")).toBe(false);
       const status = await getJobConcurrencyStatus(dir, 5, 1000);
       expect(status.activeJobs.map((l) => l.jobId).sort()).toEqual(["job-fresh"]);
       expect(existsSync(join(dir, "pending", "job-existing-seed.json"))).toBe(true);
@@ -112,6 +195,55 @@ describe("drainPendingQueue", () => {
     }
   });
 
+  test("AC3: seed with staged artifacts (current/ absent) promotes, moves artifacts, spawns once", async () => {
+    const dir = await setupDir("drain-staged-");
+    try {
+      await writeSeed(dir, "job-staged", { pipeline: "p" }, 1700000000);
+      await mkdir(join(dir, "staging", "job-staged"), { recursive: true });
+      await writeFile(join(dir, "staging", "job-staged", "notes.md"), "hello");
+      const pids = new Map<string, number>();
+      const result = await drainPendingQueue({
+        dataDir: dir,
+        maxConcurrentJobs: 5,
+        lockTimeoutMs: 1000,
+        spawnRunner: fakeSpawnRunner(pids),
+      });
+      expect(result.promoted).toEqual(["job-staged"]);
+      expect(pids.has("job-staged")).toBe(true);
+      expect(existsSync(join(dir, "pending", "job-staged-seed.json"))).toBe(false);
+      expect(existsSync(join(dir, "current", "job-staged", "seed.json"))).toBe(true);
+      expect(
+        await readFile(join(dir, "current", "job-staged", "files", "artifacts", "notes.md"), "utf-8"),
+      ).toBe("hello");
+      expect(existsSync(join(dir, "staging", "job-staged"))).toBe(false);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
+  test("AC6: crash-mid-promote self-heal — artifacts present, seed absent, no staging → promotes", async () => {
+    const dir = await setupDir("drain-selfheal-");
+    try {
+      await writeSeed(dir, "job-heal", { pipeline: "p" }, 1700000000);
+      await mkdir(join(dir, "current", "job-heal", "files", "artifacts"), { recursive: true });
+      await writeFile(join(dir, "current", "job-heal", "files", "artifacts", "x.bin"), "x");
+      const pids = new Map<string, number>();
+      const result = await drainPendingQueue({
+        dataDir: dir,
+        maxConcurrentJobs: 5,
+        lockTimeoutMs: 1000,
+        spawnRunner: fakeSpawnRunner(pids),
+      });
+      expect(result.promoted).toEqual(["job-heal"]);
+      expect(pids.has("job-heal")).toBe(true);
+      expect(existsSync(join(dir, "pending", "job-heal-seed.json"))).toBe(false);
+      expect(existsSync(join(dir, "current", "job-heal", "seed.json"))).toBe(true);
+      expect(existsSync(join(dir, "current", "job-heal", "files", "artifacts", "x.bin"))).toBe(true);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
   test("promotes seeds in deterministic order (mtime asc)", async () => {
     const dir = await setupDir("drain-order-");
     try {
@@ -181,6 +313,35 @@ describe("drainPendingQueue", () => {
     }
   });
 
+  test("AC8: rejection relocates staged artifacts under rejected/<jobId>/files/artifacts and removes staging", async () => {
+    const dir = await setupDir("drain-reject-staging-");
+    try {
+      await writeSeed(dir, "job-bad", { pipeline: "p" }, 1700000000);
+      await mkdir(join(dir, "staging", "job-bad"), { recursive: true });
+      await writeFile(join(dir, "staging", "job-bad", "notes.md"), "hello");
+      const failingSpawn = async (): Promise<{ pid: number }> => {
+        throw new Error("boom");
+      };
+
+      for (let i = 0; i < 3; i++) {
+        await drainPendingQueue({
+          dataDir: dir,
+          maxConcurrentJobs: 2,
+          lockTimeoutMs: 1000,
+          spawnRunner: failingSpawn,
+        });
+      }
+
+      expect(existsSync(join(dir, "rejected", "job-bad", "seed.json"))).toBe(true);
+      expect(
+        await readFile(join(dir, "rejected", "job-bad", "files", "artifacts", "notes.md"), "utf-8"),
+      ).toBe("hello");
+      expect(existsSync(join(dir, "staging", "job-bad"))).toBe(false);
+    } finally {
+      await rm(dir, { recursive: true });
+    }
+  });
+
   test("repeated invalid seed reads move the seed to rejected", async () => {
     const dir = await setupDir("drain-invalid-reject-");
     try {

package/src/core/orchestrator.ts

@@ -20,6 +20,7 @@ interface ResolvedDirs {
   current: string;
   complete: string;
   rejected: string;
+  staging: string;
 }
 
 /** Parsed seed file content (fields consumed by orchestrator). */
@@ -138,6 +139,7 @@ export function resolveDirs(dataDir: string): ResolvedDirs {
     current: join(root, "current"),
     complete: join(root, "complete"),
     rejected: join(root, "rejected"),
+    staging: join(root, "staging"),
   };
 }
 
@@ -438,18 +440,30 @@ async function dirExists(path: string): Promise<boolean> {
   }
 }
 
-async function promoteSeedFile(
+export async function promoteSeedFile(
   pendingSeedPath: string,
   currentJobDir: string,
+  stagingJobDir: string,
 ): Promise<void> {
   await mkdir(currentJobDir, { recursive: true });
+  if (await dirExists(stagingJobDir)) {
+    await mkdir(join(currentJobDir, "files"), { recursive: true });
+    await rename(stagingJobDir, join(currentJobDir, "files", "artifacts"));
+  }
   await rename(pendingSeedPath, join(currentJobDir, "seed.json"));
 }
 
-async function restoreSeedFile(currentJobDir: string, pendingSeedPath: string): Promise<void> {
-  const currentSeedPath = join(currentJobDir, "seed.json");
+export async function restoreSeedFile(
+  currentJobDir: string,
+  pendingSeedPath: string,
+  stagingJobDir: string,
+): Promise<void> {
+  const artifactsDir = join(currentJobDir, "files", "artifacts");
+  if (await dirExists(artifactsDir)) {
+    await rename(artifactsDir, stagingJobDir);
+  }
   try {
-    await rename(currentSeedPath, pendingSeedPath);
+    await rename(join(currentJobDir, "seed.json"), pendingSeedPath);
   } catch (err) {
     if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
   }
@@ -465,6 +479,11 @@ async function rejectSeedFile(
 ): Promise<void> {
   const rejectedJobDir = join(dataDir, "rejected", jobId);
   await mkdir(rejectedJobDir, { recursive: true });
+  const stagingJobDir = join(dataDir, "staging", jobId);
+  if (await dirExists(stagingJobDir)) {
+    await mkdir(join(rejectedJobDir, "files"), { recursive: true });
+    await rename(stagingJobDir, join(rejectedJobDir, "files", "artifacts"));
+  }
   try {
     await rename(seedPath, join(rejectedJobDir, "seed.json"));
   } catch (err) {
@@ -558,14 +577,15 @@ export async function drainPendingQueue(
     }
 
     const currentJobDir = join(dataDir, "current", jobId);
-    if (await dirExists(currentJobDir)) {
+    const stagingJobDir = join(dataDir, "staging", jobId);
+    if (await Bun.file(join(currentJobDir, "seed.json")).exists()) {
       await releaseJobSlot(dataDir, jobId, lockTimeoutMs);
-      logger.warn(`current job directory already exists for ${jobId}; skipping promotion`);
+      logger.warn(`seed already promoted for ${jobId}; skipping`);
       continue;
     }
 
     try {
-      await promoteSeedFile(seedPath, currentJobDir);
+      await promoteSeedFile(seedPath, currentJobDir, stagingJobDir);
     } catch (err) {
       await releaseJobSlot(dataDir, jobId, lockTimeoutMs);
       logger.error(`failed to promote seed file for job ${jobId}`, err);
@@ -578,7 +598,7 @@ export async function drainPendingQueue(
     } catch (err) {
       await releaseJobSlot(dataDir, jobId, lockTimeoutMs);
       try {
-        await restoreSeedFile(currentJobDir, seedPath);
+        await restoreSeedFile(currentJobDir, seedPath, stagingJobDir);
       } catch (restoreErr) {
         logger.error(`failed to restore queued seed after spawn failure for job ${jobId}`, restoreErr);
       }
@@ -607,7 +627,7 @@ export async function drainPendingQueue(
         logger.error(`failed to release slot after pid update failure for job ${jobId}`, releaseErr);
       }
       try {
-        await restoreSeedFile(currentJobDir, seedPath);
+        await restoreSeedFile(currentJobDir, seedPath, stagingJobDir);
       } catch (restoreErr) {
         logger.error(`failed to restore queued seed after pid update failure for job ${jobId}`, restoreErr);
       }
@@ -643,6 +663,7 @@ export function startOrchestrator(opts: OrchestratorOptions): Promise<Orchestrat
     .then(() => mkdir(dirs.current, { recursive: true }))
     .then(() => mkdir(dirs.complete, { recursive: true }))
     .then(() => mkdir(dirs.rejected, { recursive: true }))
+    .then(() => mkdir(dirs.staging, { recursive: true }))
     .then(() => new Promise<OrchestratorHandle>((resolve, reject) => {
       const running = new Map<string, ChildHandle>();
       const spawnFn = opts.spawn ?? createDefaultSpawn();

package/src/ui/server/endpoints/__tests__/upload-endpoints.test.ts

@@ -1,7 +1,20 @@
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
 import { zipSync } from "fflate";
 import { describe, expect, it } from "vitest";
 
-import { normalizeSeedUpload } from "../upload-endpoints";
+import { handleSeedUploadDirect, normalizeSeedUpload } from "../upload-endpoints";
+
+async function pathExists(target: string): Promise<boolean> {
+  try {
+    await stat(target);
+    return true;
+  } catch {
+    return false;
+  }
+}
 
 function concatBytes(parts: Uint8Array[]): Uint8Array<ArrayBuffer> {
   const total = parts.reduce((sum, part) => sum + part.length, 0);
@@ -55,3 +68,35 @@ describe("normalizeSeedUpload", () => {
     expect(Array.from(blob.content)).toEqual(Array.from(artifact));
   });
 });
+
+describe("handleSeedUploadDirect", () => {
+  it("stages artifacts under staging/{jobId}/ without creating current/{jobId}/", async () => {
+    const dataDir = await mkdtemp(path.join(tmpdir(), "upload-direct-"));
+    try {
+      const content = new TextEncoder().encode("# notes\nhello\n");
+      const seedObject = { name: "demo", pipeline: "x" };
+
+      const response = await handleSeedUploadDirect(seedObject, dataDir, [
+        { filename: "notes.md", content },
+      ]);
+      expect(response.status).toBe(201);
+      const body = (await response.json()) as { data: { jobId: string } };
+      const jobId = body.data.jobId;
+
+      const pipelineData = path.join(dataDir, "pipeline-data");
+
+      // Artifact staged byte-identical under staging/{jobId}/.
+      const staged = path.join(pipelineData, "staging", jobId, "notes.md");
+      const stagedBytes = await readFile(staged);
+      expect(Array.from(stagedBytes)).toEqual(Array.from(content));
+
+      // Pending seed trigger written.
+      expect(await pathExists(path.join(pipelineData, "pending", `${jobId}-seed.json`))).toBe(true);
+
+      // The endpoint must not create current/{jobId}/.
+      expect(await pathExists(path.join(pipelineData, "current", jobId))).toBe(false);
+    } finally {
+      await rm(dataDir, { recursive: true, force: true });
+    }
+  });
+});

package/src/ui/server/endpoints/upload-endpoints.ts

@@ -35,12 +35,13 @@ export async function handleSeedUploadDirect(
   const pipelineData = path.join(dataDir, "pipeline-data");
   const pendingDir = path.join(pipelineData, "pending");
 
-  // Write artifacts to current/{jobId}/ first (before the trigger file)
-  // so they are in place when the orchestrator processes the seed.
+  // Stage artifacts under staging/{jobId}/ (before the trigger file).
+  // The orchestrator owns current/{jobId}/ and moves staged artifacts into
+  // current/{jobId}/files/artifacts/ during promotion.
   if (artifacts.length > 0) {
-    const jobDir = path.join(pipelineData, "current", jobId);
+    const stagingJobDir = path.join(pipelineData, "staging", jobId);
     for (const artifact of artifacts) {
-      const artifactPath = path.join(jobDir, "files", "artifacts", artifact.filename);
+      const artifactPath = path.join(stagingJobDir, artifact.filename);
       await mkdir(path.dirname(artifactPath), { recursive: true });
       await Bun.write(artifactPath, artifact.content);
     }