@ryanfw/prompt-orchestration-pipeline 0.5.0 → 0.7.0

package/src/core/module-loader.js

@@ -2,8 +2,6 @@ import fsp from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 
-const WORKSPACE_CACHE_DIR = path.join(process.cwd(), ".tmp-task-modules");
-
 /**
  * Convert supported modulePath formats into a file:// URL.
  * @param {string | URL} modulePath
@@ -88,70 +86,86 @@ function createMissingModuleError(modulePath, originalError) {
 }
 
 /**
- * Copy a module file into a workspace-local cache directory so Vite/Vitest can load it.
+ * Copy a module file adjacent to its original location with a unique name.
  * @param {string} sourcePath
  * @returns {Promise<string>}
  */
-async function copyModuleToWorkspaceCache(sourcePath) {
-  await fsp.mkdir(WORKSPACE_CACHE_DIR, { recursive: true });
+async function copyModuleAdjacent(sourcePath) {
+  const dir = path.dirname(sourcePath);
   const ext = path.extname(sourcePath) || ".js";
   const base = path.basename(sourcePath, ext);
   const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
-  const destFile = path.join(
-    WORKSPACE_CACHE_DIR,
-    `${base}.${uniqueSuffix}${ext}`
-  );
+  const destFile = path.join(dir, `.cache.${base}.${uniqueSuffix}${ext}`);
   await fsp.copyFile(sourcePath, destFile);
   return destFile;
 }
 
 /**
  * Dynamically import a module with cache busting while remaining compatible with Node's file:/// resolution.
- * Falls back to copying the module into a workspace-local cache when query parameters break filesystem resolution.
+ * Falls back to copying the module adjacent to its original location when query parameters break filesystem resolution.
  * @param {string | URL} modulePath
  * @returns {Promise<any>} Module namespace object
  */
 export async function loadFreshModule(modulePath) {
   const fileUrl = resolveToFileURL(modulePath);
-  const cacheBustedUrl = `${fileUrl.href}?t=${Date.now()}`;
 
+  // First attempt direct import without cache busting
   try {
-    return await import(cacheBustedUrl);
+    return await import(fileUrl.href);
   } catch (error) {
-    if (!isModuleNotFoundError(error) || fileUrl.protocol !== "file:") {
+    if (!isModuleNotFoundError(error)) {
       throw error;
     }
 
-    const absolutePath = fileURLToPath(fileUrl);
-
+    // Second attempt: try cache-busted import
+    const cacheBustedUrl = `${fileUrl.href}?t=${Date.now()}`;
     try {
-      await fsp.access(absolutePath);
-    } catch {
-      throw createMissingModuleError(
-        absolutePath,
-        /** @type {Error} */ (error)
-      );
-    }
+      return await import(cacheBustedUrl);
+    } catch (cacheBustedError) {
+      if (
+        !isModuleNotFoundError(cacheBustedError) ||
+        fileUrl.protocol !== "file:"
+      ) {
+        throw cacheBustedError;
+      }
 
-    try {
-      const cacheCopy = await copyModuleToWorkspaceCache(absolutePath);
-      const cacheUrl = `${pathToFileURL(cacheCopy).href}?t=${Date.now()}`;
-      return await import(cacheUrl);
-    } catch (fallbackError) {
-      const messageLines = [
-        `Failed to load module "${absolutePath}" after attempting cache-busting import.`,
-        `Cache-busted URL: ${cacheBustedUrl}`,
-        `Original error: ${/** @type {Error} */ (error).message}`,
-        `Fallback error: ${/** @type {Error} */ (fallbackError).message}`,
-      ];
-      const combined = new Error(messageLines.join("\n"));
-      if ("cause" in Error.prototype) {
-        combined.cause = fallbackError;
-      } else {
-        combined.fallbackError = fallbackError;
+      const absolutePath = fileURLToPath(fileUrl);
+
+      try {
+        await fsp.access(absolutePath);
+      } catch {
+        throw createMissingModuleError(
+          absolutePath,
+          /** @type {Error} */ (cacheBustedError)
+        );
+      }
+
+      // Third attempt: copy adjacent and import
+      let adjacentCopy;
+      try {
+        adjacentCopy = await copyModuleAdjacent(absolutePath);
+        const adjacentUrl = `${pathToFileURL(adjacentCopy).href}?t=${Date.now()}`;
+        return await import(adjacentUrl);
+      } catch (fallbackError) {
+        const messageLines = [
+          `Failed to load module "${absolutePath}" after attempting direct import, cache-busting import, and adjacent copy fallback.`,
+          `Direct import URL: ${fileUrl.href}`,
+          `Cache-busted URL: ${cacheBustedUrl}`,
+          `Adjacent fallback path attempted: ${adjacentCopy || "[adjacent copy creation failed]"}`,
+          `Original error: ${/** @type {Error} */ (error).message}`,
+          `Cache-bust error: ${/** @type {Error} */ (cacheBustedError).message}`,
+          `Fallback error: ${/** @type {Error} */ (fallbackError).message}`,
+        ];
+        const combined = new Error(messageLines.join("\n"));
+        if ("cause" in Error.prototype) {
+          combined.cause = fallbackError;
+        } else {
+          combined.fallbackError = fallbackError;
+        }
+        combined.initialError = error;
+        combined.cacheBustedError = cacheBustedError;
+        throw combined;
       }
-      combined.initialError = error;
-      throw combined;
     }
   }
 }

package/src/core/pipeline-runner.js

@@ -4,6 +4,11 @@ import { runPipeline } from "./task-runner.js";
 import { loadFreshModule } from "./module-loader.js";
 import { validatePipelineOrThrow } from "./validation.js";
 import { getPipelineConfig } from "./config.js";
+import { writeJobStatus } from "./status-writer.js";
+import { TaskState } from "../config/statuses.js";
+import { ensureTaskSymlinkBridge } from "./symlink-bridge.js";
+import { cleanupTaskSymlinks } from "./symlink-utils.js";
+import { createTaskFileIO } from "./file-io.js";
 
 const ROOT = process.env.PO_ROOT || process.cwd();
 const DATA_DIR = path.join(ROOT, process.env.PO_DATA_DIR || "pipeline-data");
@@ -17,6 +22,8 @@ if (!jobId) throw new Error("runner requires jobId as argument");
 
 const workDir = path.join(CURRENT_DIR, jobId);
 
+const startFromTask = process.env.PO_START_FROM_TASK;
+
 // Get pipeline slug from environment or fallback to seed.json
 let pipelineSlug = process.env.PO_PIPELINE_SLUG;
 if (!pipelineSlug) {
@@ -60,7 +67,15 @@ const seed = JSON.parse(
 let pipelineArtifacts = {};
 
 for (const taskName of pipeline.tasks) {
-  if (status.tasks[taskName]?.state === "done") {
+  // Skip tasks before startFromTask when targeting a specific restart point
+  if (
+    startFromTask &&
+    pipeline.tasks.indexOf(taskName) < pipeline.tasks.indexOf(startFromTask)
+  ) {
+    continue;
+  }
+
+  if (status.tasks[taskName]?.state === TaskState.DONE) {
     try {
       const outputPath = path.join(workDir, "tasks", taskName, "output.json");
       const output = JSON.parse(await fs.readFile(outputPath, "utf8"));
@@ -70,17 +85,13 @@ for (const taskName of pipeline.tasks) {
   }
 
   await updateStatus(taskName, {
-    state: "running",
+    state: TaskState.RUNNING,
     startedAt: now(),
     attempts: (status.tasks[taskName]?.attempts || 0) + 1,
   });
 
   const taskDir = path.join(workDir, "tasks", taskName);
   await fs.mkdir(taskDir, { recursive: true });
-  await atomicWrite(
-    path.join(taskDir, "letter.json"),
-    JSON.stringify({ task: taskName, at: now() }, null, 2)
-  );
 
   try {
     const ctx = {
@@ -103,14 +114,31 @@ for (const taskName of pipeline.tasks) {
       ? modulePath
       : path.resolve(path.dirname(TASK_REGISTRY), modulePath);
 
-    const result = await runPipeline(absoluteModulePath, ctx);
+    // Create symlink bridge for deterministic module resolution
+    const poRoot = process.env.PO_ROOT || process.cwd();
+    const relocatedEntry = await ensureTaskSymlinkBridge({
+      taskDir,
+      poRoot,
+      taskModulePath: absoluteModulePath,
+    });
+
+    // Create fileIO for this task
+    const fileIO = createTaskFileIO({
+      workDir,
+      taskName,
+      getStage: () => null, // pipeline-runner doesn't have stages
+      statusPath: tasksStatusPath,
+    });
+
+    const result = await runPipeline(relocatedEntry, ctx);
 
     if (!result.ok) {
-      // Persist execution-logs.json and failure-details.json on task failure
+      // Persist execution-logs.json and failure-details.json on task failure via IO
       if (result.logs) {
-        await atomicWrite(
-          path.join(taskDir, "execution-logs.json"),
-          JSON.stringify(result.logs, null, 2)
+        await fileIO.writeLog(
+          "execution-logs.json",
+          JSON.stringify(result.logs, null, 2),
+          { mode: "replace" }
         );
       }
       const failureDetails = {
@@ -120,14 +148,15 @@ for (const taskName of pipeline.tasks) {
         context: result.context,
         refinementAttempts: result.refinementAttempts || 0,
       };
-      await atomicWrite(
-        path.join(taskDir, "failure-details.json"),
-        JSON.stringify(failureDetails, null, 2)
+      await fileIO.writeLog(
+        "failure-details.json",
+        JSON.stringify(failureDetails, null, 2),
+        { mode: "replace" }
       );
 
       // Update tasks-status.json with enriched failure context
       await updateStatus(taskName, {
-        state: "failed",
+        state: TaskState.FAILED,
         endedAt: now(),
         error: result.error, // Don't double-normalize - use result.error as-is
         failedStage: result.failedStage,
@@ -155,14 +184,15 @@ for (const taskName of pipeline.tasks) {
     // No need to manually write output.json or enumerate artifacts
 
     if (result.logs) {
-      await atomicWrite(
-        path.join(taskDir, "execution-logs.json"),
-        JSON.stringify(result.logs, null, 2)
+      await fileIO.writeLog(
+        "execution-logs.json",
+        JSON.stringify(result.logs, null, 2),
+        { mode: "replace" }
       );
     }
 
     await updateStatus(taskName, {
-      state: "done",
+      state: TaskState.DONE,
       endedAt: now(),
       executionTimeMs:
         result.logs?.reduce((total, log) => total + (log.ms || 0), 0) || 0,
@@ -170,7 +200,7 @@ for (const taskName of pipeline.tasks) {
     });
   } catch (err) {
     await updateStatus(taskName, {
-      state: "failed",
+      state: TaskState.FAILED,
       endedAt: now(),
       error: normalizeError(err),
     });
@@ -200,17 +230,26 @@ await appendLine(
   }) + "\n"
 );
 
+// Clean up task symlinks to avoid dangling links in archives
+await cleanupTaskSymlinks(dest);
+
 function now() {
   return new Date().toISOString();
 }
 
 async function updateStatus(taskName, patch) {
-  const current = JSON.parse(await fs.readFile(tasksStatusPath, "utf8"));
-  current.current = taskName;
-  current.tasks = current.tasks || {};
-  current.tasks[taskName] = { ...(current.tasks[taskName] || {}), ...patch };
-  await atomicWrite(tasksStatusPath, JSON.stringify(current, null, 2));
-  Object.assign(status, current);
+  return await writeJobStatus(workDir, (snapshot) => {
+    snapshot.current = taskName;
+    snapshot.tasks = snapshot.tasks || {};
+    snapshot.tasks[taskName] = {
+      ...(snapshot.tasks[taskName] || {}),
+      ...patch,
+    };
+    return snapshot;
+  }).then((snap) => {
+    Object.assign(status, snap);
+    return snap;
+  });
 }
 
 async function appendLine(file, line) {

package/src/core/status-writer.js

@@ -1,5 +1,6 @@
 import fs from "node:fs/promises";
 import path from "node:path";
+import { TaskState } from "../config/statuses.js";
 
 // Lazy import SSE registry to avoid circular dependencies
 let sseRegistry = null;
@@ -16,6 +17,9 @@ async function getSSERegistry() {
   return sseRegistry;
 }
 
+// Per-job write queues to serialize writes to tasks-status.json
+const writeQueues = new Map(); // Map<string jobDir, Promise<any>>
+
 // Instrumentation helper for status writer
 const createStatusWriterLogger = (jobId) => {
   const prefix = `[StatusWriter:${jobId || "unknown"}]`;
@@ -54,7 +58,7 @@ const createStatusWriterLogger = (jobId) => {
 function createDefaultStatus(jobId) {
   return {
     id: jobId,
-    state: "pending",
+    state: TaskState.PENDING,
     current: null,
     currentStage: null,
     lastUpdated: new Date().toISOString(),
@@ -128,7 +132,7 @@ function validateStatusSnapshot(snapshot) {
 
   // Ensure required root fields exist
   if (typeof snapshot.state !== "string") {
-    snapshot.state = "pending";
+    snapshot.state = TaskState.PENDING;
   }
   if (snapshot.current !== null && typeof snapshot.current !== "string") {
     snapshot.current = null;
@@ -193,66 +197,80 @@ export async function writeJobStatus(jobDir, updateFn) {
   const jobId = path.basename(jobDir);
   const logger = createStatusWriterLogger(jobId);
 
-  logger.group("Status Write Operation");
-  logger.log(`Updating status for job: ${jobId}`);
-  logger.log(`Status file path: ${statusPath}`);
-
-  // Read existing status or create default
-  let snapshot = await readStatusFile(statusPath, jobId);
-  logger.log("Current status snapshot:", snapshot);
-
-  // Validate basic structure
-  snapshot = validateStatusSnapshot(snapshot);
-
-  // Apply user updates
-  try {
-    const result = updateFn(snapshot);
-    // If updateFn returns a value, use it as new snapshot
-    if (result !== undefined) {
-      snapshot = result;
-    }
-    logger.log("Status after update function:", snapshot);
-  } catch (error) {
-    logger.error("Update function failed:", error);
-    throw new Error(`Update function failed: ${error.message}`);
-  }
-
-  // Validate final structure
-  snapshot = validateStatusSnapshot(snapshot);
+  // Get or create the write queue for this job directory
+  const prev = writeQueues.get(jobDir) || Promise.resolve();
+  let resultSnapshot;
+
+  const next = prev
+    .then(async () => {
+      logger.group("Status Write Operation");
+      logger.log(`Updating status for job: ${jobId}`);
+      logger.log(`Status file path: ${statusPath}`);
+
+      // Read existing status or create default
+      const current = await readStatusFile(statusPath, jobId);
+      logger.log("Current status snapshot:", current);
+
+      // Validate basic structure
+      const validated = validateStatusSnapshot(current);
+
+      // Apply user updates
+      let maybeUpdated;
+      try {
+        maybeUpdated = updateFn(validated);
+      } catch (error) {
+        console.error(`[${jobId}] Error executing update function:`, error);
+        throw new Error(`Update function failed: ${error.message}`);
+      }
+      const snapshot = validateStatusSnapshot(
+        maybeUpdated === undefined ? validated : maybeUpdated
+      );
 
-  // Update timestamp
-  snapshot.lastUpdated = new Date().toISOString();
+      snapshot.lastUpdated = new Date().toISOString();
+      logger.log("Status after update function:", snapshot);
+
+      // Atomic write
+      await atomicWrite(statusPath, snapshot);
+      logger.log("Status file written successfully");
+
+      // Emit SSE event for tasks-status.json change
+      const registry = (await getSSERegistry().catch(() => null)) || null;
+      if (registry) {
+        try {
+          const eventData = {
+            type: "state:change",
+            data: {
+              path: path.join(jobDir, "tasks-status.json"),
+              id: jobId,
+              jobId,
+            },
+          };
+          registry.broadcast(eventData);
+          logger.sse("state:change", eventData.data);
+          logger.log("SSE event broadcasted successfully");
+        } catch (error) {
+          // Don't fail the write if SSE emission fails
+          logger.error("Failed to emit SSE event:", error);
+          console.warn(`Failed to emit SSE event: ${error.message}`);
+        }
+      } else {
+        logger.warn("SSE registry not available - no event broadcasted");
+      }
 
-  // Atomic write
-  await atomicWrite(statusPath, snapshot);
-  logger.log("Status file written successfully");
+      logger.groupEnd();
+      resultSnapshot = snapshot;
+    })
+    .catch((e) => {
+      throw e;
+    });
 
-  // Emit SSE event for tasks-status.json change
-  const registry = await getSSERegistry();
-  if (registry) {
-    try {
-      const eventData = {
-        type: "state:change",
-        data: {
-          path: path.join(jobDir, "tasks-status.json"),
-          id: jobId,
-          jobId,
-        },
-      };
-      registry.broadcast(eventData);
-      logger.sse("state:change", eventData.data);
-      logger.log("SSE event broadcasted successfully");
-    } catch (error) {
-      // Don't fail the write if SSE emission fails
-      logger.error("Failed to emit SSE event:", error);
-      console.warn(`Failed to emit SSE event: ${error.message}`);
-    }
-  } else {
-    logger.warn("SSE registry not available - no event broadcasted");
-  }
+  // Store the promise chain and set up cleanup
+  writeQueues.set(
+    jobDir,
+    next.finally(() => {})
+  );
 
-  logger.groupEnd();
-  return snapshot;
+  return next.then(() => resultSnapshot);
 }
 
 /**
@@ -329,3 +347,140 @@ export async function updateTaskStatus(jobDir, taskId, taskUpdateFn) {
     return snapshot;
   });
 }
+
+/**
+ * Reset a job from a specific task onward, preserving prior completed tasks
+ *
+ * @param {string} jobDir - Job directory path containing tasks-status.json
+ * @param {string} fromTask - Task identifier to restart from (inclusive)
+ * @param {Object} options - Reset options
+ * @param {boolean} [options.clearTokenUsage=true] - Whether to clear token usage arrays
+ * @returns {Promise<Object>} The updated status snapshot
+ */
+export async function resetJobFromTask(
+  jobDir,
+  fromTask,
+  { clearTokenUsage = true } = {}
+) {
+  if (!jobDir || typeof jobDir !== "string") {
+    throw new Error("jobDir must be a non-empty string");
+  }
+
+  if (!fromTask || typeof fromTask !== "string") {
+    throw new Error("fromTask must be a non-empty string");
+  }
+
+  return writeJobStatus(jobDir, (snapshot) => {
+    // Reset root-level status
+    snapshot.state = TaskState.PENDING;
+    snapshot.current = null;
+    snapshot.currentStage = null;
+    snapshot.progress = 0;
+    snapshot.lastUpdated = new Date().toISOString();
+
+    // Ensure tasks object exists
+    if (!snapshot.tasks || typeof snapshot.tasks !== "object") {
+      snapshot.tasks = {};
+    }
+
+    // Compute progress based on preserved (done) tasks before fromTask
+    let doneCount = 0;
+    const taskKeys = Object.keys(snapshot.tasks);
+    for (const taskId of taskKeys) {
+      if (snapshot.tasks[taskId]?.state === TaskState.DONE) {
+        doneCount++;
+      }
+    }
+    snapshot.progress =
+      taskKeys.length > 0 ? (doneCount / taskKeys.length) * 100 : 0;
+
+    // Reset tasks from fromTask onward to pending; keep earlier tasks as-is
+    for (const taskId of taskKeys) {
+      const task = snapshot.tasks[taskId];
+      if (!task) continue; // ensure task object exists
+
+      const shouldReset =
+        taskKeys.indexOf(taskId) >= taskKeys.indexOf(fromTask);
+      if (shouldReset) {
+        // Reset task state and metadata
+        task.state = TaskState.PENDING;
+        task.currentStage = null;
+
+        // Remove error-related fields
+        delete task.failedStage;
+        delete task.error;
+
+        // Reset counters
+        task.attempts = 0;
+        task.refinementAttempts = 0;
+
+        // Clear token usage if requested
+        if (clearTokenUsage) {
+          task.tokenUsage = [];
+        }
+      }
+      // If task appears before fromTask and is not done, keep its state untouched
+      // This preserves upstream work if user restarts from a mid-pipeline task
+    }
+
+    // Preserve files.* arrays - do not modify them
+    // This ensures generated files are preserved during restart
+
+    return snapshot;
+  });
+}
+
+/**
+ * Reset a job and all its tasks to clean-slate state atomically
+ *
+ * @param {string} jobDir - Job directory path containing tasks-status.json
+ * @param {Object} options - Reset options
+ * @param {boolean} [options.clearTokenUsage=true] - Whether to clear token usage arrays
+ * @returns {Promise<Object>} The updated status snapshot
+ */
+export async function resetJobToCleanSlate(
+  jobDir,
+  { clearTokenUsage = true } = {}
+) {
+  if (!jobDir || typeof jobDir !== "string") {
+    throw new Error("jobDir must be a non-empty string");
+  }
+
+  return writeJobStatus(jobDir, (snapshot) => {
+    // Reset root-level status
+    snapshot.state = TaskState.PENDING;
+    snapshot.current = null;
+    snapshot.currentStage = null;
+    snapshot.progress = 0;
+    snapshot.lastUpdated = new Date().toISOString();
+
+    // Reset all tasks
+    if (snapshot.tasks && typeof snapshot.tasks === "object") {
+      for (const taskId of Object.keys(snapshot.tasks)) {
+        const task = snapshot.tasks[taskId];
+
+        // Reset task state
+        task.state = TaskState.PENDING;
+        task.currentStage = null;
+
+        // Remove error-related fields
+        delete task.failedStage;
+        delete task.error;
+
+        // Reset counters
+        task.attempts = 0;
+        task.refinementAttempts = 0;
+
+        // Clear token usage if requested
+        if (clearTokenUsage) {
+          task.tokenUsage = [];
+        }
+      }
+    }
+
+    // Preserve files.* arrays - do not modify them
+    // This ensures generated files are preserved during restart
+
+    return snapshot;
+  });
+}

package/src/core/symlink-bridge.js

@@ -0,0 +1,57 @@
+import path from "node:path";
+import { ensureSymlink } from "./symlink-utils.js";
+
+/**
+ * Creates a taskDir symlink bridge to ensure deterministic module resolution.
+ *
+ * This function creates three symlinks in the task directory:
+ * - taskDir/node_modules -> {poRoot}/node_modules (for bare package specifiers)
+ * - taskDir/project -> {poRoot} (optional convenience for absolute project paths)
+ * - taskDir/_task_root -> dirname(taskModulePath) (for relative imports)
+ *
+ * @param {Object} options - Configuration options
+ * @param {string} options.taskDir - The task directory where symlinks should be created
+ * @param {string} options.poRoot - The repository root directory
+ * @param {string} options.taskModulePath - Absolute path to the original task module
+ * @returns {string} The relocated entry path for the task module
+ * @throws {Error} If symlink creation fails
+ */
+export async function ensureTaskSymlinkBridge({
+  taskDir,
+  poRoot,
+  taskModulePath,
+}) {
+  // Normalize all paths to absolute paths
+  const normalizedTaskDir = path.resolve(taskDir);
+  const normalizedPoRoot = path.resolve(poRoot);
+  const normalizedTaskModulePath = path.resolve(taskModulePath);
+
+  // Ensure the task directory exists
+  await import("node:fs/promises").then((fs) =>
+    fs.mkdir(normalizedTaskDir, { recursive: true })
+  );
+
+  // Create symlink for node_modules -> {poRoot}/node_modules
+  const nodeModulesLink = path.join(normalizedTaskDir, "node_modules");
+  const nodeModulesTarget = path.join(normalizedPoRoot, "node_modules");
+  await ensureSymlink(nodeModulesLink, nodeModulesTarget, "dir");
+
+  // Create symlink for project -> {poRoot}
+  const projectLink = path.join(normalizedTaskDir, "project");
+  await ensureSymlink(projectLink, normalizedPoRoot, "dir");
+
+  // Create symlink for _task_root -> dirname(taskModulePath)
+  const taskRootLink = path.join(normalizedTaskDir, "_task_root");
+  const taskRootTarget = path.dirname(normalizedTaskModulePath);
+  await ensureSymlink(taskRootLink, taskRootTarget, "dir");
+
+  // Return the relocated entry path
+  const taskModuleBasename = path.basename(normalizedTaskModulePath);
+  const relocatedEntry = path.join(
+    normalizedTaskDir,
+    "_task_root",
+    taskModuleBasename
+  );
+
+  return relocatedEntry;
+}