@ryanfw/prompt-orchestration-pipeline 0.9.1 → 0.11.0

package/src/core/orchestrator.js

@@ -5,6 +5,8 @@ import chokidar from "chokidar";
 import { spawn as defaultSpawn } from "node:child_process";
 import { getConfig, getPipelineConfig } from "./config.js";
 import { createLogger } from "./logger.js";
+import { createTaskFileIO, generateLogName } from "./file-io.js";
+import { LogEvent } from "../config/log-events.js";
 
 /**
  * Resolve canonical pipeline directories for the given data root.
@@ -152,7 +154,68 @@ export async function startOrchestrator(opts) {
         tasks: {}, // Initialize empty tasks object for pipeline runner
       };
       await fs.writeFile(statusPath, JSON.stringify(status, null, 2));
+
+      // Initialize status from artifacts if any exist
+      try {
+        const { initializeStatusFromArtifacts } = await import(
+          "./status-initializer.js"
+        );
+        const pipelineConfig = getPipelineConfig(seed?.pipeline || "default");
+        const pipelineSnapshot = JSON.parse(
+          await fs.readFile(pipelineConfig.pipelineJsonPath, "utf8")
+        );
+
+        const applyArtifacts = await initializeStatusFromArtifacts({
+          jobDir: workDir,
+          pipeline: pipelineSnapshot,
+        });
+
+        // Apply artifact initialization to the status
+        const updatedStatus = applyArtifacts(status);
+        await fs.writeFile(statusPath, JSON.stringify(updatedStatus, null, 2));
+
+        logger.log("Initialized status from upload artifacts", {
+          jobId,
+          pipeline: seed?.pipeline,
+          artifactsCount: updatedStatus.files?.artifacts?.length || 0,
+        });
+      } catch (artifactError) {
+        // Don't fail job startup if artifact initialization fails, just log
+        logger.warn("Failed to initialize status from artifacts", {
+          jobId,
+          error: artifactError.message,
+        });
+      }
     }
+    // Create fileIO for orchestrator-level logging
+    const fileIO = createTaskFileIO({
+      workDir,
+      taskName: jobId,
+      getStage: () => "orchestrator",
+      statusPath,
+      trackTaskFiles: false,
+    });
+
+    // Write job start log
+    await fileIO.writeLog(
+      generateLogName(jobId, "orchestrator", LogEvent.START),
+      JSON.stringify(
+        {
+          jobId,
+          pipeline: seed?.pipeline,
+          timestamp: new Date().toISOString(),
+          seedSummary: {
+            name: seed?.name,
+            pipeline: seed?.pipeline,
+            keys: Object.keys(seed || {}),
+          },
+        },
+        null,
+        2
+      ),
+      { mode: "replace" }
+    );
+
     // Spawn runner for this job
     const child = spawnRunner(
       logger,
@@ -161,7 +224,8 @@ export async function startOrchestrator(opts) {
       running,
       spawn,
       testMode,
-      seed
+      seed,
+      fileIO
     );
     // child registered inside spawnRunner
     return child;
@@ -223,6 +287,12 @@ export async function startOrchestrator(opts) {
   return { stop };
 }
 
+/**
+ * @typedef {Object} TaskFileIO
+ * @property {(name: string, content: string, options?: { mode?: 'append'|'replace' }) => Promise<string>} writeLog
+ * @property {(name: string, content: string, options?: { mode?: 'append'|'replace' }) => string} writeLogSync
+ */
+
 /**
  * Spawn a pipeline runner. In testMode we still call spawn() so tests can assert,
  * but we resolve immediately and let tests drive the lifecycle (emit 'exit', etc.).
@@ -234,8 +304,18 @@ export async function startOrchestrator(opts) {
  * @param {typeof defaultSpawn} spawn
  * @param {boolean} testMode
  * @param {Object} seed - Seed data containing pipeline information
+ * @param {TaskFileIO} fileIO - Task-scoped file I/O interface for writing logs
  */
-function spawnRunner(logger, jobId, dirs, running, spawn, testMode, seed) {
+function spawnRunner(
+  logger,
+  jobId,
+  dirs,
+  running,
+  spawn,
+  testMode,
+  seed,
+  fileIO
+) {
   // Use path relative to this file to avoid process.cwd() issues
   const orchestratorDir = path.dirname(new URL(import.meta.url).pathname);
   const runnerPath = path.join(orchestratorDir, "pipeline-runner.js");
@@ -316,11 +396,67 @@ function spawnRunner(logger, jobId, dirs, running, spawn, testMode, seed) {
 
     running.set(jobId, child);
 
-    child.on("exit", () => {
+    child.on("exit", (code, signal) => {
       running.delete(jobId);
+
+      // Write job completion log synchronously
+      if (fileIO) {
+        try {
+          fileIO.writeLogSync(
+            generateLogName(jobId, "orchestrator", LogEvent.COMPLETE),
+            JSON.stringify(
+              {
+                jobId,
+                exitCode: code,
+                signal: signal,
+                timestamp: new Date().toISOString(),
+                completionType: code === 0 ? "success" : "failure",
+              },
+              null,
+              2
+            ),
+            { mode: "replace" }
+          );
+        } catch (error) {
+          logger.error("Failed to write job completion log", {
+            jobId,
+            error: error.message,
+          });
+        }
+      }
     });
-    child.on("error", () => {
+
+    child.on("error", (error) => {
       running.delete(jobId);
+
+      // Write job error log synchronously
+      if (fileIO) {
+        try {
+          fileIO.writeLogSync(
+            generateLogName(jobId, "orchestrator", LogEvent.ERROR),
+            JSON.stringify(
+              {
+                jobId,
+                error: {
+                  message: error.message,
+                  name: error.name,
+                  code: error.code,
+                },
+                timestamp: new Date().toISOString(),
+                completionType: "error",
+              },
+              null,
+              2
+            ),
+            { mode: "replace" }
+          );
+        } catch (logError) {
+          logger.error("Failed to write job error log", {
+            jobId,
+            error: logError.message,
+          });
+        }
+      }
     });
 
     // In test mode: return immediately; in real mode you might await readiness

package/src/core/pipeline-runner.js

@@ -7,9 +7,14 @@ import { getPipelineConfig } from "./config.js";
 import { writeJobStatus } from "./status-writer.js";
 import { TaskState } from "../config/statuses.js";
 import { ensureTaskSymlinkBridge } from "./symlink-bridge.js";
-import { cleanupTaskSymlinks } from "./symlink-utils.js";
-import { createTaskFileIO } from "./file-io.js";
+import {
+  cleanupTaskSymlinks,
+  validateTaskSymlinks,
+  repairTaskSymlinks,
+} from "./symlink-utils.js";
+import { createTaskFileIO, generateLogName } from "./file-io.js";
 import { createJobLogger } from "./logger.js";
+import { LogEvent, LogFileExtension } from "../config/log-events.js";
 
 const ROOT = process.env.PO_ROOT || process.cwd();
 const DATA_DIR = path.join(ROOT, process.env.PO_DATA_DIR || "pipeline-data");
@@ -132,8 +137,66 @@ for (const taskName of pipeline.tasks) {
       ? modulePath
       : path.resolve(path.dirname(TASK_REGISTRY), modulePath);
 
-    // Create symlink bridge for deterministic module resolution
+    // Validate symlinks before task execution to ensure restart reliability
     const poRoot = process.env.PO_ROOT || process.cwd();
+    const expectedTargets = {
+      nodeModules: path.join(path.resolve(poRoot, ".."), "node_modules"),
+      taskRoot: path.dirname(absoluteModulePath),
+    };
+
+    const validationResult = await validateTaskSymlinks(
+      taskDir,
+      expectedTargets
+    );
+
+    if (!validationResult.isValid) {
+      logger.warn("Task symlinks validation failed, attempting repair", {
+        taskName,
+        taskDir,
+        errors: validationResult.errors,
+        validationDuration: validationResult.duration,
+      });
+
+      const repairResult = await repairTaskSymlinks(
+        taskDir,
+        poRoot,
+        absoluteModulePath
+      );
+
+      if (!repairResult.success) {
+        const errorMessage = `Failed to repair task symlinks for ${taskName}: ${repairResult.errors.join(", ")}`;
+        logger.error("Task symlink repair failed, aborting execution", {
+          taskName,
+          taskDir,
+          errors: repairResult.errors,
+          repairDuration: repairResult.duration,
+        });
+
+        await updateStatus(taskName, {
+          state: TaskState.FAILED,
+          endedAt: now(),
+          error: { message: errorMessage, type: "SymlinkRepairFailed" },
+        });
+
+        process.exitCode = 1;
+        process.exit(1);
+      }
+
+      logger.log("Task symlinks repaired successfully", {
+        taskName,
+        taskDir,
+        repairDuration: repairResult.duration,
+        relocatedEntry: repairResult.relocatedEntry,
+      });
+    } else {
+      logger.debug("Task symlinks validation passed", {
+        taskName,
+        taskDir,
+        validationDuration: validationResult.duration,
+      });
+    }
+
+    // Create symlink bridge for deterministic module resolution
     const relocatedEntry = await ensureTaskSymlinkBridge({
       taskDir,
       poRoot,
@@ -162,7 +225,12 @@ for (const taskName of pipeline.tasks) {
       // Persist execution-logs.json and failure-details.json on task failure via IO
       if (result.logs) {
         await fileIO.writeLog(
-          "execution-logs.json",
+          generateLogName(
+            taskName,
+            "pipeline",
+            LogEvent.EXECUTION_LOGS,
+            LogFileExtension.JSON
+          ),
           JSON.stringify(result.logs, null, 2),
           { mode: "replace" }
         );
@@ -175,7 +243,12 @@ for (const taskName of pipeline.tasks) {
         refinementAttempts: result.refinementAttempts || 0,
       };
       await fileIO.writeLog(
-        "failure-details.json",
+        generateLogName(
+          taskName,
+          "pipeline",
+          LogEvent.FAILURE_DETAILS,
+          LogFileExtension.JSON
+        ),
         JSON.stringify(failureDetails, null, 2),
         { mode: "replace" }
       );
@@ -218,7 +291,12 @@ for (const taskName of pipeline.tasks) {
 
     if (result.logs) {
       await fileIO.writeLog(
-        "execution-logs.json",
+        generateLogName(
+          taskName,
+          "pipeline",
+          LogEvent.EXECUTION_LOGS,
+          LogFileExtension.JSON
+        ),
         JSON.stringify(result.logs, null, 2),
         { mode: "replace" }
       );

package/src/core/status-initializer.js

@@ -0,0 +1,155 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Initialize status snapshot from artifacts in the filesystem
+ * @param {Object} options - Options object
+ * @param {string} options.jobDir - Job directory path
+ * @param {Object} options.pipeline - Pipeline configuration object
+ * @returns {Promise<Function>} Function that applies artifact initialization to a snapshot
+ */
+export async function initializeStatusFromArtifacts({ jobDir, pipeline }) {
+  if (!jobDir || typeof jobDir !== "string") {
+    throw new Error("jobDir must be a non-empty string");
+  }
+
+  if (!pipeline || typeof pipeline !== "object") {
+    throw new Error("pipeline must be an object");
+  }
+
+  const artifactsDir = path.join(jobDir, "files", "artifacts");
+  let artifactFilenames = [];
+
+  try {
+    // Read artifacts directory
+    const entries = await fs.readdir(artifactsDir, { withFileTypes: true });
+
+    // Collect filenames for regular files only
+    artifactFilenames = entries
+      .filter((entry) => entry.isFile())
+      .map((entry) => entry.name);
+
+    console.log("[STATUS_INIT] Found artifacts in directory", {
+      artifactsDir,
+      artifactCount: artifactFilenames.length,
+      artifactNames: artifactFilenames,
+    });
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      // Directory doesn't exist, no artifacts to initialize
+      console.log(
+        "[STATUS_INIT] Artifacts directory does not exist, skipping initialization",
+        {
+          artifactsDir,
+        }
+      );
+    } else {
+      console.error("[STATUS_INIT] Failed to read artifacts directory", {
+        artifactsDir,
+        error: error.message,
+      });
+    }
+    // Return a no-op function for non-existent or unreadable directory
+    return (snapshot) => snapshot;
+  }
+
+  // Determine first task ID from pipeline
+  const firstTaskId = Array.isArray(pipeline.tasks) ? pipeline.tasks[0] : null;
+  console.log("[STATUS_INIT] Determined first task", {
+    firstTaskId,
+    hasTasks: Array.isArray(pipeline.tasks),
+    taskCount: pipeline.tasks?.length || 0,
+  });
+
+  // Return function that applies the artifact initialization to a snapshot
+  return function apply(snapshot) {
+    console.log("[STATUS_INIT] Applying artifact initialization to snapshot", {
+      existingArtifacts: snapshot.files?.artifacts?.length || 0,
+      newArtifacts: artifactFilenames.length,
+      firstTaskId,
+    });
+
+    // Ensure files object exists with proper structure
+    if (!snapshot.files || typeof snapshot.files !== "object") {
+      snapshot.files = { artifacts: [], logs: [], tmp: [] };
+    } else {
+      // Ensure each files array exists
+      for (const type of ["artifacts", "logs", "tmp"]) {
+        if (!Array.isArray(snapshot.files[type])) {
+          snapshot.files[type] = [];
+        }
+      }
+    }
+
+    // Add artifact filenames to root level (deduplicate)
+    const existingArtifacts = new Set(snapshot.files.artifacts || []);
+    for (const filename of artifactFilenames) {
+      if (!existingArtifacts.has(filename)) {
+        snapshot.files.artifacts.push(filename);
+        existingArtifacts.add(filename);
+      }
+    }
+
+    // Add artifact filenames to first task (if it exists)
+    if (firstTaskId) {
+      // Ensure tasks object exists
+      if (!snapshot.tasks || typeof snapshot.tasks !== "object") {
+        snapshot.tasks = {};
+      }
+
+      // Ensure first task exists
+      if (!snapshot.tasks[firstTaskId]) {
+        snapshot.tasks[firstTaskId] = {};
+      }
+
+      // Ensure task files object exists with proper structure
+      if (
+        !snapshot.tasks[firstTaskId].files ||
+        typeof snapshot.tasks[firstTaskId].files !== "object"
+      ) {
+        snapshot.tasks[firstTaskId].files = {
+          artifacts: [],
+          logs: [],
+          tmp: [],
+        };
+      } else {
+        // Ensure each task files array exists
+        for (const type of ["artifacts", "logs", "tmp"]) {
+          if (!Array.isArray(snapshot.tasks[firstTaskId].files[type])) {
+            snapshot.tasks[firstTaskId].files[type] = [];
+          }
+        }
+      }
+
+      // Add artifact filenames to first task (deduplicate)
+      const existingTaskArtifacts = new Set(
+        snapshot.tasks[firstTaskId].files.artifacts || []
+      );
+      for (const filename of artifactFilenames) {
+        if (!existingTaskArtifacts.has(filename)) {
+          snapshot.tasks[firstTaskId].files.artifacts.push(filename);
+          existingTaskArtifacts.add(filename);
+        }
+      }
+
+      console.log("[STATUS_INIT] Added artifacts to first task", {
+        firstTaskId,
+        taskArtifactCount: snapshot.tasks[firstTaskId].files.artifacts.length,
+        artifactNames: artifactFilenames,
+      });
+    }
+
+    console.log("[STATUS_INIT] Final snapshot state", {
+      rootArtifacts: snapshot.files.artifacts.length,
+      rootArtifactNames: snapshot.files.artifacts,
+      firstTaskArtifacts: firstTaskId
+        ? snapshot.tasks[firstTaskId].files.artifacts.length
+        : 0,
+      firstTaskArtifactNames: firstTaskId
+        ? snapshot.tasks[firstTaskId].files.artifacts
+        : [],
+    });
+
+    return snapshot;
+  };
+}

package/src/core/status-writer.js

@@ -281,22 +281,71 @@ export async function updateTaskStatus(jobDir, taskId, taskUpdateFn) {
     throw new Error("taskUpdateFn must be a function");
   }
 
-  return writeJobStatus(jobDir, (snapshot) => {
-    // Ensure task exists
-    if (!snapshot.tasks[taskId]) {
-      snapshot.tasks[taskId] = {};
-    }
+  const jobId = path.basename(jobDir);
+  const logger = createJobLogger("StatusWriter", jobId);
 
-    const task = snapshot.tasks[taskId];
+  // Get or create the write queue for this job directory
+  const prev = writeQueues.get(jobDir) || Promise.resolve();
+  let resultSnapshot;
 
-    // Apply task updates
-    const result = taskUpdateFn(task);
-    if (result !== undefined) {
-      snapshot.tasks[taskId] = result;
-    }
+  const next = prev
+    .then(async () => {
+      logger.group("Task Status Update Operation");
+      logger.log(`Updating task ${taskId} for job: ${jobId}`);
 
-    return snapshot;
-  });
+      const statusPath = path.join(jobDir, "tasks-status.json");
+
+      // Read existing status or create default
+      const current = await readStatusFile(statusPath, jobId);
+      const validated = validateStatusSnapshot(current);
+
+      // Ensure task exists
+      if (!validated.tasks[taskId]) {
+        validated.tasks[taskId] = {};
+      }
+
+      const task = validated.tasks[taskId];
+
+      // Apply task updates
+      const result = taskUpdateFn(task);
+      if (result !== undefined) {
+        validated.tasks[taskId] = result;
+      }
+
+      validated.lastUpdated = new Date().toISOString();
+
+      // Atomic write
+      await atomicWrite(statusPath, validated);
+      logger.log("Task status file written successfully");
+
+      // Emit task:updated SSE event after successful write
+      try {
+        const eventData = {
+          jobId,
+          taskId,
+          task: validated.tasks[taskId],
+        };
+        await logger.sse("task:updated", eventData);
+        logger.log("task:updated SSE event broadcasted successfully");
+      } catch (error) {
+        // Don't fail the write if SSE emission fails
+        logger.error("Failed to emit task:updated SSE event:", error);
+      }
+
+      logger.groupEnd();
+      resultSnapshot = validated;
+    })
+    .catch((e) => {
+      throw e;
+    });
+
+  // Store the promise chain and set up cleanup
+  writeQueues.set(
+    jobDir,
+    next.finally(() => {})
+  );
+
+  return next.then(() => resultSnapshot);
 }
 
 /**
@@ -435,3 +484,92 @@ export async function resetJobToCleanSlate(
     return snapshot;
   });
 }
+
+/**
+ * Consolidated path jail security validation with generic error messages
+ * @param {string} filename - Filename to validate
+ * @returns {Object|null} Validation result or null if valid
+ */
+function validateFilePath(filename) {
+  // Check for path traversal patterns
+  if (filename.includes("..")) {
+    console.error("Path security: path traversal detected", { filename });
+    return {
+      allowed: false,
+      message: "Path validation failed",
+    };
+  }
+
+  // Check for absolute paths (POSIX, Windows, backslashes, ~)
+  if (
+    path.isAbsolute(filename) ||
+    /^[a-zA-Z]:/.test(filename) ||
+    filename.includes("\\") ||
+    filename.startsWith("~")
+  ) {
+    console.error("Path security: absolute path detected", { filename });
+    return {
+      allowed: false,
+      message: "Path validation failed",
+    };
+  }
+
+  // Check for empty filename
+  if (!filename || filename.trim() === "") {
+    console.error("Path security: empty filename detected");
+    return {
+      allowed: false,
+      message: "Path validation failed",
+    };
+  }
+
+  // Path is valid
+  return null;
+}
+
+/**
+ * Initialize job-level artifact index and copy artifacts to job directory
+ * @param {string} jobDir - Job directory path
+ * @param {Array} uploadArtifacts - Array of {filename, content} objects
+ * @returns {Promise<void>}
+ */
+export async function initializeJobArtifacts(jobDir, uploadArtifacts = []) {
+  if (!jobDir || typeof jobDir !== "string") {
+    throw new Error("jobDir must be a non-empty string");
+  }
+
+  if (!Array.isArray(uploadArtifacts)) {
+    throw new Error("uploadArtifacts must be an array");
+  }
+
+  if (uploadArtifacts.length === 0) {
+    return;
+  }
+
+  const jobFilesDir = path.join(jobDir, "files");
+  const jobArtifactsDir = path.join(jobFilesDir, "artifacts");
+
+  await fs.mkdir(jobFilesDir, { recursive: true });
+  await fs.mkdir(jobArtifactsDir, { recursive: true });
+
+  for (const artifact of uploadArtifacts) {
+    const { filename, content } = artifact || {};
+
+    if (!filename || typeof filename !== "string") {
+      continue; // Skip invalid entries rather than throwing
+    }
+
+    // Validate filename using the consolidated function
+    const validation = validateFilePath(filename);
+    if (validation) {
+      console.error("Path security: skipping invalid artifact", {
+        filename,
+        reason: validation.message,
+      });
+      continue; // Skip invalid filenames rather than throwing
+    }
+
+    const artifactPath = path.join(jobArtifactsDir, filename);
+    await fs.writeFile(artifactPath, content);
+  }
+}