@ryanfw/prompt-orchestration-pipeline 0.0.1 → 0.4.0

package/src/core/pipeline-runner.js

@@ -1,7 +1,9 @@
 import fs from "node:fs/promises";
 import path from "node:path";
-import { pathToFileURL } from "node:url";
 import { runPipeline } from "./task-runner.js";
+import { loadFreshModule } from "./module-loader.js";
+import { validatePipelineOrThrow } from "./validation.js";
+import { getPipelineConfig } from "./config.js";
 
 const ROOT = process.env.PO_ROOT || process.cwd();
 const DATA_DIR = path.join(ROOT, process.env.PO_DATA_DIR || "pipeline-data");
@@ -10,23 +12,45 @@ const CURRENT_DIR =
 const COMPLETE_DIR =
   process.env.PO_COMPLETE_DIR || path.join(DATA_DIR, "complete");
 
-const CONFIG_DIR =
-  process.env.PO_CONFIG_DIR || path.join(ROOT, "pipeline-config");
+const jobId = process.argv[2];
+if (!jobId) throw new Error("runner requires jobId as argument");
+
+const workDir = path.join(CURRENT_DIR, jobId);
+
+// Get pipeline slug from environment or fallback to seed.json
+let pipelineSlug = process.env.PO_PIPELINE_SLUG;
+if (!pipelineSlug) {
+  try {
+    const seedPath = path.join(workDir, "seed.json");
+    const seedData = JSON.parse(await fs.readFile(seedPath, "utf8"));
+    pipelineSlug = seedData?.pipeline;
+    if (!pipelineSlug) {
+      throw new Error("No pipeline slug found in seed.json");
+    }
+  } catch (error) {
+    throw new Error(
+      `Pipeline slug is required. Set PO_PIPELINE_SLUG environment variable or ensure seed.json contains a 'pipeline' field. Error: ${error.message}`
+    );
+  }
+}
+
+// Use explicit pipeline configuration
+const pipelineConfig = getPipelineConfig(pipelineSlug);
+
 const TASK_REGISTRY =
-  process.env.PO_TASK_REGISTRY || path.join(CONFIG_DIR, "tasks/index.js");
+  process.env.PO_TASK_REGISTRY ||
+  path.join(pipelineConfig.tasksDir, "index.js");
 const PIPELINE_DEF_PATH =
-  process.env.PO_PIPELINE_PATH || path.join(CONFIG_DIR, "pipeline.json");
-
-const name = process.argv[2];
-if (!name) throw new Error("runner requires pipeline name");
+  process.env.PO_PIPELINE_PATH || pipelineConfig.pipelineJsonPath;
 
-const workDir = path.join(CURRENT_DIR, name);
 const tasksStatusPath = path.join(workDir, "tasks-status.json");
 
 const pipeline = JSON.parse(await fs.readFile(PIPELINE_DEF_PATH, "utf8"));
-// Add cache busting to force task registry reload
-const taskRegistryUrl = `${pathToFileURL(TASK_REGISTRY).href}?t=${Date.now()}`;
-const tasks = (await import(taskRegistryUrl)).default;
+
+// Validate pipeline format early with a friendly error message
+validatePipelineOrThrow(pipeline, PIPELINE_DEF_PATH);
+
+const tasks = (await loadFreshModule(TASK_REGISTRY)).default;
 
 const status = JSON.parse(await fs.readFile(tasksStatusPath, "utf8"));
 const seed = JSON.parse(
@@ -63,9 +87,13 @@ for (const taskName of pipeline.tasks) {
       workDir,
       taskDir,
       seed,
-      artifacts: pipelineArtifacts,
       taskName,
       taskConfig: pipeline.taskConfig?.[taskName] || {},
+      statusPath: tasksStatusPath,
+      jobId,
+      meta: {
+        pipelineTasks: [...pipeline.tasks],
+      },
     };
     const modulePath = tasks[taskName];
     if (!modulePath) throw new Error(`Task not registered: ${taskName}`);
@@ -78,19 +106,54 @@ for (const taskName of pipeline.tasks) {
     const result = await runPipeline(absoluteModulePath, ctx);
 
     if (!result.ok) {
-      throw new Error(
-        `${taskName} failed after ${result.refinementAttempts || 0} attempts: ${result.error?.message || "unknown"}`
-      );
-    }
-
-    if (result.context?.output) {
+      // Persist execution-logs.json and failure-details.json on task failure
+      if (result.logs) {
+        await atomicWrite(
+          path.join(taskDir, "execution-logs.json"),
+          JSON.stringify(result.logs, null, 2)
+        );
+      }
+      const failureDetails = {
+        failedStage: result.failedStage,
+        error: result.error,
+        logs: result.logs,
+        context: result.context,
+        refinementAttempts: result.refinementAttempts || 0,
+      };
       await atomicWrite(
-        path.join(taskDir, "output.json"),
-        JSON.stringify(result.context.output, null, 2)
+        path.join(taskDir, "failure-details.json"),
+        JSON.stringify(failureDetails, null, 2)
       );
-      pipelineArtifacts[taskName] = result.context.output;
+
+      // Update tasks-status.json with enriched failure context
+      await updateStatus(taskName, {
+        state: "failed",
+        endedAt: now(),
+        error: result.error, // Don't double-normalize - use result.error as-is
+        failedStage: result.failedStage,
+        refinementAttempts: result.refinementAttempts || 0,
+        stageLogPath: path.join(
+          workDir,
+          "files",
+          "logs",
+          `stage-${result.failedStage}.log`
+        ),
+        errorContext: {
+          previousStage: result.context?.previousStage || "seed",
+          dataHasSeed: !!result.context?.data?.seed,
+          seedHasData: result.context?.data?.seed?.data !== undefined,
+          flagsKeys: Object.keys(result.context?.flags || {}),
+        },
+      });
+
+      // Exit with non-zero status but do not throw to keep consistent flow
+      process.exitCode = 1;
+      process.exit(1);
     }
 
+    // The file I/O system automatically handles writing outputs and updating tasks-status.json
+    // No need to manually write output.json or enumerate artifacts
+
     if (result.logs) {
       await atomicWrite(
         path.join(taskDir, "execution-logs.json"),
@@ -98,12 +161,10 @@ for (const taskName of pipeline.tasks) {
       );
     }
 
-    const artifacts = await getArtifacts(taskDir);
     await updateStatus(taskName, {
       state: "done",
       endedAt: now(),
-      artifacts,
-      executionTime:
+      executionTimeMs:
         result.logs?.reduce((total, log) => total + (log.ms || 0), 0) || 0,
       refinementAttempts: result.refinementAttempts || 0,
     });
@@ -119,17 +180,16 @@ for (const taskName of pipeline.tasks) {
 }
 
 await fs.mkdir(COMPLETE_DIR, { recursive: true });
-const dest = path.join(COMPLETE_DIR, name);
+const dest = path.join(COMPLETE_DIR, jobId);
 await fs.rename(workDir, dest);
 await appendLine(
   path.join(COMPLETE_DIR, "runs.jsonl"),
   JSON.stringify({
-    name,
-    pipelineId: status.pipelineId,
+    id: status.id,
     finishedAt: now(),
     tasks: Object.keys(status.tasks),
     totalExecutionTime: Object.values(status.tasks).reduce(
-      (total, t) => total + (t.executionTime || 0),
+      (total, t) => total + (t.executionTimeMs || 0),
       0
     ),
     totalRefinementAttempts: Object.values(status.tasks).reduce(
@@ -147,6 +207,7 @@ function now() {
 async function updateStatus(taskName, patch) {
   const current = JSON.parse(await fs.readFile(tasksStatusPath, "utf8"));
   current.current = taskName;
+  current.tasks = current.tasks || {};
   current.tasks[taskName] = { ...(current.tasks[taskName] || {}), ...patch };
   await atomicWrite(tasksStatusPath, JSON.stringify(current, null, 2));
   Object.assign(status, current);
@@ -164,19 +225,12 @@ async function atomicWrite(file, data) {
 }
 
 function normalizeError(e) {
+  // If it's already a structured error object with a message string, pass it through
+  if (e && typeof e === "object" && typeof e.message === "string") {
+    return e;
+  }
+
   if (e instanceof Error)
     return { name: e.name, message: e.message, stack: e.stack };
   return { message: String(e) };
 }
-
-async function getArtifacts(dir) {
-  const potentialFiles = ["output.json", "letter.json", "execution-logs.json"];
-  const artifacts = [];
-  for (const file of potentialFiles) {
-    try {
-      await fs.stat(path.join(dir, file));
-      artifacts.push(file);
-    } catch {}
-  }
-  return artifacts;
-}

package/src/core/progress.js

@@ -0,0 +1,66 @@
+/**
+ * Deterministic progress computation for pipeline tasks.
+ * Provides a single authoritative mapping from (pipelineTaskIds, currentTaskId, currentStageName) → progress percentage.
+ */
+
+/**
+ * Fixed ordered list of all possible stages in a pipeline.
+ * The order is canonical and used for deterministic progress calculation.
+ */
+export const KNOWN_STAGES = [
+  "ingestion",
+  "preProcessing",
+  "promptTemplating",
+  "inference",
+  "parsing",
+  "validateStructure",
+  "validateQuality",
+  "critique",
+  "refine",
+  "finalValidation",
+  "integration",
+];
+
+/**
+ * Computes deterministic progress percentage for a pipeline execution.
+ *
+ * Progress is calculated based on the position of the current task in the ordered pipeline
+ * and the position of the current stage in the fixed stage list. This ensures that
+ * identical inputs always produce the same progress value.
+ *
+ * @param {string[]} pipelineTaskIds - Ordered list of task IDs in the pipeline
+ * @param {string} currentTaskId - ID of the currently executing task
+ * @param {string} currentStageName - Name of the current stage being executed
+ * @param {string[]} [stages=KNOWN_STAGES] - Stage list to use for calculation (defaults to KNOWN_STAGES)
+ * @returns {number} Progress percentage as integer in [0, 100]
+ *
+ * @example
+ * computeDeterministicProgress(
+ *   ["task-1", "task-2"],
+ *   "task-1",
+ *   "ingestion"
+ * ); // → 5
+ */
+export function computeDeterministicProgress(
+  pipelineTaskIds,
+  currentTaskId,
+  currentStageName,
+  stages = KNOWN_STAGES
+) {
+  // Guard against empty pipeline to avoid division by zero
+  const totalSteps = Math.max(1, pipelineTaskIds.length * stages.length);
+
+  // Find task position, fallback to 0 if not found
+  const taskIdx = Math.max(0, pipelineTaskIds.indexOf(currentTaskId));
+
+  // Find stage position, fallback to 0 if not found
+  const stageIdx = Math.max(0, stages.indexOf(currentStageName));
+
+  // Completed steps = (completed tasks * stages per task) + (completed stages in current task)
+  // We count the current stage as completed since this is called after stage completion
+  const completed = taskIdx * stages.length + (stageIdx + 1);
+
+  // Calculate percentage and clamp to [0, 100]
+  const percent = Math.round((100 * completed) / totalSteps);
+  return Math.max(0, Math.min(100, percent));
+}

package/src/core/status-writer.js

@@ -0,0 +1,331 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+// Lazy import SSE registry to avoid circular dependencies
+let sseRegistry = null;
+async function getSSERegistry() {
+  if (!sseRegistry) {
+    try {
+      const module = await import("../ui/sse.js");
+      sseRegistry = module.sseRegistry;
+    } catch (error) {
+      // SSE not available in all environments
+      return null;
+    }
+  }
+  return sseRegistry;
+}
+
+// Instrumentation helper for status writer
+const createStatusWriterLogger = (jobId) => {
+  const prefix = `[StatusWriter:${jobId || "unknown"}]`;
+  return {
+    log: (message, data = null) => {
+      console.log(`${prefix} ${message}`, data ? data : "");
+    },
+    warn: (message, data = null) => {
+      console.warn(`${prefix} ${message}`, data ? data : "");
+    },
+    error: (message, data = null) => {
+      console.error(`${prefix} ${message}`, data ? data : "");
+    },
+    group: (label) => console.group(`${prefix} ${label}`),
+    groupEnd: () => console.groupEnd(),
+    sse: (eventType, eventData) => {
+      console.log(
+        `%c${prefix} SSE Broadcast: ${eventType}`,
+        "color: #cc6600; font-weight: bold;",
+        eventData
+      );
+    },
+  };
+};
+
+/**
+ * Atomic status writer utility for tasks-status.json
+ *
+ * Provides atomic updates to job status files with proper error handling
+ * and shape validation for the new status schema.
+ */
+
+/**
+ * Default status shape for new files
+ */
+function createDefaultStatus(jobId) {
+  return {
+    id: jobId,
+    state: "pending",
+    current: null,
+    currentStage: null,
+    lastUpdated: new Date().toISOString(),
+    tasks: {},
+    files: {
+      artifacts: [],
+      logs: [],
+      tmp: [],
+    },
+  };
+}
+
+/**
+ * Reads and parses tasks-status.json, creates default if missing
+ */
+async function readStatusFile(statusPath, jobId) {
+  try {
+    const content = await fs.readFile(statusPath, "utf8");
+    const parsed = JSON.parse(content);
+    return parsed;
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      // File doesn't exist, return default structure
+      return createDefaultStatus(jobId);
+    }
+    if (error instanceof SyntaxError) {
+      // Invalid JSON, log warning and return default
+      console.warn(
+        `Invalid JSON in ${statusPath}, creating new status:`,
+        error.message
+      );
+      return createDefaultStatus(jobId);
+    }
+    throw error;
+  }
+}
+
+/**
+ * Atomic write using temp file + rename pattern
+ */
+async function atomicWrite(filePath, data) {
+  const tmpPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).slice(2)}`;
+  try {
+    await fs.writeFile(tmpPath, JSON.stringify(data, null, 2), "utf8");
+    await fs.rename(tmpPath, filePath);
+  } catch (error) {
+    // Clean up temp file if write failed
+    try {
+      await fs.unlink(tmpPath);
+    } catch {
+      // Ignore cleanup errors
+    }
+    throw error;
+  }
+}
+
+/**
+ * Validates that the status snapshot has required structure.
+ *
+ * This function preserves all unknown fields, including optional numeric fields
+ * like `snapshot.progress`. Only the required root fields are validated and
+ * fixed if missing or malformed. Extra fields are passed through unchanged.
+ *
+ * @param {Object} snapshot - The status snapshot to validate
+ * @returns {Object} The validated and normalized snapshot
+ */
+function validateStatusSnapshot(snapshot) {
+  if (!snapshot || typeof snapshot !== "object") {
+    throw new Error("Status snapshot must be an object");
+  }
+
+  // Ensure required root fields exist
+  if (typeof snapshot.state !== "string") {
+    snapshot.state = "pending";
+  }
+  if (snapshot.current !== null && typeof snapshot.current !== "string") {
+    snapshot.current = null;
+  }
+  if (
+    snapshot.currentStage !== null &&
+    typeof snapshot.currentStage !== "string"
+  ) {
+    snapshot.currentStage = null;
+  }
+
+  // Ensure timestamp exists
+  if (!snapshot.lastUpdated || typeof snapshot.lastUpdated !== "string") {
+    snapshot.lastUpdated = new Date().toISOString();
+  }
+
+  // Ensure tasks object exists
+  if (!snapshot.tasks || typeof snapshot.tasks !== "object") {
+    snapshot.tasks = {};
+  }
+
+  // Ensure files object exists with proper structure
+  if (!snapshot.files || typeof snapshot.files !== "object") {
+    snapshot.files = { artifacts: [], logs: [], tmp: [] };
+  } else {
+    // Ensure each files array exists
+    for (const type of ["artifacts", "logs", "tmp"]) {
+      if (!Array.isArray(snapshot.files[type])) {
+        snapshot.files[type] = [];
+      }
+    }
+  }
+
+  return snapshot;
+}
+
+/**
+ * Atomically updates tasks-status.json with the provided update function
+ *
+ * @param {string} jobDir - Job directory path containing tasks-status.json
+ * @param {Function} updateFn - Function that receives and mutates the status snapshot
+ * @returns {Promise<Object>} The updated status snapshot
+ *
+ * Example:
+ * await writeJobStatus(jobDir, (snapshot) => {
+ *   snapshot.current = "task-1";
+ *   snapshot.currentStage = "processing";
+ *   snapshot.tasks["task-1"].currentStage = "processing";
+ *   snapshot.tasks["task-1"].state = "running";
+ * });
+ */
+export async function writeJobStatus(jobDir, updateFn) {
+  if (!jobDir || typeof jobDir !== "string") {
+    throw new Error("jobDir must be a non-empty string");
+  }
+
+  if (typeof updateFn !== "function") {
+    throw new Error("updateFn must be a function");
+  }
+
+  const statusPath = path.join(jobDir, "tasks-status.json");
+  const jobId = path.basename(jobDir);
+  const logger = createStatusWriterLogger(jobId);
+
+  logger.group("Status Write Operation");
+  logger.log(`Updating status for job: ${jobId}`);
+  logger.log(`Status file path: ${statusPath}`);
+
+  // Read existing status or create default
+  let snapshot = await readStatusFile(statusPath, jobId);
+  logger.log("Current status snapshot:", snapshot);
+
+  // Validate basic structure
+  snapshot = validateStatusSnapshot(snapshot);
+
+  // Apply user updates
+  try {
+    const result = updateFn(snapshot);
+    // If updateFn returns a value, use it as new snapshot
+    if (result !== undefined) {
+      snapshot = result;
+    }
+    logger.log("Status after update function:", snapshot);
+  } catch (error) {
+    logger.error("Update function failed:", error);
+    throw new Error(`Update function failed: ${error.message}`);
+  }
+
+  // Validate final structure
+  snapshot = validateStatusSnapshot(snapshot);
+
+  // Update timestamp
+  snapshot.lastUpdated = new Date().toISOString();
+
+  // Atomic write
+  await atomicWrite(statusPath, snapshot);
+  logger.log("Status file written successfully");
+
+  // Emit SSE event for tasks-status.json change
+  const registry = await getSSERegistry();
+  if (registry) {
+    try {
+      const eventData = {
+        type: "state:change",
+        data: {
+          path: path.join(jobDir, "tasks-status.json"),
+          id: jobId,
+          jobId,
+        },
+      };
+      registry.broadcast(eventData);
+      logger.sse("state:change", eventData.data);
+      logger.log("SSE event broadcasted successfully");
+    } catch (error) {
+      // Don't fail the write if SSE emission fails
+      logger.error("Failed to emit SSE event:", error);
+      console.warn(`Failed to emit SSE event: ${error.message}`);
+    }
+  } else {
+    logger.warn("SSE registry not available - no event broadcasted");
+  }
+
+  logger.groupEnd();
+  return snapshot;
+}
+
+/**
+ * Reads tasks-status.json with proper error handling
+ *
+ * @param {string} jobDir - Job directory path
+ * @returns {Promise<Object|null>} Status snapshot or null if file cannot be read
+ */
+export async function readJobStatus(jobDir) {
+  if (!jobDir || typeof jobDir !== "string") {
+    throw new Error("jobDir must be a non-empty string");
+  }
+
+  const statusPath = path.join(jobDir, "tasks-status.json");
+
+  try {
+    // Check if file exists first
+    await fs.access(statusPath);
+
+    const content = await fs.readFile(statusPath, "utf8");
+    const parsed = JSON.parse(content);
+    return validateStatusSnapshot(parsed);
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      return null;
+    }
+    if (error instanceof SyntaxError) {
+      console.warn(
+        `Invalid JSON in ${statusPath}, cannot read status:`,
+        error.message
+      );
+      return null;
+    }
+    console.warn(`Failed to read status from ${jobDir}:`, error.message);
+    return null;
+  }
+}
+
+/**
+ * Utility to update task-specific fields atomically
+ *
+ * @param {string} jobDir - Job directory path
+ * @param {string} taskId - Task identifier
+ * @param {Function} taskUpdateFn - Function that receives and mutates the task object
+ * @returns {Promise<Object>} The updated status snapshot
+ */
+export async function updateTaskStatus(jobDir, taskId, taskUpdateFn) {
+  if (!jobDir || typeof jobDir !== "string") {
+    throw new Error("jobDir must be a non-empty string");
+  }
+
+  if (!taskId || typeof taskId !== "string") {
+    throw new Error("taskId must be a non-empty string");
+  }
+
+  if (typeof taskUpdateFn !== "function") {
+    throw new Error("taskUpdateFn must be a function");
+  }
+
+  return writeJobStatus(jobDir, (snapshot) => {
+    // Ensure task exists
+    if (!snapshot.tasks[taskId]) {
+      snapshot.tasks[taskId] = {};
+    }
+
+    const task = snapshot.tasks[taskId];
+
+    // Apply task updates
+    const result = taskUpdateFn(task);
+    if (result !== undefined) {
+      snapshot.tasks[taskId] = result;
+    }
+
+    return snapshot;
+  });
+}