@ryanfw/prompt-orchestration-pipeline 0.6.0 → 0.8.0

package/src/config/statuses.js

@@ -0,0 +1,119 @@
+/**
+ * Canonical status constants and utilities for the prompt orchestration pipeline.
+ * This module serves as the single source of truth for all status-related values.
+ */
+
+// Task states (per-task execution status)
+export const TaskState = Object.freeze({
+  PENDING: "pending",
+  RUNNING: "running",
+  DONE: "done",
+  FAILED: "failed",
+});
+
+// Job statuses (computed aggregate from task states)
+export const JobStatus = Object.freeze({
+  PENDING: "pending",
+  RUNNING: "running",
+  FAILED: "failed",
+  COMPLETE: "complete",
+});
+
+// Job locations (filesystem lifecycle buckets)
+export const JobLocation = Object.freeze({
+  PENDING: "pending",
+  CURRENT: "current",
+  COMPLETE: "complete",
+  REJECTED: "rejected",
+});
+
+// Validation sets
+export const VALID_TASK_STATES = new Set(Object.values(TaskState));
+export const VALID_JOB_STATUSES = new Set(Object.values(JobStatus));
+export const VALID_JOB_LOCATIONS = new Set(Object.values(JobLocation));
+
+/**
+ * Normalizes a task state string to canonical form.
+ * @param {string} state - Raw task state
+ * @returns {string} Canonical task state
+ */
+export function normalizeTaskState(state) {
+  if (typeof state !== "string") {
+    return TaskState.PENDING;
+  }
+
+  const normalized = state.toLowerCase().trim();
+
+  // Handle common synonyms
+  switch (normalized) {
+    case "error":
+      return TaskState.FAILED;
+    case "succeeded":
+      return TaskState.DONE;
+    case TaskState.PENDING:
+    case TaskState.RUNNING:
+    case TaskState.DONE:
+    case TaskState.FAILED:
+      return normalized;
+    default:
+      return TaskState.PENDING;
+  }
+}
+
+/**
+ * Normalizes a job status string to canonical form.
+ * @param {string} status - Raw job status
+ * @returns {string} Canonical job status
+ */
+export function normalizeJobStatus(status) {
+  if (typeof status !== "string") {
+    return JobStatus.PENDING;
+  }
+
+  const normalized = status.toLowerCase().trim();
+
+  // Handle common synonyms
+  switch (normalized) {
+    case "completed":
+      return JobStatus.COMPLETE;
+    case "error":
+      return JobStatus.FAILED;
+    case JobStatus.PENDING:
+    case JobStatus.RUNNING:
+    case JobStatus.FAILED:
+    case JobStatus.COMPLETE:
+      return normalized;
+    default:
+      return JobStatus.PENDING;
+  }
+}
+
+/**
+ * Derives job status from an array of task states.
+ * Priority: failed > running > complete > pending
+ * @param {Array<Object>} tasks - Array of task objects with state property
+ * @returns {string} Canonical job status
+ */
+export function deriveJobStatusFromTasks(tasks) {
+  if (!Array.isArray(tasks) || tasks.length === 0) {
+    return JobStatus.PENDING;
+  }
+
+  // Normalize all task states first
+  const normalizedStates = tasks.map((task) => normalizeTaskState(task.state));
+
+  // Apply priority rules
+  if (normalizedStates.some((state) => state === TaskState.FAILED)) {
+    return JobStatus.FAILED;
+  }
+
+  if (normalizedStates.some((state) => state === TaskState.RUNNING)) {
+    return JobStatus.RUNNING;
+  }
+
+  if (normalizedStates.every((state) => state === TaskState.DONE)) {
+    return JobStatus.COMPLETE;
+  }
+
+  return JobStatus.PENDING;
+}

package/src/core/config.js

@@ -187,168 +187,6 @@ export const defaultConfig = {
     maxConcurrency: 5,
     retryMaxAttempts: 3,
     retryBackoffMs: 1000,
-    models: {
-      // DeepSeek (2025)
-      "deepseek:chat": {
-        provider: "deepseek",
-        model: "deepseek-chat", // V3.2 Exp (non-thinking) under the hood
-        tokenCostInPerMillion: 0.27,
-        tokenCostOutPerMillion: 1.1,
-      },
-      "deepseek:reasoner": {
-        provider: "deepseek",
-        model: "deepseek-reasoner", // R1 family
-        tokenCostInPerMillion: 0.55,
-        tokenCostOutPerMillion: 2.19,
-      },
-
-      // — OpenAI (2024 legacy still callable) —
-      "openai:gpt-4": {
-        provider: "openai",
-        model: "gpt-4",
-        tokenCostInPerMillion: 30.0,
-        tokenCostOutPerMillion: 60.0,
-      },
-      "openai:gpt-4-turbo": {
-        provider: "openai",
-        model: "gpt-4-turbo",
-        tokenCostInPerMillion: 10.0,
-        tokenCostOutPerMillion: 30.0,
-      },
-
-      // — OpenAI (2025) —
-      "openai:gpt-5": {
-        provider: "openai",
-        model: "gpt-5-chat-latest", // alias tracks GPT-5 pricing
-        tokenCostInPerMillion: 1.25,
-        tokenCostOutPerMillion: 10.0,
-      },
-
-      "openai:gpt-5-core": {
-        provider: "openai",
-        model: "gpt-5", // flagship
-        tokenCostInPerMillion: 1.25,
-        tokenCostOutPerMillion: 10.0,
-      },
-      "openai:gpt-5-chat": {
-        provider: "openai",
-        model: "gpt-5-chat-latest", // Chat variant
-        tokenCostInPerMillion: 1.25,
-        tokenCostOutPerMillion: 10.0,
-      },
-      "openai:gpt-5-pro": {
-        provider: "openai",
-        model: "gpt-5-pro", // higher-compute tier
-        tokenCostInPerMillion: 15.0,
-        tokenCostOutPerMillion: 120.0,
-      },
-      "openai:gpt-5-mini": {
-        provider: "openai",
-        model: "gpt-5-mini",
-        tokenCostInPerMillion: 0.25,
-        tokenCostOutPerMillion: 2.0,
-      },
-      "openai:gpt-5-nano": {
-        provider: "openai",
-        model: "gpt-5-nano",
-        tokenCostInPerMillion: 0.05,
-        tokenCostOutPerMillion: 0.4,
-      },
-
-      // — Google Gemini (2025) —
-      "gemini:2.5-pro": {
-        provider: "google",
-        model: "gemini-2.5-pro", // ≤200k input tier shown; >200k is higher
-        tokenCostInPerMillion: 1.25,
-        tokenCostOutPerMillion: 10.0,
-      },
-      "gemini:2.5-flash": {
-        provider: "google",
-        model: "gemini-2.5-flash",
-        tokenCostInPerMillion: 0.3,
-        tokenCostOutPerMillion: 2.5,
-      },
-      "gemini:2.5-flash-lite": {
-        provider: "google",
-        model: "gemini-2.5-flash-lite",
-        tokenCostInPerMillion: 0.1,
-        tokenCostOutPerMillion: 0.4,
-      },
-      "gemini:2.5-flash-image": {
-        provider: "google",
-        model: "gemini-2.5-flash-image",
-        // Inputs follow 2.5 Flash text pricing; outputs are **image tokens** at $30/M (≈$0.039 per 1024² image)
-        tokenCostInPerMillion: 0.3,
-        tokenCostOutPerMillion: 30.0,
-      },
-
-      // — Z.ai (formerly Zhipu) —
-      "zai:glm-4.6": {
-        provider: "zai",
-        model: "GLM-4.6",
-        tokenCostInPerMillion: 0.6,
-        tokenCostOutPerMillion: 2.2,
-      },
-      "zai:glm-4.5": {
-        provider: "zai",
-        model: "GLM-4.5",
-        tokenCostInPerMillion: 0.6,
-        tokenCostOutPerMillion: 2.2,
-      },
-      "zai:glm-4.5-air": {
-        provider: "zai",
-        model: "GLM-4.5-Air",
-        tokenCostInPerMillion: 0.2,
-        tokenCostOutPerMillion: 1.1,
-      },
-
-      // — Anthropic —
-      // current (Claude 4.5 / 4.1)
-      "anthropic:sonnet-4-5": {
-        provider: "anthropic",
-        model: "claude-sonnet-4-5",
-        tokenCostInPerMillion: 3.0,
-        tokenCostOutPerMillion: 15.0,
-      },
-      "anthropic:haiku-4-5": {
-        provider: "anthropic",
-        model: "claude-haiku-4-5",
-        tokenCostInPerMillion: 1.0,
-        tokenCostOutPerMillion: 5.0,
-      },
-      "anthropic:opus-4-1": {
-        provider: "anthropic",
-        model: "claude-opus-4-1",
-        tokenCostInPerMillion: 15.0,
-        tokenCostOutPerMillion: 75.0,
-      },
-
-      // legacy / still available
-      "anthropic:sonnet-4": {
-        provider: "anthropic",
-        model: "claude-sonnet-4-0",
-        tokenCostInPerMillion: 3.0,
-        tokenCostOutPerMillion: 15.0,
-      },
-      "anthropic:sonnet-3-7": {
-        provider: "anthropic",
-        model: "claude-3-7-sonnet-20250219",
-        tokenCostInPerMillion: 3.0,
-        tokenCostOutPerMillion: 15.0,
-      },
-      "anthropic:opus-4": {
-        provider: "anthropic",
-        model: "claude-opus-4-0",
-        tokenCostInPerMillion: 15.0,
-        tokenCostOutPerMillion: 75.0,
-      },
-      "anthropic:haiku-3-5": {
-        provider: "anthropic",
-        model: "claude-3-5-haiku-20241022",
-        tokenCostInPerMillion: 0.8,
-        tokenCostOutPerMillion: 4.0,
-      },
-    },
   },
   ui: {
     port: 3000,
@@ -645,14 +483,14 @@ export async function loadConfig(options = {}) {
     await validateConfig(config);
   }
 
-  // Cache the loaded config
+  // Cache
   currentConfig = config;
 
   return config;
 }
 
 /**
- * Get the current configuration
+ * Get current configuration
  * Loads default config if not already loaded
  *
  * @returns {Object} Current configuration

package/src/core/file-io.js

@@ -117,7 +117,7 @@ export function createTaskFileIO({ workDir, taskName, getStage, statusPath }) {
         logsDir,
         name,
         content,
-        options.mode || "append"
+        options.mode || "replace"
       );
       await updateStatusWithFiles("logs", name);
       return filePath;

package/src/core/module-loader.js

@@ -2,8 +2,6 @@ import fsp from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 
-const WORKSPACE_CACHE_DIR = path.join(process.cwd(), ".tmp-task-modules");
-
 /**
  * Convert supported modulePath formats into a file:// URL.
  * @param {string | URL} modulePath
@@ -88,70 +86,86 @@ function createMissingModuleError(modulePath, originalError) {
 }
 
 /**
- * Copy a module file into a workspace-local cache directory so Vite/Vitest can load it.
+ * Copy a module file adjacent to its original location with a unique name.
  * @param {string} sourcePath
  * @returns {Promise<string>}
  */
-async function copyModuleToWorkspaceCache(sourcePath) {
-  await fsp.mkdir(WORKSPACE_CACHE_DIR, { recursive: true });
+async function copyModuleAdjacent(sourcePath) {
+  const dir = path.dirname(sourcePath);
   const ext = path.extname(sourcePath) || ".js";
   const base = path.basename(sourcePath, ext);
   const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
-  const destFile = path.join(
-    WORKSPACE_CACHE_DIR,
-    `${base}.${uniqueSuffix}${ext}`
-  );
+  const destFile = path.join(dir, `.cache.${base}.${uniqueSuffix}${ext}`);
   await fsp.copyFile(sourcePath, destFile);
   return destFile;
 }
 
 /**
  * Dynamically import a module with cache busting while remaining compatible with Node's file:/// resolution.
- * Falls back to copying the module into a workspace-local cache when query parameters break filesystem resolution.
+ * Falls back to copying the module adjacent to its original location when query parameters break filesystem resolution.
  * @param {string | URL} modulePath
  * @returns {Promise<any>} Module namespace object
  */
 export async function loadFreshModule(modulePath) {
   const fileUrl = resolveToFileURL(modulePath);
-  const cacheBustedUrl = `${fileUrl.href}?t=${Date.now()}`;
 
+  // First attempt direct import without cache busting
   try {
-    return await import(cacheBustedUrl);
+    return await import(fileUrl.href);
   } catch (error) {
-    if (!isModuleNotFoundError(error) || fileUrl.protocol !== "file:") {
+    if (!isModuleNotFoundError(error)) {
       throw error;
     }
 
-    const absolutePath = fileURLToPath(fileUrl);
-
+    // Second attempt: try cache-busted import
+    const cacheBustedUrl = `${fileUrl.href}?t=${Date.now()}`;
     try {
-      await fsp.access(absolutePath);
-    } catch {
-      throw createMissingModuleError(
-        absolutePath,
-        /** @type {Error} */ (error)
-      );
-    }
+      return await import(cacheBustedUrl);
+    } catch (cacheBustedError) {
+      if (
+        !isModuleNotFoundError(cacheBustedError) ||
+        fileUrl.protocol !== "file:"
+      ) {
+        throw cacheBustedError;
+      }
 
-    try {
-      const cacheCopy = await copyModuleToWorkspaceCache(absolutePath);
-      const cacheUrl = `${pathToFileURL(cacheCopy).href}?t=${Date.now()}`;
-      return await import(cacheUrl);
-    } catch (fallbackError) {
-      const messageLines = [
-        `Failed to load module "${absolutePath}" after attempting cache-busting import.`,
-        `Cache-busted URL: ${cacheBustedUrl}`,
-        `Original error: ${/** @type {Error} */ (error).message}`,
-        `Fallback error: ${/** @type {Error} */ (fallbackError).message}`,
-      ];
-      const combined = new Error(messageLines.join("\n"));
-      if ("cause" in Error.prototype) {
-        combined.cause = fallbackError;
-      } else {
-        combined.fallbackError = fallbackError;
+      const absolutePath = fileURLToPath(fileUrl);
+
+      try {
+        await fsp.access(absolutePath);
+      } catch {
+        throw createMissingModuleError(
+          absolutePath,
+          /** @type {Error} */ (cacheBustedError)
+        );
+      }
+
+      // Third attempt: copy adjacent and import
+      let adjacentCopy;
+      try {
+        adjacentCopy = await copyModuleAdjacent(absolutePath);
+        const adjacentUrl = `${pathToFileURL(adjacentCopy).href}?t=${Date.now()}`;
+        return await import(adjacentUrl);
+      } catch (fallbackError) {
+        const messageLines = [
+          `Failed to load module "${absolutePath}" after attempting direct import, cache-busting import, and adjacent copy fallback.`,
+          `Direct import URL: ${fileUrl.href}`,
+          `Cache-busted URL: ${cacheBustedUrl}`,
+          `Adjacent fallback path attempted: ${adjacentCopy || "[adjacent copy creation failed]"}`,
+          `Original error: ${/** @type {Error} */ (error).message}`,
+          `Cache-bust error: ${/** @type {Error} */ (cacheBustedError).message}`,
+          `Fallback error: ${/** @type {Error} */ (fallbackError).message}`,
+        ];
+        const combined = new Error(messageLines.join("\n"));
+        if ("cause" in Error.prototype) {
+          combined.cause = fallbackError;
+        } else {
+          combined.fallbackError = fallbackError;
+        }
+        combined.initialError = error;
+        combined.cacheBustedError = cacheBustedError;
+        throw combined;
       }
-      combined.initialError = error;
-      throw combined;
     }
   }
 }

package/src/core/pipeline-runner.js

@@ -5,6 +5,10 @@ import { loadFreshModule } from "./module-loader.js";
 import { validatePipelineOrThrow } from "./validation.js";
 import { getPipelineConfig } from "./config.js";
 import { writeJobStatus } from "./status-writer.js";
+import { TaskState } from "../config/statuses.js";
+import { ensureTaskSymlinkBridge } from "./symlink-bridge.js";
+import { cleanupTaskSymlinks } from "./symlink-utils.js";
+import { createTaskFileIO } from "./file-io.js";
 
 const ROOT = process.env.PO_ROOT || process.cwd();
 const DATA_DIR = path.join(ROOT, process.env.PO_DATA_DIR || "pipeline-data");
@@ -18,6 +22,8 @@ if (!jobId) throw new Error("runner requires jobId as argument");
 
 const workDir = path.join(CURRENT_DIR, jobId);
 
+const startFromTask = process.env.PO_START_FROM_TASK;
+
 // Get pipeline slug from environment or fallback to seed.json
 let pipelineSlug = process.env.PO_PIPELINE_SLUG;
 if (!pipelineSlug) {
@@ -61,7 +67,15 @@ const seed = JSON.parse(
 let pipelineArtifacts = {};
 
 for (const taskName of pipeline.tasks) {
-  if (status.tasks[taskName]?.state === "done") {
+  // Skip tasks before startFromTask when targeting a specific restart point
+  if (
+    startFromTask &&
+    pipeline.tasks.indexOf(taskName) < pipeline.tasks.indexOf(startFromTask)
+  ) {
+    continue;
+  }
+
+  if (status.tasks[taskName]?.state === TaskState.DONE) {
     try {
       const outputPath = path.join(workDir, "tasks", taskName, "output.json");
       const output = JSON.parse(await fs.readFile(outputPath, "utf8"));
@@ -71,17 +85,13 @@ for (const taskName of pipeline.tasks) {
   }
 
   await updateStatus(taskName, {
-    state: "running",
+    state: TaskState.RUNNING,
     startedAt: now(),
     attempts: (status.tasks[taskName]?.attempts || 0) + 1,
   });
 
   const taskDir = path.join(workDir, "tasks", taskName);
   await fs.mkdir(taskDir, { recursive: true });
-  await atomicWrite(
-    path.join(taskDir, "letter.json"),
-    JSON.stringify({ task: taskName, at: now() }, null, 2)
-  );
 
   try {
     const ctx = {
@@ -104,14 +114,31 @@ for (const taskName of pipeline.tasks) {
       ? modulePath
       : path.resolve(path.dirname(TASK_REGISTRY), modulePath);
 
-    const result = await runPipeline(absoluteModulePath, ctx);
+    // Create symlink bridge for deterministic module resolution
+    const poRoot = process.env.PO_ROOT || process.cwd();
+    const relocatedEntry = await ensureTaskSymlinkBridge({
+      taskDir,
+      poRoot,
+      taskModulePath: absoluteModulePath,
+    });
+
+    // Create fileIO for this task
+    const fileIO = createTaskFileIO({
+      workDir,
+      taskName,
+      getStage: () => null, // pipeline-runner doesn't have stages
+      statusPath: tasksStatusPath,
+    });
+
+    const result = await runPipeline(relocatedEntry, ctx);
 
     if (!result.ok) {
-      // Persist execution-logs.json and failure-details.json on task failure
+      // Persist execution-logs.json and failure-details.json on task failure via IO
       if (result.logs) {
-        await atomicWrite(
-          path.join(taskDir, "execution-logs.json"),
-          JSON.stringify(result.logs, null, 2)
+        await fileIO.writeLog(
+          "execution-logs.json",
+          JSON.stringify(result.logs, null, 2),
+          { mode: "replace" }
         );
       }
       const failureDetails = {
@@ -121,14 +148,15 @@ for (const taskName of pipeline.tasks) {
         context: result.context,
         refinementAttempts: result.refinementAttempts || 0,
       };
-      await atomicWrite(
-        path.join(taskDir, "failure-details.json"),
-        JSON.stringify(failureDetails, null, 2)
+      await fileIO.writeLog(
+        "failure-details.json",
+        JSON.stringify(failureDetails, null, 2),
+        { mode: "replace" }
       );
 
       // Update tasks-status.json with enriched failure context
       await updateStatus(taskName, {
-        state: "failed",
+        state: TaskState.FAILED,
         endedAt: now(),
         error: result.error, // Don't double-normalize - use result.error as-is
         failedStage: result.failedStage,
@@ -156,14 +184,15 @@ for (const taskName of pipeline.tasks) {
     // No need to manually write output.json or enumerate artifacts
 
     if (result.logs) {
-      await atomicWrite(
-        path.join(taskDir, "execution-logs.json"),
-        JSON.stringify(result.logs, null, 2)
+      await fileIO.writeLog(
+        "execution-logs.json",
+        JSON.stringify(result.logs, null, 2),
+        { mode: "replace" }
       );
     }
 
     await updateStatus(taskName, {
-      state: "done",
+      state: TaskState.DONE,
       endedAt: now(),
       executionTimeMs:
         result.logs?.reduce((total, log) => total + (log.ms || 0), 0) || 0,
@@ -171,7 +200,7 @@ for (const taskName of pipeline.tasks) {
     });
   } catch (err) {
     await updateStatus(taskName, {
-      state: "failed",
+      state: TaskState.FAILED,
       endedAt: now(),
       error: normalizeError(err),
     });
@@ -201,6 +230,9 @@ await appendLine(
   }) + "\n"
 );
 
+// Clean up task symlinks to avoid dangling links in archives
+await cleanupTaskSymlinks(dest);
+
 function now() {
   return new Date().toISOString();
 }
@@ -225,12 +257,6 @@ async function appendLine(file, line) {
   await fs.appendFile(file, line);
 }
 
-async function atomicWrite(file, data) {
-  const tmp = file + ".tmp";
-  await fs.writeFile(tmp, data);
-  await fs.rename(tmp, file);
-}
-
 function normalizeError(e) {
   // If it's already a structured error object with a message string, pass it through
   if (e && typeof e === "object" && typeof e.message === "string") {