@ryanfw/prompt-orchestration-pipeline 0.4.0 → 0.6.0

package/src/core/config.js

@@ -182,39 +182,171 @@ export const defaultConfig = {
     llmRequestTimeout: 60000,
   },
   llm: {
-    defaultProvider: "openai",
-    defaultModel: "gpt-5-chat-latest",
+    defaultProvider: "deepseek",
+    defaultModel: "chat",
     maxConcurrency: 5,
     retryMaxAttempts: 3,
     retryBackoffMs: 1000,
     models: {
+      // DeepSeek (2025)
+      "deepseek:chat": {
+        provider: "deepseek",
+        model: "deepseek-chat", // V3.2 Exp (non-thinking) under the hood
+        tokenCostInPerMillion: 0.27,
+        tokenCostOutPerMillion: 1.1,
+      },
+      "deepseek:reasoner": {
+        provider: "deepseek",
+        model: "deepseek-reasoner", // R1 family
+        tokenCostInPerMillion: 0.55,
+        tokenCostOutPerMillion: 2.19,
+      },
+
+      // — OpenAI (2024 legacy still callable) —
       "openai:gpt-4": {
         provider: "openai",
         model: "gpt-4",
+        tokenCostInPerMillion: 30.0,
+        tokenCostOutPerMillion: 60.0,
       },
       "openai:gpt-4-turbo": {
         provider: "openai",
         model: "gpt-4-turbo",
+        tokenCostInPerMillion: 10.0,
+        tokenCostOutPerMillion: 30.0,
       },
+
+      // — OpenAI (2025) —
       "openai:gpt-5": {
         provider: "openai",
-        model: "gpt-5-chat-latest",
+        model: "gpt-5-chat-latest", // alias tracks GPT-5 pricing
+        tokenCostInPerMillion: 1.25,
+        tokenCostOutPerMillion: 10.0,
       },
-      "deepseek:reasoner": {
-        provider: "deepseek",
-        model: "deepseek-reasoner",
+
+      "openai:gpt-5-core": {
+        provider: "openai",
+        model: "gpt-5", // flagship
+        tokenCostInPerMillion: 1.25,
+        tokenCostOutPerMillion: 10.0,
       },
-      "deepseek:chat": {
-        provider: "deepseek",
-        model: "deepseek-chat",
+      "openai:gpt-5-chat": {
+        provider: "openai",
+        model: "gpt-5-chat-latest", // Chat variant
+        tokenCostInPerMillion: 1.25,
+        tokenCostOutPerMillion: 10.0,
+      },
+      "openai:gpt-5-pro": {
+        provider: "openai",
+        model: "gpt-5-pro", // higher-compute tier
+        tokenCostInPerMillion: 15.0,
+        tokenCostOutPerMillion: 120.0,
+      },
+      "openai:gpt-5-mini": {
+        provider: "openai",
+        model: "gpt-5-mini",
+        tokenCostInPerMillion: 0.25,
+        tokenCostOutPerMillion: 2.0,
+      },
+      "openai:gpt-5-nano": {
+        provider: "openai",
+        model: "gpt-5-nano",
+        tokenCostInPerMillion: 0.05,
+        tokenCostOutPerMillion: 0.4,
+      },
+
+      // — Google Gemini (2025) —
+      "gemini:2.5-pro": {
+        provider: "google",
+        model: "gemini-2.5-pro", // ≤200k input tier shown; >200k is higher
+        tokenCostInPerMillion: 1.25,
+        tokenCostOutPerMillion: 10.0,
+      },
+      "gemini:2.5-flash": {
+        provider: "google",
+        model: "gemini-2.5-flash",
+        tokenCostInPerMillion: 0.3,
+        tokenCostOutPerMillion: 2.5,
+      },
+      "gemini:2.5-flash-lite": {
+        provider: "google",
+        model: "gemini-2.5-flash-lite",
+        tokenCostInPerMillion: 0.1,
+        tokenCostOutPerMillion: 0.4,
+      },
+      "gemini:2.5-flash-image": {
+        provider: "google",
+        model: "gemini-2.5-flash-image",
+        // Inputs follow 2.5 Flash text pricing; outputs are **image tokens** at $30/M (≈$0.039 per 1024² image)
+        tokenCostInPerMillion: 0.3,
+        tokenCostOutPerMillion: 30.0,
+      },
+
+      // — Z.ai (formerly Zhipu) —
+      "zai:glm-4.6": {
+        provider: "zai",
+        model: "GLM-4.6",
+        tokenCostInPerMillion: 0.6,
+        tokenCostOutPerMillion: 2.2,
+      },
+      "zai:glm-4.5": {
+        provider: "zai",
+        model: "GLM-4.5",
+        tokenCostInPerMillion: 0.6,
+        tokenCostOutPerMillion: 2.2,
+      },
+      "zai:glm-4.5-air": {
+        provider: "zai",
+        model: "GLM-4.5-Air",
+        tokenCostInPerMillion: 0.2,
+        tokenCostOutPerMillion: 1.1,
+      },
+
+      // — Anthropic —
+      // current (Claude 4.5 / 4.1)
+      "anthropic:sonnet-4-5": {
+        provider: "anthropic",
+        model: "claude-sonnet-4-5",
+        tokenCostInPerMillion: 3.0,
+        tokenCostOutPerMillion: 15.0,
+      },
+      "anthropic:haiku-4-5": {
+        provider: "anthropic",
+        model: "claude-haiku-4-5",
+        tokenCostInPerMillion: 1.0,
+        tokenCostOutPerMillion: 5.0,
+      },
+      "anthropic:opus-4-1": {
+        provider: "anthropic",
+        model: "claude-opus-4-1",
+        tokenCostInPerMillion: 15.0,
+        tokenCostOutPerMillion: 75.0,
+      },
+
+      // legacy / still available
+      "anthropic:sonnet-4": {
+        provider: "anthropic",
+        model: "claude-sonnet-4-0",
+        tokenCostInPerMillion: 3.0,
+        tokenCostOutPerMillion: 15.0,
+      },
+      "anthropic:sonnet-3-7": {
+        provider: "anthropic",
+        model: "claude-3-7-sonnet-20250219",
+        tokenCostInPerMillion: 3.0,
+        tokenCostOutPerMillion: 15.0,
       },
-      "anthropic:opus": {
+      "anthropic:opus-4": {
         provider: "anthropic",
-        model: "claude-3-opus",
+        model: "claude-opus-4-0",
+        tokenCostInPerMillion: 15.0,
+        tokenCostOutPerMillion: 75.0,
       },
-      "anthropic:sonnet": {
+      "anthropic:haiku-3-5": {
         provider: "anthropic",
-        model: "claude-3-sonnet",
+        model: "claude-3-5-haiku-20241022",
+        tokenCostInPerMillion: 0.8,
+        tokenCostOutPerMillion: 4.0,
       },
     },
   },

package/src/core/file-io.js

@@ -1,5 +1,6 @@
 import fs from "node:fs/promises";
 import path from "node:path";
+import { writeJobStatus } from "./status-writer.js";
 
 /**
  * Creates a task-scoped file I/O interface that manages file operations
@@ -30,41 +31,25 @@ export function createTaskFileIO({ workDir, taskName, getStage, statusPath }) {
    * Updates tasks-status.json with file information, ensuring de-duplication
    */
   async function updateStatusWithFiles(fileType, fileName) {
-    try {
-      const statusContent = await fs.readFile(statusPath, "utf8");
-      const status = JSON.parse(statusContent);
-
-      // Initialize files object if it doesn't exist
-      if (!status.files) {
-        status.files = { artifacts: [], logs: [], tmp: [] };
-      }
-
-      // Initialize task files if they don't exist
-      if (!status.tasks[taskName].files) {
-        status.tasks[taskName].files = { artifacts: [], logs: [], tmp: [] };
-      }
-
-      // Add to job-level files array (de-duped)
-      const jobArray = status.files[fileType];
+    const jobDir = path.dirname(statusPath);
+    await writeJobStatus(jobDir, (snapshot) => {
+      snapshot.files ||= { artifacts: [], logs: [], tmp: [] };
+      snapshot.tasks ||= {};
+      snapshot.tasks[taskName] ||= {};
+      snapshot.tasks[taskName].files ||= { artifacts: [], logs: [], tmp: [] };
+
+      const jobArray = snapshot.files[fileType];
       if (!jobArray.includes(fileName)) {
         jobArray.push(fileName);
       }
 
-      // Add to task-level files array (de-duped)
-      const taskArray = status.tasks[taskName].files[fileType];
+      const taskArray = snapshot.tasks[taskName].files[fileType];
       if (!taskArray.includes(fileName)) {
         taskArray.push(fileName);
       }
 
-      // Write back to file atomically
-      await atomicWrite(statusPath, JSON.stringify(status, null, 2));
-    } catch (error) {
-      // If status file doesn't exist or is invalid, we'll log but not fail
-      console.warn(
-        `Failed to update status with file ${fileName}:`,
-        error.message
-      );
-    }
+      return snapshot;
+    });
   }
 
   /**

package/src/core/orchestrator.js

@@ -220,97 +220,111 @@ export async function startOrchestrator(opts) {
  * @param {Object} seed - Seed data containing pipeline information
  */
 function spawnRunner(jobId, dirs, running, spawn, testMode, seed) {
-  const runnerPath = path.join(
-    process.cwd(),
-    "src",
-    "core",
-    "pipeline-runner.js"
-  );
-
-  const configSnapshot = getConfig();
-  const availablePipelines = Object.keys(configSnapshot?.pipelines ?? {});
-  const pipelineSlug = seed?.pipeline;
-
-  console.log("[Orchestrator] spawnRunner invoked", {
-    jobId,
-    pipelineSlug: pipelineSlug ?? null,
-    availablePipelines,
-    seedKeys: seed ? Object.keys(seed) : null,
-  });
+  // Use path relative to this file to avoid process.cwd() issues
+  const orchestratorDir = path.dirname(new URL(import.meta.url).pathname);
+  const runnerPath = path.join(orchestratorDir, "pipeline-runner.js");
 
-  if (!availablePipelines.length) {
-    console.warn(
-      "[Orchestrator] No pipelines registered in config() when spawnRunner invoked"
-    );
-  } else if (!availablePipelines.includes(pipelineSlug)) {
-    console.warn(
-      "[Orchestrator] Requested pipeline slug missing from registry snapshot",
-      {
-        jobId,
-        pipelineSlug,
-        availablePipelines,
-      }
-    );
-  }
+  // Set PO_ROOT for the orchestrator process to match what the runner will use
+  const originalPoRoot = process.env.PO_ROOT;
+  const poRoot = path.resolve(dirs.dataDir, "..");
+  process.env.PO_ROOT = poRoot;
 
-  if (!pipelineSlug) {
-    console.error("[Orchestrator] Missing pipeline slug in seed", {
+  try {
+    const configSnapshot = getConfig();
+    const availablePipelines = Object.keys(configSnapshot?.pipelines ?? {});
+    const pipelineSlug = seed?.pipeline;
+
+    console.log("[Orchestrator] spawnRunner invoked", {
       jobId,
-      seed,
+      pipelineSlug: pipelineSlug ?? null,
       availablePipelines,
+      seedKeys: seed ? Object.keys(seed) : null,
     });
-    throw new Error(
-      "Pipeline slug is required in seed data. Include a 'pipeline' field in your seed."
-    );
-  }
 
-  let pipelineConfig;
-  try {
-    pipelineConfig = getPipelineConfig(pipelineSlug);
-  } catch (error) {
-    console.error("[Orchestrator] Pipeline lookup failed", {
-      jobId,
-      pipelineSlug,
-      availablePipelines,
+    if (!availablePipelines.length) {
+      console.warn(
+        "[Orchestrator] No pipelines registered in config() when spawnRunner invoked"
+      );
+    } else if (!availablePipelines.includes(pipelineSlug)) {
+      console.warn(
+        "[Orchestrator] Requested pipeline slug missing from registry snapshot",
+        {
+          jobId,
+          pipelineSlug,
+          availablePipelines,
+        }
+      );
+    }
+
+    if (!pipelineSlug) {
+      console.error("[Orchestrator] Missing pipeline slug in seed", {
+        jobId,
+        seed,
+        availablePipelines,
+      });
+      throw new Error(
+        "Pipeline slug is required in seed data. Include a 'pipeline' field in your seed."
+      );
+    }
+
+    let pipelineConfig;
+    try {
+      pipelineConfig = getPipelineConfig(pipelineSlug);
+    } catch (error) {
+      console.error("[Orchestrator] Pipeline lookup failed", {
+        jobId,
+        pipelineSlug,
+        availablePipelines,
+      });
+      throw error;
+    }
+
+    // Use environment variables with explicit slug propagation
+    // PO_ROOT should point to the directory containing pipeline-config
+    // In our case, it's the parent of pipeline-data directory
+    const env = {
+      ...process.env,
+      PO_ROOT: poRoot,
+      PO_DATA_DIR: dirs.dataDir,
+      PO_PENDING_DIR: dirs.pending,
+      PO_CURRENT_DIR: dirs.current,
+      PO_COMPLETE_DIR: dirs.complete,
+      PO_PIPELINE_SLUG: pipelineSlug,
+      // Force mock provider for testing
+      PO_DEFAULT_PROVIDER: "mock",
+    };
+
+    // Always call spawn so tests can capture it
+    const child = spawn(process.execPath, [runnerPath, jobId], {
+      stdio: ["ignore", "inherit", "inherit"],
+      env,
+      cwd: process.cwd(),
     });
-    throw error;
-  }
 
-  // Use environment variables with explicit slug propagation
-  const env = {
-    ...process.env,
-    PO_DATA_DIR: dirs.dataDir,
-    PO_PENDING_DIR: dirs.pending,
-    PO_CURRENT_DIR: dirs.current,
-    PO_COMPLETE_DIR: dirs.complete,
-    PO_PIPELINE_SLUG: pipelineSlug,
-    // Force mock provider for testing
-    PO_DEFAULT_PROVIDER: "mock",
-  };
-
-  // Always call spawn so tests can capture it
-  const child = spawn(process.execPath, [runnerPath, jobId], {
-    stdio: ["ignore", "inherit", "inherit"],
-    env,
-    cwd: process.cwd(),
-  });
+    running.set(jobId, child);
 
-  running.set(jobId, child);
+    child.on("exit", () => {
+      running.delete(jobId);
+    });
+    child.on("error", () => {
+      running.delete(jobId);
+    });
 
-  child.on("exit", () => {
-    running.delete(jobId);
-  });
-  child.on("error", () => {
-    running.delete(jobId);
-  });
+    // In test mode: return immediately; in real mode you might await readiness
+    if (testMode) {
+      return child;
+    }
 
-  // In test mode: return immediately; in real mode you might await readiness
-  if (testMode) {
+    // Non-test: we can consider "started" immediately for simplicity
     return child;
+  } finally {
+    // Restore original PO_ROOT
+    if (originalPoRoot) {
+      process.env.PO_ROOT = originalPoRoot;
+    } else {
+      delete process.env.PO_ROOT;
+    }
   }
-
-  // Non-test: we can consider "started" immediately for simplicity
-  return child;
 }
 
 export default { startOrchestrator };

package/src/core/pipeline-runner.js

@@ -4,6 +4,7 @@ import { runPipeline } from "./task-runner.js";
 import { loadFreshModule } from "./module-loader.js";
 import { validatePipelineOrThrow } from "./validation.js";
 import { getPipelineConfig } from "./config.js";
+import { writeJobStatus } from "./status-writer.js";
 
 const ROOT = process.env.PO_ROOT || process.cwd();
 const DATA_DIR = path.join(ROOT, process.env.PO_DATA_DIR || "pipeline-data");
@@ -205,12 +206,18 @@ function now() {
 }
 
 async function updateStatus(taskName, patch) {
-  const current = JSON.parse(await fs.readFile(tasksStatusPath, "utf8"));
-  current.current = taskName;
-  current.tasks = current.tasks || {};
-  current.tasks[taskName] = { ...(current.tasks[taskName] || {}), ...patch };
-  await atomicWrite(tasksStatusPath, JSON.stringify(current, null, 2));
-  Object.assign(status, current);
+  return await writeJobStatus(workDir, (snapshot) => {
+    snapshot.current = taskName;
+    snapshot.tasks = snapshot.tasks || {};
+    snapshot.tasks[taskName] = {
+      ...(snapshot.tasks[taskName] || {}),
+      ...patch,
+    };
+    return snapshot;
+  }).then((snap) => {
+    Object.assign(status, snap);
+    return snap;
+  });
 }
 
 async function appendLine(file, line) {

package/src/core/status-writer.js

@@ -16,6 +16,9 @@ async function getSSERegistry() {
   return sseRegistry;
 }
 
+// Per-job write queues to serialize writes to tasks-status.json
+const writeQueues = new Map(); // Map<string jobDir, Promise<any>>
+
 // Instrumentation helper for status writer
 const createStatusWriterLogger = (jobId) => {
   const prefix = `[StatusWriter:${jobId || "unknown"}]`;
@@ -193,66 +196,74 @@ export async function writeJobStatus(jobDir, updateFn) {
   const jobId = path.basename(jobDir);
   const logger = createStatusWriterLogger(jobId);
 
-  logger.group("Status Write Operation");
-  logger.log(`Updating status for job: ${jobId}`);
-  logger.log(`Status file path: ${statusPath}`);
+  // Get or create the write queue for this job directory
+  const prev = writeQueues.get(jobDir) || Promise.resolve();
+  let resultSnapshot;
 
-  // Read existing status or create default
-  let snapshot = await readStatusFile(statusPath, jobId);
-  logger.log("Current status snapshot:", snapshot);
+  const next = prev
+    .then(async () => {
+      logger.group("Status Write Operation");
+      logger.log(`Updating status for job: ${jobId}`);
+      logger.log(`Status file path: ${statusPath}`);
 
-  // Validate basic structure
-  snapshot = validateStatusSnapshot(snapshot);
+      // Read existing status or create default
+      const current = await readStatusFile(statusPath, jobId);
+      logger.log("Current status snapshot:", current);
 
-  // Apply user updates
-  try {
-    const result = updateFn(snapshot);
-    // If updateFn returns a value, use it as new snapshot
-    if (result !== undefined) {
-      snapshot = result;
-    }
-    logger.log("Status after update function:", snapshot);
-  } catch (error) {
-    logger.error("Update function failed:", error);
-    throw new Error(`Update function failed: ${error.message}`);
-  }
+      // Validate basic structure
+      const validated = validateStatusSnapshot(current);
 
-  // Validate final structure
-  snapshot = validateStatusSnapshot(snapshot);
+      // Apply user updates
+      const maybeUpdated = updateFn(validated);
+      const snapshot = validateStatusSnapshot(
+        maybeUpdated === undefined ? validated : maybeUpdated
+      );
 
-  // Update timestamp
-  snapshot.lastUpdated = new Date().toISOString();
+      snapshot.lastUpdated = new Date().toISOString();
+      logger.log("Status after update function:", snapshot);
+
+      // Atomic write
+      await atomicWrite(statusPath, snapshot);
+      logger.log("Status file written successfully");
+
+      // Emit SSE event for tasks-status.json change
+      const registry = (await getSSERegistry().catch(() => null)) || null;
+      if (registry) {
+        try {
+          const eventData = {
+            type: "state:change",
+            data: {
+              path: path.join(jobDir, "tasks-status.json"),
+              id: jobId,
+              jobId,
+            },
+          };
+          registry.broadcast(eventData);
+          logger.sse("state:change", eventData.data);
+          logger.log("SSE event broadcasted successfully");
+        } catch (error) {
+          // Don't fail the write if SSE emission fails
+          logger.error("Failed to emit SSE event:", error);
+          console.warn(`Failed to emit SSE event: ${error.message}`);
+        }
+      } else {
+        logger.warn("SSE registry not available - no event broadcasted");
+      }
 
-  // Atomic write
-  await atomicWrite(statusPath, snapshot);
-  logger.log("Status file written successfully");
+      logger.groupEnd();
+      resultSnapshot = snapshot;
+    })
+    .catch((e) => {
+      throw e;
+    });
 
-  // Emit SSE event for tasks-status.json change
-  const registry = await getSSERegistry();
-  if (registry) {
-    try {
-      const eventData = {
-        type: "state:change",
-        data: {
-          path: path.join(jobDir, "tasks-status.json"),
-          id: jobId,
-          jobId,
-        },
-      };
-      registry.broadcast(eventData);
-      logger.sse("state:change", eventData.data);
-      logger.log("SSE event broadcasted successfully");
-    } catch (error) {
-      // Don't fail the write if SSE emission fails
-      logger.error("Failed to emit SSE event:", error);
-      console.warn(`Failed to emit SSE event: ${error.message}`);
-    }
-  } else {
-    logger.warn("SSE registry not available - no event broadcasted");
-  }
+  // Store the promise chain and set up cleanup
+  writeQueues.set(
+    jobDir,
+    next.finally(() => {})
+  );
 
-  logger.groupEnd();
-  return snapshot;
+  return next.then(() => resultSnapshot);
 }
 
 /**