@ryanfw/prompt-orchestration-pipeline 0.0.1

package/src/core/orchestrator.js

@@ -0,0 +1,335 @@
+// ESM
+import fs from "node:fs/promises";
+import path from "node:path";
+import crypto from "node:crypto";
+import chokidar from "chokidar";
+import { spawn } from "node:child_process";
+import url from "node:url";
+import { validateSeed, formatValidationErrors } from "./validation.js";
+import { getConfig } from "./config.js";
+import { withRetry } from "./retry.js";
+
+export class Orchestrator {
+  constructor({ paths, pipelineDefinition }) {
+    this.paths = paths;
+    this.pipelineDefinition = pipelineDefinition;
+    this.runningProcesses = new Map();
+    this.watcher = null;
+  }
+
+  async start() {
+    await fs.mkdir(this.paths.pending, { recursive: true });
+    await fs.mkdir(this.paths.current, { recursive: true });
+    await fs.mkdir(this.paths.complete, { recursive: true });
+
+    for (const name of await this.#listDirs(this.paths.current)) {
+      this.#ensureRunner(name);
+    }
+
+    const config = getConfig();
+    this.watcher = chokidar
+      .watch(path.join(this.paths.pending, "*-seed.json"), {
+        awaitWriteFinish: {
+          stabilityThreshold: config.orchestrator.watchStabilityThreshold,
+          pollInterval: config.orchestrator.watchPollInterval,
+        },
+      })
+      .on("add", (p) => this.#onSeed(p));
+
+    return this;
+  }
+
+  async stop() {
+    if (this.watcher) {
+      await this.watcher.close();
+      this.watcher = null;
+    }
+
+    for (const [name, info] of this.runningProcesses) {
+      info.process.kill("SIGTERM");
+    }
+
+    // Skip the shutdown timeout in test environment
+    if (process.env.NODE_ENV !== "test") {
+      const config = getConfig();
+      await new Promise((r) =>
+        setTimeout(r, config.orchestrator.shutdownTimeout)
+      );
+    }
+
+    for (const [name, info] of this.runningProcesses) {
+      if (!info.process.killed) info.process.kill("SIGKILL");
+    }
+
+    this.runningProcesses.clear();
+  }
+
+  async #onSeed(seedPath) {
+    const base = path.basename(seedPath);
+    const name = base.replace(/-seed\.json$/, "");
+    const workDir = path.join(this.paths.current, name);
+    const lockFile = path.join(this.paths.current, `${name}.lock`);
+
+    try {
+      await fs.writeFile(lockFile, process.pid.toString(), { flag: "wx" });
+    } catch (err) {
+      if (err.code === "EEXIST") return;
+      throw err;
+    }
+
+    try {
+      try {
+        await fs.mkdir(workDir, { recursive: false });
+      } catch (err) {
+        if (err.code === "EEXIST") return;
+        throw err;
+      }
+
+      const seed = JSON.parse(await fs.readFile(seedPath, "utf8"));
+
+      // Validate seed file structure
+      const validation = validateSeed(seed);
+      if (!validation.valid) {
+        const errorMsg = formatValidationErrors(validation.errors);
+        console.error(`Invalid seed file ${base}:\n${errorMsg}`);
+        // Move invalid seed to a rejected directory for inspection
+        const rejectedDir = path.join(
+          path.dirname(this.paths.pending),
+          "rejected"
+        );
+        await fs.mkdir(rejectedDir, { recursive: true });
+        const rejectedPath = path.join(rejectedDir, base);
+        await fs.rename(seedPath, rejectedPath);
+        return;
+      }
+
+      const pipelineId = this.#makeId();
+
+      await this.#atomicWrite(
+        path.join(workDir, "seed.json"),
+        JSON.stringify(seed, null, 2)
+      );
+      await this.#atomicWrite(
+        path.join(workDir, "tasks-status.json"),
+        JSON.stringify(
+          {
+            pipelineId,
+            name,
+            current: null,
+            createdAt: new Date().toISOString(),
+            tasks: {},
+          },
+          null,
+          2
+        )
+      );
+
+      await fs.mkdir(path.join(workDir, "tasks"), { recursive: true });
+
+      // Move the seed file to a 'processed' directory after successful processing
+      const processedDir = path.join(
+        path.dirname(this.paths.pending),
+        "processed"
+      );
+      await fs.mkdir(processedDir, { recursive: true });
+      const processedPath = path.join(processedDir, base);
+      await fs.rename(seedPath, processedPath);
+    } finally {
+      try {
+        await fs.unlink(lockFile);
+      } catch {}
+    }
+
+    // Start runner after all file operations are complete
+    this.#ensureRunner(name);
+  }
+
+  #ensureRunner(name) {
+    if (this.runningProcesses.has(name)) return;
+
+    const config = getConfig();
+
+    // Wrap process spawn in retry logic (fire-and-forget)
+    // This is intentionally not awaited - we want to start runners asynchronously
+    // and let them run in the background. Failures are handled via dead letter queue.
+    withRetry(() => this.#spawnRunner(name), {
+      maxAttempts: config.orchestrator.processSpawnRetries,
+      initialDelay: config.orchestrator.processSpawnRetryDelay,
+      onRetry: ({ attempt, delay, error }) => {
+        console.warn(
+          `Failed to start pipeline ${name} (attempt ${attempt}): ${error.message}. Retrying in ${delay}ms...`
+        );
+      },
+      shouldRetry: (error) => {
+        // Don't retry if the error is due to missing files or invalid config
+        const nonRetryableCodes = ["ENOENT", "EACCES", "MODULE_NOT_FOUND"];
+        const nonRetryableMessages = ["Invalid pipeline"];
+        if (error.code && nonRetryableCodes.includes(error.code)) {
+          return false;
+        }
+        if (error.message && nonRetryableMessages.includes(error.message)) {
+          return false;
+        }
+        return true;
+      },
+    }).catch((error) => {
+      console.error(
+        `Failed to start pipeline ${name} after ${config.orchestrator.processSpawnRetries} attempts:`,
+        error
+      );
+      // Move to dead letter queue
+      this.#moveToDeadLetter(name, error).catch((dlqError) => {
+        console.error(`Failed to move ${name} to dead letter queue:`, dlqError);
+      });
+    });
+  }
+
+  #spawnRunner(name) {
+    return new Promise((resolve, reject) => {
+      const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
+      const runnerPath = path.join(__dirname, "pipeline-runner.js");
+
+      const env = {
+        ...process.env,
+        PO_ROOT: process.cwd(),
+        PO_DATA_DIR: path.relative(
+          process.cwd(),
+          path.dirname(this.paths.pending)
+        ),
+        PO_CURRENT_DIR: this.paths.current,
+        PO_COMPLETE_DIR: this.paths.complete,
+        PO_CONFIG_DIR: path.join(process.cwd(), "pipeline-config"),
+        PO_PIPELINE_PATH:
+          this.pipelineDefinition?.__path ||
+          path.join(process.cwd(), "pipeline-config", "pipeline.json"),
+        PO_TASK_REGISTRY: path.join(
+          process.cwd(),
+          "pipeline-config",
+          "tasks/index.js"
+        ),
+      };
+
+      const child = spawn(process.execPath, [runnerPath, name], {
+        stdio: ["ignore", "inherit", "inherit"],
+        env,
+        cwd: process.cwd(),
+      });
+
+      // Track if process started successfully
+      let started = false;
+
+      // Consider spawn successful after a short delay
+      const startupTimeout = setTimeout(() => {
+        started = true;
+        resolve();
+      }, 100);
+
+      this.runningProcesses.set(name, {
+        process: child,
+        startedAt: new Date().toISOString(),
+        name,
+      });
+
+      child.on("exit", (code, signal) => {
+        clearTimeout(startupTimeout);
+        this.runningProcesses.delete(name);
+        if (code !== 0) {
+          console.error(
+            `Pipeline ${name} exited with code ${code}, signal ${signal}`
+          );
+        } else {
+          console.log(`Pipeline ${name} completed successfully`);
+        }
+      });
+
+      child.on("error", (err) => {
+        clearTimeout(startupTimeout);
+        this.runningProcesses.delete(name);
+        if (!started) {
+          reject(err);
+        } else {
+          console.error(`Pipeline ${name} encountered error:`, err);
+        }
+      });
+    });
+  }
+
+  async #moveToDeadLetter(name, error) {
+    const workDir = path.join(this.paths.current, name);
+    const deadLetterDir = path.join(
+      path.dirname(this.paths.pending),
+      "dead-letter"
+    );
+    await fs.mkdir(deadLetterDir, { recursive: true });
+
+    const errorLog = {
+      name,
+      error: {
+        message: error.message,
+        stack: error.stack,
+      },
+      timestamp: new Date().toISOString(),
+      attempts: getConfig().orchestrator.processSpawnRetries,
+    };
+
+    await this.#atomicWrite(
+      path.join(deadLetterDir, `${name}-error.json`),
+      JSON.stringify(errorLog, null, 2)
+    );
+
+    // Move the work directory to dead letter
+    const deadLetterWorkDir = path.join(deadLetterDir, name);
+    try {
+      await fs.rename(workDir, deadLetterWorkDir);
+    } catch (err) {
+      // If rename fails, try to copy
+      console.warn(`Could not move ${name} to dead letter, attempting copy`);
+      // If rename fails, try to copy
+      console.warn(`Could not move ${name} to dead letter, attempting copy`);
+      try {
+        await this.#copyDirRecursive(workDir, deadLetterWorkDir);
+        await fs.rm(workDir, { recursive: true, force: true });
+      } catch (copyErr) {
+        console.error(`Failed to copy ${name} to dead letter:`, copyErr);
+      }
+    }
+  }
+
+  async #copyDirRecursive(src, dest) {
+    await fs.mkdir(dest, { recursive: true });
+    const entries = await fs.readdir(src, { withFileTypes: true });
+    for (const entry of entries) {
+      const srcPath = path.join(src, entry.name);
+      const destPath = path.join(dest, entry.name);
+      if (entry.isDirectory()) {
+        await this.#copyDirRecursive(srcPath, destPath);
+      } else if (entry.isFile()) {
+        await fs.copyFile(srcPath, destPath);
+      }
+    }
+  }
+  async #listDirs(dir) {
+    try {
+      const entries = await fs.readdir(dir, { withFileTypes: true });
+      return entries.filter((e) => e.isDirectory()).map((e) => e.name);
+    } catch (err) {
+      if (err.code === "ENOENT") return [];
+      throw err;
+    }
+  }
+
+  #makeId() {
+    return (
+      "pl-" +
+      new Date().toISOString().replaceAll(/[:.]/g, "-") +
+      "-" +
+      crypto.randomBytes(3).toString("hex")
+    );
+  }
+
+  async #atomicWrite(file, data) {
+    const tmp = file + ".tmp";
+    await fs.writeFile(tmp, data);
+    await fs.rename(tmp, file);
+  }
+}

package/src/core/pipeline-runner.js

@@ -0,0 +1,182 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { runPipeline } from "./task-runner.js";
+
+const ROOT = process.env.PO_ROOT || process.cwd();
+const DATA_DIR = path.join(ROOT, process.env.PO_DATA_DIR || "pipeline-data");
+const CURRENT_DIR =
+  process.env.PO_CURRENT_DIR || path.join(DATA_DIR, "current");
+const COMPLETE_DIR =
+  process.env.PO_COMPLETE_DIR || path.join(DATA_DIR, "complete");
+
+const CONFIG_DIR =
+  process.env.PO_CONFIG_DIR || path.join(ROOT, "pipeline-config");
+const TASK_REGISTRY =
+  process.env.PO_TASK_REGISTRY || path.join(CONFIG_DIR, "tasks/index.js");
+const PIPELINE_DEF_PATH =
+  process.env.PO_PIPELINE_PATH || path.join(CONFIG_DIR, "pipeline.json");
+
+const name = process.argv[2];
+if (!name) throw new Error("runner requires pipeline name");
+
+const workDir = path.join(CURRENT_DIR, name);
+const tasksStatusPath = path.join(workDir, "tasks-status.json");
+
+const pipeline = JSON.parse(await fs.readFile(PIPELINE_DEF_PATH, "utf8"));
+// Add cache busting to force task registry reload
+const taskRegistryUrl = `${pathToFileURL(TASK_REGISTRY).href}?t=${Date.now()}`;
+const tasks = (await import(taskRegistryUrl)).default;
+
+const status = JSON.parse(await fs.readFile(tasksStatusPath, "utf8"));
+const seed = JSON.parse(
+  await fs.readFile(path.join(workDir, "seed.json"), "utf8")
+);
+
+let pipelineArtifacts = {};
+
+for (const taskName of pipeline.tasks) {
+  if (status.tasks[taskName]?.state === "done") {
+    try {
+      const outputPath = path.join(workDir, "tasks", taskName, "output.json");
+      const output = JSON.parse(await fs.readFile(outputPath, "utf8"));
+      pipelineArtifacts[taskName] = output;
+    } catch {}
+    continue;
+  }
+
+  await updateStatus(taskName, {
+    state: "running",
+    startedAt: now(),
+    attempts: (status.tasks[taskName]?.attempts || 0) + 1,
+  });
+
+  const taskDir = path.join(workDir, "tasks", taskName);
+  await fs.mkdir(taskDir, { recursive: true });
+  await atomicWrite(
+    path.join(taskDir, "letter.json"),
+    JSON.stringify({ task: taskName, at: now() }, null, 2)
+  );
+
+  try {
+    const ctx = {
+      workDir,
+      taskDir,
+      seed,
+      artifacts: pipelineArtifacts,
+      taskName,
+      taskConfig: pipeline.taskConfig?.[taskName] || {},
+    };
+    const modulePath = tasks[taskName];
+    if (!modulePath) throw new Error(`Task not registered: ${taskName}`);
+
+    // Resolve relative paths from task registry to absolute paths
+    const absoluteModulePath = path.isAbsolute(modulePath)
+      ? modulePath
+      : path.resolve(path.dirname(TASK_REGISTRY), modulePath);
+
+    const result = await runPipeline(absoluteModulePath, ctx);
+
+    if (!result.ok) {
+      throw new Error(
+        `${taskName} failed after ${result.refinementAttempts || 0} attempts: ${result.error?.message || "unknown"}`
+      );
+    }
+
+    if (result.context?.output) {
+      await atomicWrite(
+        path.join(taskDir, "output.json"),
+        JSON.stringify(result.context.output, null, 2)
+      );
+      pipelineArtifacts[taskName] = result.context.output;
+    }
+
+    if (result.logs) {
+      await atomicWrite(
+        path.join(taskDir, "execution-logs.json"),
+        JSON.stringify(result.logs, null, 2)
+      );
+    }
+
+    const artifacts = await getArtifacts(taskDir);
+    await updateStatus(taskName, {
+      state: "done",
+      endedAt: now(),
+      artifacts,
+      executionTime:
+        result.logs?.reduce((total, log) => total + (log.ms || 0), 0) || 0,
+      refinementAttempts: result.refinementAttempts || 0,
+    });
+  } catch (err) {
+    await updateStatus(taskName, {
+      state: "failed",
+      endedAt: now(),
+      error: normalizeError(err),
+    });
+    process.exitCode = 1;
+    process.exit(1);
+  }
+}
+
+await fs.mkdir(COMPLETE_DIR, { recursive: true });
+const dest = path.join(COMPLETE_DIR, name);
+await fs.rename(workDir, dest);
+await appendLine(
+  path.join(COMPLETE_DIR, "runs.jsonl"),
+  JSON.stringify({
+    name,
+    pipelineId: status.pipelineId,
+    finishedAt: now(),
+    tasks: Object.keys(status.tasks),
+    totalExecutionTime: Object.values(status.tasks).reduce(
+      (total, t) => total + (t.executionTime || 0),
+      0
+    ),
+    totalRefinementAttempts: Object.values(status.tasks).reduce(
+      (total, t) => total + (t.refinementAttempts || 0),
+      0
+    ),
+    finalArtifacts: Object.keys(pipelineArtifacts),
+  }) + "\n"
+);
+
+function now() {
+  return new Date().toISOString();
+}
+
+async function updateStatus(taskName, patch) {
+  const current = JSON.parse(await fs.readFile(tasksStatusPath, "utf8"));
+  current.current = taskName;
+  current.tasks[taskName] = { ...(current.tasks[taskName] || {}), ...patch };
+  await atomicWrite(tasksStatusPath, JSON.stringify(current, null, 2));
+  Object.assign(status, current);
+}
+
+async function appendLine(file, line) {
+  await fs.mkdir(path.dirname(file), { recursive: true });
+  await fs.appendFile(file, line);
+}
+
+async function atomicWrite(file, data) {
+  const tmp = file + ".tmp";
+  await fs.writeFile(tmp, data);
+  await fs.rename(tmp, file);
+}
+
+function normalizeError(e) {
+  if (e instanceof Error)
+    return { name: e.name, message: e.message, stack: e.stack };
+  return { message: String(e) };
+}
+
+async function getArtifacts(dir) {
+  const potentialFiles = ["output.json", "letter.json", "execution-logs.json"];
+  const artifacts = [];
+  for (const file of potentialFiles) {
+    try {
+      await fs.stat(path.join(dir, file));
+      artifacts.push(file);
+    } catch {}
+  }
+  return artifacts;
+}

package/src/core/retry.js

@@ -0,0 +1,83 @@
+/**
+ * Retry utilities for handling transient failures
+ * Implements exponential backoff with configurable options
+ */
+
+/**
+ * Sleep for specified milliseconds
+ * @param {number} ms - Milliseconds to sleep
+ * @returns {Promise<void>}
+ */
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Execute a function with retry logic and exponential backoff
+ * @param {Function} fn - Async function to execute
+ * @param {object} options - Retry options
+ * @param {number} options.maxAttempts - Maximum number of attempts (default: 3)
+ * @param {number} options.initialDelay - Initial delay in ms (default: 1000)
+ * @param {number} options.maxDelay - Maximum delay in ms (default: 10000)
+ * @param {number} options.backoffMultiplier - Backoff multiplier (default: 2)
+ * @param {Function} options.onRetry - Callback on retry (default: noop)
+ * @param {Function} options.shouldRetry - Function to determine if error should be retried (default: always retry)
+ * @returns {Promise<any>} Result of successful function execution
+ * @throws {Error} Last error if all attempts fail
+ */
+export async function withRetry(fn, options = {}) {
+  const {
+    maxAttempts = 3,
+    initialDelay = 1000,
+    maxDelay = 10000,
+    backoffMultiplier = 2,
+    onRetry = () => {},
+    shouldRetry = () => true,
+  } = options;
+
+  let lastError;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+
+      // Check if we should retry this error
+      if (!shouldRetry(error)) {
+        throw error;
+      }
+
+      // If this was the last attempt, throw the error
+      if (attempt === maxAttempts) {
+        break;
+      }
+
+      // Calculate delay with exponential backoff
+      const delay = Math.min(
+        initialDelay * Math.pow(backoffMultiplier, attempt - 1),
+        maxDelay
+      );
+
+      // Call retry callback
+      onRetry({ attempt, delay, error, maxAttempts });
+
+      // Wait before retrying
+      await sleep(delay);
+    }
+  }
+
+  // All attempts failed
+  throw lastError;
+}
+
+/**
+ * Create a retry wrapper with preset options
+ * @param {object} defaultOptions - Default retry options
+ * @returns {Function} Retry function with preset options
+ */
+export function createRetryWrapper(defaultOptions = {}) {
+  return (fn, options = {}) => {
+    return withRetry(fn, { ...defaultOptions, ...options });
+  };
+}