@ryanfw/prompt-orchestration-pipeline 0.0.1 → 0.4.0

package/src/core/orchestrator.js

@@ -1,335 +1,316 @@
-// ESM
+// ESM Orchestrator - clean, test-friendly, no JSX or ellipses
 import fs from "node:fs/promises";
 import path from "node:path";
-import crypto from "node:crypto";
 import chokidar from "chokidar";
-import { spawn } from "node:child_process";
-import url from "node:url";
-import { validateSeed, formatValidationErrors } from "./validation.js";
-import { getConfig } from "./config.js";
-import { withRetry } from "./retry.js";
-
-export class Orchestrator {
-  constructor({ paths, pipelineDefinition }) {
-    this.paths = paths;
-    this.pipelineDefinition = pipelineDefinition;
-    this.runningProcesses = new Map();
-    this.watcher = null;
+import { spawn as defaultSpawn } from "node:child_process";
+import { getConfig, getPipelineConfig } from "./config.js";
+
+/**
+ * Resolve canonical pipeline directories for the given data root.
+ * @param {string} dataDir
+ */
+function resolveDirs(dataDir) {
+  // Normalize incoming dataDir: callers may pass either the project root,
+  // the pipeline-data root, or even pipeline-data/pending by mistake.
+  // Detect if 'pipeline-data' is present in the provided path and normalize
+  // to the canonical pipeline-data root to avoid duplicated segments.
+  const normalized = path.normalize(String(dataDir || ""));
+  const parts = normalized.split(path.sep).filter(Boolean);
+  const idx = parts.lastIndexOf("pipeline-data");
+  let root;
+  if (idx !== -1) {
+    // Preserve original root (drive letter on Windows, '/' on POSIX, or '' for relative)
+    const originalRoot = path.parse(normalized).root; // '' | '/' | 'C:\\'
+    if (originalRoot) {
+      // Prepend original root to preserve absolute / drive-letter semantics
+      root = path.join(originalRoot, ...parts.slice(0, idx + 1));
+    } else {
+      // Relative input -> keep relative result
+      root = path.join(...parts.slice(0, idx + 1));
+    }
+  } else {
+    root = path.join(dataDir, "pipeline-data");
   }
 
-  async start() {
-    await fs.mkdir(this.paths.pending, { recursive: true });
-    await fs.mkdir(this.paths.current, { recursive: true });
-    await fs.mkdir(this.paths.complete, { recursive: true });
+  const pending = path.join(root, "pending");
+  const current = path.join(root, "current");
+  const complete = path.join(root, "complete");
+  return { dataDir: root, pending, current, complete };
+}
 
-    for (const name of await this.#listDirs(this.paths.current)) {
-      this.#ensureRunner(name);
-    }
+/**
+ * Ensure directory exists (mkdir -p).
+ */
+async function ensureDir(dir) {
+  await fs.mkdir(dir, { recursive: true });
+}
 
-    const config = getConfig();
-    this.watcher = chokidar
-      .watch(path.join(this.paths.pending, "*-seed.json"), {
-        awaitWriteFinish: {
-          stabilityThreshold: config.orchestrator.watchStabilityThreshold,
-          pollInterval: config.orchestrator.watchPollInterval,
-        },
-      })
-      .on("add", (p) => this.#onSeed(p));
-
-    return this;
+/**
+ * Move a file atomically by writing through a tmp file, then rename.
+ * If src is on same FS, a regular rename is enough. We keep it simple for tests.
+ */
+async function moveFile(src, dest) {
+  await fs.mkdir(path.dirname(dest), { recursive: true });
+  await fs.rename(src, dest);
+}
+
+/**
+ * Start the orchestrator.
+ * - Ensures pipeline dirs
+ * - Watches pending/*.json seeds
+ * - On add: move to current/{jobId}/seed.json and spawn runner
+ *
+ * @param {{ dataDir: string, spawn?: typeof defaultSpawn, watcherFactory?: Function, testMode?: boolean }} opts
+ * @returns {Promise<{ stop: () => Promise<void> }>}
+ */
+export async function startOrchestrator(opts) {
+  const dataDir = opts?.dataDir;
+  if (!dataDir) throw new Error("startOrchestrator: dataDir is required");
+  const spawn = opts?.spawn ?? defaultSpawn;
+  const watcherFactory = opts?.watcherFactory ?? chokidar.watch;
+  const testMode = !!opts?.testMode;
+
+  const dirs = resolveDirs(dataDir);
+  await ensureDir(dirs.pending);
+  await ensureDir(dirs.current);
+  await ensureDir(dirs.complete);
+
+  /** @type {Map<string, import('node:child_process').ChildProcess>} */
+  const running = new Map();
+
+  // Guard: if job already running or already in current/, do nothing
+  function isJobActive(name) {
+    return running.has(name);
   }
 
-  async stop() {
-    if (this.watcher) {
-      await this.watcher.close();
-      this.watcher = null;
-    }
+  function currentSeedPath(name) {
+    return path.join(dirs.current, name, "seed.json");
+  }
 
-    for (const [name, info] of this.runningProcesses) {
-      info.process.kill("SIGTERM");
-    }
+  async function handleSeedAdd(filePath) {
+    if (!filePath || !filePath.endsWith(".json")) return;
 
-    // Skip the shutdown timeout in test environment
-    if (process.env.NODE_ENV !== "test") {
-      const config = getConfig();
-      await new Promise((r) =>
-        setTimeout(r, config.orchestrator.shutdownTimeout)
-      );
+    // Extract jobId from filename pattern: ^([A-Za-z0-9-_]+)-seed\.json$
+    const base = path.basename(filePath);
+    const match = base.match(/^([A-Za-z0-9-_]+)-seed\.json$/);
+    if (!match) {
+      console.warn("Rejecting non-id seed file:", base);
+      return;
     }
+    const jobId = match[1];
 
-    for (const [name, info] of this.runningProcesses) {
-      if (!info.process.killed) info.process.kill("SIGKILL");
+    let seed;
+    try {
+      const text = await fs.readFile(filePath, "utf8");
+      seed = JSON.parse(text);
+    } catch {
+      // If not valid JSON, ignore and leave file for later/manual cleanup
+      return;
     }
 
-    this.runningProcesses.clear();
-  }
-
-  async #onSeed(seedPath) {
-    const base = path.basename(seedPath);
-    const name = base.replace(/-seed\.json$/, "");
-    const workDir = path.join(this.paths.current, name);
-    const lockFile = path.join(this.paths.current, `${name}.lock`);
-
+    // If already running or already moved to current, skip (idempotent)
+    if (isJobActive(jobId)) return;
+    const dest = currentSeedPath(jobId);
     try {
-      await fs.writeFile(lockFile, process.pid.toString(), { flag: "wx" });
-    } catch (err) {
-      if (err.code === "EEXIST") return;
-      throw err;
-    }
+      await fs.access(dest);
+      // Already picked up
+      return;
+    } catch {}
 
+    // Move seed to current/{jobId}/seed.json
+    console.log(`[Orchestrator] Moving file from ${filePath} to ${dest}`);
     try {
-      try {
-        await fs.mkdir(workDir, { recursive: false });
-      } catch (err) {
-        if (err.code === "EEXIST") return;
-        throw err;
-      }
+      await moveFile(filePath, dest);
+      console.log(`[Orchestrator] ✓ Successfully moved file to ${dest}`);
+    } catch (error) {
+      console.log(`[Orchestrator] ✗ Failed to move file: ${error.message}`);
+      throw error; // Re-throw to see the actual error
+    }
 
-      const seed = JSON.parse(await fs.readFile(seedPath, "utf8"));
-
-      // Validate seed file structure
-      const validation = validateSeed(seed);
-      if (!validation.valid) {
-        const errorMsg = formatValidationErrors(validation.errors);
-        console.error(`Invalid seed file ${base}:\n${errorMsg}`);
-        // Move invalid seed to a rejected directory for inspection
-        const rejectedDir = path.join(
-          path.dirname(this.paths.pending),
-          "rejected"
-        );
-        await fs.mkdir(rejectedDir, { recursive: true });
-        const rejectedPath = path.join(rejectedDir, base);
-        await fs.rename(seedPath, rejectedPath);
-        return;
-      }
+    // Ensure tasks directory and status file exist in work dir
+    const workDir = path.dirname(dest);
+    const tasksDir = path.join(workDir, "tasks");
+    await fs.mkdir(tasksDir, { recursive: true });
 
-      const pipelineId = this.#makeId();
-
-      await this.#atomicWrite(
-        path.join(workDir, "seed.json"),
-        JSON.stringify(seed, null, 2)
-      );
-      await this.#atomicWrite(
-        path.join(workDir, "tasks-status.json"),
-        JSON.stringify(
-          {
-            pipelineId,
-            name,
-            current: null,
-            createdAt: new Date().toISOString(),
-            tasks: {},
-          },
-          null,
-          2
-        )
-      );
-
-      await fs.mkdir(path.join(workDir, "tasks"), { recursive: true });
-
-      // Move the seed file to a 'processed' directory after successful processing
-      const processedDir = path.join(
-        path.dirname(this.paths.pending),
-        "processed"
-      );
-      await fs.mkdir(processedDir, { recursive: true });
-      const processedPath = path.join(processedDir, base);
-      await fs.rename(seedPath, processedPath);
-    } finally {
-      try {
-        await fs.unlink(lockFile);
-      } catch {}
+    const statusPath = path.join(workDir, "tasks-status.json");
+    try {
+      await fs.access(statusPath);
+    } catch {
+      const status = {
+        id: jobId,
+        name: seed?.name ?? jobId,
+        pipeline: seed?.pipeline, // Include pipeline slug from seed
+        createdAt: new Date().toISOString(),
+        state: "pending",
+        tasks: {}, // Initialize empty tasks object for pipeline runner
+      };
+      await fs.writeFile(statusPath, JSON.stringify(status, null, 2));
     }
-
-    // Start runner after all file operations are complete
-    this.#ensureRunner(name);
+    // Spawn runner for this job
+    const child = spawnRunner(jobId, dirs, running, spawn, testMode, seed);
+    // child registered inside spawnRunner
+    return child;
   }
 
-  #ensureRunner(name) {
-    if (this.runningProcesses.has(name)) return;
-
-    const config = getConfig();
-
-    // Wrap process spawn in retry logic (fire-and-forget)
-    // This is intentionally not awaited - we want to start runners asynchronously
-    // and let them run in the background. Failures are handled via dead letter queue.
-    withRetry(() => this.#spawnRunner(name), {
-      maxAttempts: config.orchestrator.processSpawnRetries,
-      initialDelay: config.orchestrator.processSpawnRetryDelay,
-      onRetry: ({ attempt, delay, error }) => {
-        console.warn(
-          `Failed to start pipeline ${name} (attempt ${attempt}): ${error.message}. Retrying in ${delay}ms...`
-        );
-      },
-      shouldRetry: (error) => {
-        // Don't retry if the error is due to missing files or invalid config
-        const nonRetryableCodes = ["ENOENT", "EACCES", "MODULE_NOT_FOUND"];
-        const nonRetryableMessages = ["Invalid pipeline"];
-        if (error.code && nonRetryableCodes.includes(error.code)) {
-          return false;
-        }
-        if (error.message && nonRetryableMessages.includes(error.message)) {
-          return false;
-        }
-        return true;
-      },
-    }).catch((error) => {
-      console.error(
-        `Failed to start pipeline ${name} after ${config.orchestrator.processSpawnRetries} attempts:`,
-        error
-      );
-      // Move to dead letter queue
-      this.#moveToDeadLetter(name, error).catch((dlqError) => {
-        console.error(`Failed to move ${name} to dead letter queue:`, dlqError);
-      });
+  // Watch pending directory for seeds
+  const watchPattern = path.join(dirs.pending, "*.json");
+  console.log("Orchestrator watching pattern:", watchPattern);
+  const watcher = watcherFactory(watchPattern, {
+    ignoreInitial: false,
+    awaitWriteFinish: false, // Disable awaitWriteFinish for faster detection
+    depth: 0,
+  });
+
+  // Wait for watcher to be ready before resolving
+  await new Promise((resolve, reject) => {
+    watcher.on("ready", () => {
+      console.log("Orchestrator watcher is ready");
+      resolve();
     });
-  }
 
-  #spawnRunner(name) {
-    return new Promise((resolve, reject) => {
-      const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
-      const runnerPath = path.join(__dirname, "pipeline-runner.js");
-
-      const env = {
-        ...process.env,
-        PO_ROOT: process.cwd(),
-        PO_DATA_DIR: path.relative(
-          process.cwd(),
-          path.dirname(this.paths.pending)
-        ),
-        PO_CURRENT_DIR: this.paths.current,
-        PO_COMPLETE_DIR: this.paths.complete,
-        PO_CONFIG_DIR: path.join(process.cwd(), "pipeline-config"),
-        PO_PIPELINE_PATH:
-          this.pipelineDefinition?.__path ||
-          path.join(process.cwd(), "pipeline-config", "pipeline.json"),
-        PO_TASK_REGISTRY: path.join(
-          process.cwd(),
-          "pipeline-config",
-          "tasks/index.js"
-        ),
-      };
-
-      const child = spawn(process.execPath, [runnerPath, name], {
-        stdio: ["ignore", "inherit", "inherit"],
-        env,
-        cwd: process.cwd(),
-      });
-
-      // Track if process started successfully
-      let started = false;
-
-      // Consider spawn successful after a short delay
-      const startupTimeout = setTimeout(() => {
-        started = true;
-        resolve();
-      }, 100);
-
-      this.runningProcesses.set(name, {
-        process: child,
-        startedAt: new Date().toISOString(),
-        name,
-      });
-
-      child.on("exit", (code, signal) => {
-        clearTimeout(startupTimeout);
-        this.runningProcesses.delete(name);
-        if (code !== 0) {
-          console.error(
-            `Pipeline ${name} exited with code ${code}, signal ${signal}`
-          );
-        } else {
-          console.log(`Pipeline ${name} completed successfully`);
-        }
-      });
-
-      child.on("error", (err) => {
-        clearTimeout(startupTimeout);
-        this.runningProcesses.delete(name);
-        if (!started) {
-          reject(err);
-        } else {
-          console.error(`Pipeline ${name} encountered error:`, err);
-        }
-      });
+    watcher.on("error", (error) => {
+      console.log("Orchestrator watcher error:", error);
+      reject(error);
     });
-  }
+  });
 
-  async #moveToDeadLetter(name, error) {
-    const workDir = path.join(this.paths.current, name);
-    const deadLetterDir = path.join(
-      path.dirname(this.paths.pending),
-      "dead-letter"
-    );
-    await fs.mkdir(deadLetterDir, { recursive: true });
-
-    const errorLog = {
-      name,
-      error: {
-        message: error.message,
-        stack: error.stack,
-      },
-      timestamp: new Date().toISOString(),
-      attempts: getConfig().orchestrator.processSpawnRetries,
-    };
-
-    await this.#atomicWrite(
-      path.join(deadLetterDir, `${name}-error.json`),
-      JSON.stringify(errorLog, null, 2)
-    );
+  watcher.on("add", (file) => {
+    console.log("Orchestrator detected file add:", file);
+    // Return the promise so tests awaiting the add handler block until processing completes
+    return handleSeedAdd(file);
+  });
 
-    // Move the work directory to dead letter
-    const deadLetterWorkDir = path.join(deadLetterDir, name);
+  async function stop() {
     try {
-      await fs.rename(workDir, deadLetterWorkDir);
-    } catch (err) {
-      // If rename fails, try to copy
-      console.warn(`Could not move ${name} to dead letter, attempting copy`);
-      // If rename fails, try to copy
-      console.warn(`Could not move ${name} to dead letter, attempting copy`);
+      await watcher.close();
+    } catch {}
+
+    // Try graceful shutdown for children
+    const kills = [];
+    for (const [name, child] of running.entries()) {
       try {
-        await this.#copyDirRecursive(workDir, deadLetterWorkDir);
-        await fs.rm(workDir, { recursive: true, force: true });
-      } catch (copyErr) {
-        console.error(`Failed to copy ${name} to dead letter:`, copyErr);
-      }
+        if (!child.killed) {
+          child.kill("SIGTERM");
+          // Give tests a chance to simulate exit; then force kill
+          setTimeout(() => {
+            try {
+              !child.killed && child.kill("SIGKILL");
+            } catch {}
+          }, 500);
+        }
+      } catch {}
+      kills.push(Promise.resolve());
     }
+    await Promise.all(kills);
+    running.clear();
   }
 
-  async #copyDirRecursive(src, dest) {
-    await fs.mkdir(dest, { recursive: true });
-    const entries = await fs.readdir(src, { withFileTypes: true });
-    for (const entry of entries) {
-      const srcPath = path.join(src, entry.name);
-      const destPath = path.join(dest, entry.name);
-      if (entry.isDirectory()) {
-        await this.#copyDirRecursive(srcPath, destPath);
-      } else if (entry.isFile()) {
-        await fs.copyFile(srcPath, destPath);
+  return { stop };
+}
+
+/**
+ * Spawn a pipeline runner. In testMode we still call spawn() so tests can assert,
+ * but we resolve immediately and let tests drive the lifecycle (emit 'exit', etc.).
+ *
+ * @param {string} jobId
+ * @param {{dataDir:string,pending:string,current:string,complete:string}} dirs
+ * @param {Map<string, import('node:child_process').ChildProcess>} running
+ * @param {typeof defaultSpawn} spawn
+ * @param {boolean} testMode
+ * @param {Object} seed - Seed data containing pipeline information
+ */
+function spawnRunner(jobId, dirs, running, spawn, testMode, seed) {
+  const runnerPath = path.join(
+    process.cwd(),
+    "src",
+    "core",
+    "pipeline-runner.js"
+  );
+
+  const configSnapshot = getConfig();
+  const availablePipelines = Object.keys(configSnapshot?.pipelines ?? {});
+  const pipelineSlug = seed?.pipeline;
+
+  console.log("[Orchestrator] spawnRunner invoked", {
+    jobId,
+    pipelineSlug: pipelineSlug ?? null,
+    availablePipelines,
+    seedKeys: seed ? Object.keys(seed) : null,
+  });
+
+  if (!availablePipelines.length) {
+    console.warn(
+      "[Orchestrator] No pipelines registered in config() when spawnRunner invoked"
+    );
+  } else if (!availablePipelines.includes(pipelineSlug)) {
+    console.warn(
+      "[Orchestrator] Requested pipeline slug missing from registry snapshot",
+      {
+        jobId,
+        pipelineSlug,
+        availablePipelines,
       }
-    }
-  }
-  async #listDirs(dir) {
-    try {
-      const entries = await fs.readdir(dir, { withFileTypes: true });
-      return entries.filter((e) => e.isDirectory()).map((e) => e.name);
-    } catch (err) {
-      if (err.code === "ENOENT") return [];
-      throw err;
-    }
+    );
   }
 
-  #makeId() {
-    return (
-      "pl-" +
-      new Date().toISOString().replaceAll(/[:.]/g, "-") +
-      "-" +
-      crypto.randomBytes(3).toString("hex")
+  if (!pipelineSlug) {
+    console.error("[Orchestrator] Missing pipeline slug in seed", {
+      jobId,
+      seed,
+      availablePipelines,
+    });
+    throw new Error(
+      "Pipeline slug is required in seed data. Include a 'pipeline' field in your seed."
     );
   }
 
-  async #atomicWrite(file, data) {
-    const tmp = file + ".tmp";
-    await fs.writeFile(tmp, data);
-    await fs.rename(tmp, file);
+  let pipelineConfig;
+  try {
+    pipelineConfig = getPipelineConfig(pipelineSlug);
+  } catch (error) {
+    console.error("[Orchestrator] Pipeline lookup failed", {
+      jobId,
+      pipelineSlug,
+      availablePipelines,
+    });
+    throw error;
+  }
+
+  // Use environment variables with explicit slug propagation
+  const env = {
+    ...process.env,
+    PO_DATA_DIR: dirs.dataDir,
+    PO_PENDING_DIR: dirs.pending,
+    PO_CURRENT_DIR: dirs.current,
+    PO_COMPLETE_DIR: dirs.complete,
+    PO_PIPELINE_SLUG: pipelineSlug,
+    // Force mock provider for testing
+    PO_DEFAULT_PROVIDER: "mock",
+  };
+
+  // Always call spawn so tests can capture it
+  const child = spawn(process.execPath, [runnerPath, jobId], {
+    stdio: ["ignore", "inherit", "inherit"],
+    env,
+    cwd: process.cwd(),
+  });
+
+  running.set(jobId, child);
+
+  child.on("exit", () => {
+    running.delete(jobId);
+  });
+  child.on("error", () => {
+    running.delete(jobId);
+  });
+
+  // In test mode: return immediately; in real mode you might await readiness
+  if (testMode) {
+    return child;
   }
+
+  // Non-test: we can consider "started" immediately for simplicity
+  return child;
 }
+
+export default { startOrchestrator };