@ryanfw/prompt-orchestration-pipeline 0.0.1 → 0.4.0

package/src/core/config.js

@@ -5,9 +5,163 @@
  * supporting both environment variables and config file overrides.
  */
 
-import { promises as fs } from "node:fs";
+import { promises as fs, existsSync, readFileSync } from "node:fs";
 import path from "node:path";
 
+async function checkFileExistence(filePath) {
+  try {
+    await fs.access(filePath);
+    return true;
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      return false;
+    } else {
+      throw error; // Re-throw other errors
+    }
+  }
+}
+
+function resolveRepoRoot(config) {
+  const configuredRoot = config?.paths?.root;
+  if (!configuredRoot) {
+    throw new Error("PO_ROOT is required");
+  }
+  return path.resolve(configuredRoot);
+}
+
+function resolveWithBase(rootDir, maybePath) {
+  if (!maybePath) {
+    return undefined;
+  }
+  return path.isAbsolute(maybePath)
+    ? maybePath
+    : path.resolve(rootDir, maybePath);
+}
+
+function normalizeRegistryEntry(slug, entry, rootDir) {
+  const pipelineJsonPath = entry?.pipelineJsonPath
+    ? resolveWithBase(rootDir, entry.pipelineJsonPath)
+    : undefined;
+
+  const configDir = entry?.configDir
+    ? resolveWithBase(rootDir, entry.configDir)
+    : pipelineJsonPath
+      ? path.dirname(pipelineJsonPath)
+      : path.join(rootDir, "pipeline-config", slug);
+
+  const tasksDir = entry?.tasksDir
+    ? resolveWithBase(rootDir, entry.tasksDir)
+    : path.join(configDir, "tasks");
+
+  return {
+    configDir,
+    tasksDir,
+    name: entry?.name,
+    description: entry?.description,
+  };
+}
+
+async function hydratePipelinesFromRegistry(config) {
+  const rootDir = resolveRepoRoot(config);
+  const registryPath = path.join(rootDir, "pipeline-config", "registry.json");
+
+  let registryData;
+  try {
+    const contents = await fs.readFile(registryPath, "utf8");
+    registryData = JSON.parse(contents);
+  } catch (error) {
+    if (error?.code === "ENOENT") {
+      return;
+    }
+    throw new Error(
+      "Failed to read pipeline registry at " +
+        registryPath +
+        ": " +
+        error.message
+    );
+  }
+
+  if (
+    !registryData ||
+    typeof registryData !== "object" ||
+    !registryData.pipelines ||
+    typeof registryData.pipelines !== "object"
+  ) {
+    if (
+      registryData &&
+      typeof registryData === "object" &&
+      registryData.slugs &&
+      typeof registryData.slugs === "object"
+    ) {
+      console.warn(
+        "[config] Detected legacy pipeline registry format using `slugs`. Expected `pipelines` object. Falling back to defaultConfig.pipelines."
+      );
+    }
+    return;
+  }
+
+  const resolved = {};
+  for (const [slug, entry] of Object.entries(registryData.pipelines)) {
+    const normalized = normalizeRegistryEntry(slug, entry, rootDir);
+    resolved[slug] = normalized;
+  }
+
+  if (Object.keys(resolved).length > 0) {
+    config.pipelines = resolved;
+  }
+}
+
+function hydratePipelinesFromRegistrySync(config) {
+  const rootDir = resolveRepoRoot(config);
+  const registryPath = path.join(rootDir, "pipeline-config", "registry.json");
+
+  if (!existsSync(registryPath)) {
+    return;
+  }
+
+  let registryData;
+  try {
+    const contents = readFileSync(registryPath, "utf8");
+    registryData = JSON.parse(contents);
+  } catch (error) {
+    throw new Error(
+      "Failed to read pipeline registry at " +
+        registryPath +
+        ": " +
+        error.message
+    );
+  }
+
+  if (
+    !registryData ||
+    typeof registryData !== "object" ||
+    !registryData.pipelines ||
+    typeof registryData.pipelines !== "object"
+  ) {
+    if (
+      registryData &&
+      typeof registryData === "object" &&
+      registryData.slugs &&
+      typeof registryData.slugs === "object"
+    ) {
+      console.warn(
+        "[config] Detected legacy pipeline registry format using `slugs`. Expected `pipelines` object. Falling back to defaultConfig.pipelines."
+      );
+    }
+    return;
+  }
+
+  const resolved = {};
+  for (const [slug, entry] of Object.entries(registryData.pipelines)) {
+    const normalized = normalizeRegistryEntry(slug, entry, rootDir);
+    resolved[slug] = normalized;
+  }
+
+  if (Object.keys(resolved).length > 0) {
+    config.pipelines = resolved;
+  }
+}
+
 /**
  * Default configuration values
  * These can be overridden by environment variables or config file
@@ -33,6 +187,36 @@ export const defaultConfig = {
     maxConcurrency: 5,
     retryMaxAttempts: 3,
     retryBackoffMs: 1000,
+    models: {
+      "openai:gpt-4": {
+        provider: "openai",
+        model: "gpt-4",
+      },
+      "openai:gpt-4-turbo": {
+        provider: "openai",
+        model: "gpt-4-turbo",
+      },
+      "openai:gpt-5": {
+        provider: "openai",
+        model: "gpt-5-chat-latest",
+      },
+      "deepseek:reasoner": {
+        provider: "deepseek",
+        model: "deepseek-reasoner",
+      },
+      "deepseek:chat": {
+        provider: "deepseek",
+        model: "deepseek-chat",
+      },
+      "anthropic:opus": {
+        provider: "anthropic",
+        model: "claude-3-opus",
+      },
+      "anthropic:sonnet": {
+        provider: "anthropic",
+        model: "claude-3-sonnet",
+      },
+    },
   },
   ui: {
     port: 3000,
@@ -41,13 +225,13 @@ export const defaultConfig = {
     maxRecentChanges: 10,
   },
   paths: {
-    root: process.env.PO_ROOT || process.cwd(),
+    root: undefined,
     dataDir: "pipeline-data",
-    configDir: "pipeline-config",
     pendingDir: "pending",
     currentDir: "current",
     completeDir: "complete",
   },
+  pipelines: {},
   validation: {
     seedNameMinLength: 1,
     seedNameMaxLength: 100,
@@ -154,9 +338,7 @@ function loadFromEnvironment(config) {
   if (process.env.PO_DATA_DIR) {
     envConfig.paths.dataDir = process.env.PO_DATA_DIR;
   }
-  if (process.env.PO_CONFIG_DIR) {
-    envConfig.paths.configDir = process.env.PO_CONFIG_DIR;
-  }
+  // Note: PO_CONFIG_DIR is deprecated - use pipelines.registry instead
 
   // Logging settings
   if (process.env.PO_LOG_LEVEL) {
@@ -215,7 +397,7 @@ async function loadFromFile(configPath) {
  * Validate configuration values
  * Throws if configuration is invalid
  */
-function validateConfig(config) {
+async function validateConfig(config) {
   const errors = [];
 
   // Validate numeric values are positive
@@ -236,7 +418,7 @@ function validateConfig(config) {
   }
 
   // Validate provider
-  const validProviders = ["openai", "deepseek", "anthropic"];
+  const validProviders = ["openai", "deepseek", "anthropic", "mock"];
   if (!validProviders.includes(config.llm.defaultProvider)) {
     errors.push(
       `llm.defaultProvider must be one of: ${validProviders.join(", ")}`
@@ -286,9 +468,49 @@ export async function loadConfig(options = {}) {
   // Override with environment variables
   config = loadFromEnvironment(config);
 
+  // Validate that PO_ROOT is set
+  if (!config.paths.root) {
+    throw new Error("PO_ROOT is required");
+  }
+
+  // Hydrate pipeline registry if present
+  await hydratePipelinesFromRegistry(config);
+
+  // Validate pipelines presence after hydration
+  if (!config.pipelines || Object.keys(config.pipelines).length === 0) {
+    const repoRoot = resolveRepoRoot(config);
+    throw new Error(
+      `No pipelines are registered. Create pipeline-config/registry.json in ${repoRoot} to register pipelines.`
+    );
+  }
+
+  // Normalize pipeline paths and validate existence
+  const repoRoot = resolveRepoRoot(config);
+  for (const slug in config.pipelines) {
+    const pipeline = config.pipelines[slug];
+
+    // Resolve to absolute paths
+    pipeline.configDir = path.resolve(repoRoot, pipeline.configDir);
+    pipeline.tasksDir = path.resolve(repoRoot, pipeline.tasksDir);
+
+    // Validate directory existence
+    if (!(await checkFileExistence(pipeline.configDir))) {
+      throw new Error(pipeline.configDir + " does not exist");
+    }
+    if (!(await checkFileExistence(pipeline.tasksDir))) {
+      throw new Error(pipeline.tasksDir + " does not exist");
+    }
+
+    // Validate pipeline.json exists
+    const pipelineJsonPath = path.join(pipeline.configDir, "pipeline.json");
+    if (!(await checkFileExistence(pipelineJsonPath))) {
+      throw new Error(pipelineJsonPath + " does not exist");
+    }
+  }
+
   // Validate if requested
   if (validate) {
-    validateConfig(config);
+    await validateConfig(config);
   }
 
   // Cache the loaded config
@@ -309,6 +531,24 @@ export function getConfig() {
     currentConfig = loadFromEnvironment(
       JSON.parse(JSON.stringify(defaultConfig))
     );
+
+    // Validate that PO_ROOT is set
+    if (!currentConfig.paths.root) {
+      throw new Error("PO_ROOT is required");
+    }
+
+    hydratePipelinesFromRegistrySync(currentConfig);
+
+    // Validate pipelines presence after hydration
+    if (
+      !currentConfig.pipelines ||
+      Object.keys(currentConfig.pipelines).length === 0
+    ) {
+      const repoRoot = resolveRepoRoot(currentConfig);
+      throw new Error(
+        `No pipelines are registered. Create pipeline-config/registry.json in ${repoRoot} to register pipelines.`
+      );
+    }
   }
   return currentConfig;
 }
@@ -343,3 +583,24 @@ export function getConfigValue(path, defaultValue = undefined) {
 
   return value;
 }
+
+/**
+ * Get pipeline configuration by slug
+ *
+ * @param {string} slug - Pipeline slug identifier
+ * @returns {Object} Object with pipelineJsonPath and tasksDir
+ */
+export function getPipelineConfig(slug) {
+  const config = getConfig();
+
+  if (!config.pipelines || !config.pipelines[slug]) {
+    throw new Error("Pipeline " + slug + " not found in registry");
+  }
+
+  const pipeline = config.pipelines[slug];
+
+  return {
+    pipelineJsonPath: path.join(pipeline.configDir, "pipeline.json"),
+    tasksDir: pipeline.tasksDir,
+  };
+}

package/src/core/file-io.js

@@ -0,0 +1,202 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Creates a task-scoped file I/O interface that manages file operations
+ * and automatically updates tasks-status.json with file tracking.
+ *
+ * @param {Object} config - Configuration object
+ * @param {string} config.workDir - Base working directory (e.g., /path/to/pipeline-data/current/jobId)
+ * @param {string} config.taskName - Name of the current task
+ * @param {Function} config.getStage - Function that returns current stage name
+ * @param {string} config.statusPath - Path to tasks-status.json file
+ * @returns {Object} File I/O interface with curried functions
+ */
+
+async function ensureDir(dir) {
+  await fs.mkdir(dir, { recursive: true });
+}
+
+export function createTaskFileIO({ workDir, taskName, getStage, statusPath }) {
+  const taskDir = path.join(workDir, "tasks", taskName);
+
+  // New directory structure: {workDir}/files/{type}
+  const filesRoot = path.join(workDir, "files");
+  const artifactsDir = path.join(filesRoot, "artifacts");
+  const logsDir = path.join(filesRoot, "logs");
+  const tmpDir = path.join(filesRoot, "tmp");
+
+  /**
+   * Updates tasks-status.json with file information, ensuring de-duplication
+   */
+  async function updateStatusWithFiles(fileType, fileName) {
+    try {
+      const statusContent = await fs.readFile(statusPath, "utf8");
+      const status = JSON.parse(statusContent);
+
+      // Initialize files object if it doesn't exist
+      if (!status.files) {
+        status.files = { artifacts: [], logs: [], tmp: [] };
+      }
+
+      // Initialize task files if they don't exist
+      if (!status.tasks[taskName].files) {
+        status.tasks[taskName].files = { artifacts: [], logs: [], tmp: [] };
+      }
+
+      // Add to job-level files array (de-duped)
+      const jobArray = status.files[fileType];
+      if (!jobArray.includes(fileName)) {
+        jobArray.push(fileName);
+      }
+
+      // Add to task-level files array (de-duped)
+      const taskArray = status.tasks[taskName].files[fileType];
+      if (!taskArray.includes(fileName)) {
+        taskArray.push(fileName);
+      }
+
+      // Write back to file atomically
+      await atomicWrite(statusPath, JSON.stringify(status, null, 2));
+    } catch (error) {
+      // If status file doesn't exist or is invalid, we'll log but not fail
+      console.warn(
+        `Failed to update status with file ${fileName}:`,
+        error.message
+      );
+    }
+  }
+
+  /**
+   * Atomic write helper
+   */
+  async function atomicWrite(filePath, data) {
+    const tmpPath = filePath + ".tmp";
+    await fs.writeFile(tmpPath, data);
+    await fs.rename(tmpPath, filePath);
+  }
+
+  /**
+   * Generic write function that handles different modes
+   */
+  async function writeFile(dirPath, fileName, content, mode = "replace") {
+    await ensureDir(dirPath);
+    const filePath = path.join(dirPath, fileName);
+
+    if (mode === "append") {
+      await fs.appendFile(filePath, content);
+    } else {
+      await atomicWrite(filePath, content);
+    }
+
+    return filePath;
+  }
+
+  /**
+   * Generic read function
+   */
+  async function readFile(dirPath, fileName) {
+    const filePath = path.join(dirPath, fileName);
+    return await fs.readFile(filePath, "utf8");
+  }
+
+  // Return curried functions for each file type
+  return {
+    /**
+     * Write an artifact file
+     * @param {string} name - File name
+     * @param {string} content - File content
+     * @param {Object} options - Options object
+     * @param {string} options.mode - "replace" (default) or "append"
+     */
+    async writeArtifact(name, content, options = {}) {
+      const filePath = await writeFile(
+        artifactsDir,
+        name,
+        content,
+        options.mode || "replace"
+      );
+      await updateStatusWithFiles("artifacts", name);
+      return filePath;
+    },
+
+    /**
+     * Write a log file
+     * @param {string} name - File name
+     * @param {string} content - Log content
+     * @param {Object} options - Options object
+     * @param {string} options.mode - "append" (default) or "replace"
+     */
+    async writeLog(name, content, options = {}) {
+      const filePath = await writeFile(
+        logsDir,
+        name,
+        content,
+        options.mode || "append"
+      );
+      await updateStatusWithFiles("logs", name);
+      return filePath;
+    },
+
+    /**
+     * Write a temporary file
+     * @param {string} name - File name
+     * @param {string} content - File content
+     * @param {Object} options - Options object
+     * @param {string} options.mode - "replace" (default) or "append"
+     */
+    async writeTmp(name, content, options = {}) {
+      const filePath = await writeFile(
+        tmpDir,
+        name,
+        content,
+        options.mode || "replace"
+      );
+      await updateStatusWithFiles("tmp", name);
+      return filePath;
+    },
+
+    /**
+     * Read an artifact file
+     * @param {string} name - File name
+     * @returns {string} File content
+     */
+    async readArtifact(name) {
+      return await readFile(artifactsDir, name);
+    },
+
+    /**
+     * Read a log file
+     * @param {string} name - File name
+     * @returns {string} File content
+     */
+    async readLog(name) {
+      return await readFile(logsDir, name);
+    },
+
+    /**
+     * Read a temporary file
+     * @param {string} name - File name
+     * @returns {string} File content
+     */
+    async readTmp(name) {
+      return await readFile(tmpDir, name);
+    },
+
+    /**
+     * Get the task directory path
+     * @returns {string} Task directory path
+     */
+    getTaskDir() {
+      return taskDir;
+    },
+
+    /**
+     * Get the current stage name
+     * @returns {string} Current stage name
+     */
+    getCurrentStage() {
+      return getStage();
+    },
+  };
+}

package/src/core/module-loader.js

@@ -0,0 +1,157 @@
+import fsp from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+const WORKSPACE_CACHE_DIR = path.join(process.cwd(), ".tmp-task-modules");
+
+/**
+ * Convert supported modulePath formats into a file:// URL.
+ * @param {string | URL} modulePath
+ * @returns {URL}
+ */
+function resolveToFileURL(modulePath) {
+  if (modulePath instanceof URL) {
+    return modulePath;
+  }
+
+  if (
+    modulePath &&
+    typeof modulePath === "object" &&
+    typeof modulePath.href === "string"
+  ) {
+    try {
+      return new URL(modulePath.href);
+    } catch {
+      // fall through to error below
+    }
+  }
+
+  if (typeof modulePath !== "string") {
+    throw new TypeError(
+      `Module path must be a string or URL. Received: ${typeof modulePath}`
+    );
+  }
+
+  if (modulePath.startsWith("file://")) {
+    return new URL(modulePath);
+  }
+
+  if (!path.isAbsolute(modulePath)) {
+    throw new Error(
+      `Module path must be absolute. Received: ${modulePath}\n` +
+        `Hint: resolve module paths before calling loadFreshModule().`
+    );
+  }
+
+  return pathToFileURL(modulePath);
+}
+
+/**
+ * Detect whether an error corresponds to a module-not-found condition.
+ * @param {unknown} error
+ * @returns {boolean}
+ */
+function isModuleNotFoundError(error) {
+  if (!error || typeof error !== "object") {
+    return false;
+  }
+
+  const err = /** @type {{ code?: string; message?: string }} */ (error);
+
+  if (err.code === "ERR_MODULE_NOT_FOUND") {
+    return true;
+  }
+
+  const message = err.message || "";
+  return (
+    message.includes("Cannot find module") ||
+    message.includes("Failed to load url")
+  );
+}
+
+/**
+ * Produce a clearer error when the underlying module file is missing.
+ * @param {string} modulePath
+ * @param {Error} originalError
+ */
+function createMissingModuleError(modulePath, originalError) {
+  const error = new Error(
+    `Module not found at "${modulePath}". Ensure the file exists before running the pipeline.`
+  );
+  error.name = originalError.name || "ERR_MODULE_NOT_FOUND";
+  if ("cause" in Error.prototype) {
+    error.cause = originalError;
+  } else {
+    error.originalError = originalError;
+  }
+  return error;
+}
+
+/**
+ * Copy a module file into a workspace-local cache directory so Vite/Vitest can load it.
+ * @param {string} sourcePath
+ * @returns {Promise<string>}
+ */
+async function copyModuleToWorkspaceCache(sourcePath) {
+  await fsp.mkdir(WORKSPACE_CACHE_DIR, { recursive: true });
+  const ext = path.extname(sourcePath) || ".js";
+  const base = path.basename(sourcePath, ext);
+  const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
+  const destFile = path.join(
+    WORKSPACE_CACHE_DIR,
+    `${base}.${uniqueSuffix}${ext}`
+  );
+  await fsp.copyFile(sourcePath, destFile);
+  return destFile;
+}
+
+/**
+ * Dynamically import a module with cache busting while remaining compatible with Node's file:/// resolution.
+ * Falls back to copying the module into a workspace-local cache when query parameters break filesystem resolution.
+ * @param {string | URL} modulePath
+ * @returns {Promise<any>} Module namespace object
+ */
+export async function loadFreshModule(modulePath) {
+  const fileUrl = resolveToFileURL(modulePath);
+  const cacheBustedUrl = `${fileUrl.href}?t=${Date.now()}`;
+
+  try {
+    return await import(cacheBustedUrl);
+  } catch (error) {
+    if (!isModuleNotFoundError(error) || fileUrl.protocol !== "file:") {
+      throw error;
+    }
+
+    const absolutePath = fileURLToPath(fileUrl);
+
+    try {
+      await fsp.access(absolutePath);
+    } catch {
+      throw createMissingModuleError(
+        absolutePath,
+        /** @type {Error} */ (error)
+      );
+    }
+
+    try {
+      const cacheCopy = await copyModuleToWorkspaceCache(absolutePath);
+      const cacheUrl = `${pathToFileURL(cacheCopy).href}?t=${Date.now()}`;
+      return await import(cacheUrl);
+    } catch (fallbackError) {
+      const messageLines = [
+        `Failed to load module "${absolutePath}" after attempting cache-busting import.`,
+        `Cache-busted URL: ${cacheBustedUrl}`,
+        `Original error: ${/** @type {Error} */ (error).message}`,
+        `Fallback error: ${/** @type {Error} */ (fallbackError).message}`,
+      ];
+      const combined = new Error(messageLines.join("\n"));
+      if ("cause" in Error.prototype) {
+        combined.cause = fallbackError;
+      } else {
+        combined.fallbackError = fallbackError;
+      }
+      combined.initialError = error;
+      throw combined;
+    }
+  }
+}