@ryanfw/prompt-orchestration-pipeline 0.0.1 → 0.3.0

package/src/cli/index.js

@@ -1,63 +1,298 @@
 #!/usr/bin/env node
 import { Command } from "commander";
+import { submitJobWithValidation } from "../api/index.js";
 import { PipelineOrchestrator } from "../api/index.js";
 import fs from "node:fs/promises";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { updatePipelineJson } from "./update-pipeline-json.js";
+
+// Canonical stage names that must match src/core/task-runner.js
+const STAGE_NAMES = [
+  "ingestion",
+  "preProcessing",
+  "promptTemplating",
+  "inference",
+  "parsing",
+  "validateStructure",
+  "validateQuality",
+  "critique",
+  "refine",
+  "finalValidation",
+  "integration",
+];
 
 const program = new Command();
 
 program
   .name("pipeline-orchestrator")
   .description("Pipeline orchestration system")
-  .version("1.0.0");
+  .version("1.0.0")
+  .option("-r, --root <path>", "Pipeline root (PO_ROOT)")
+  .option("-p, --port <port>", "UI server port", "4000");
 
 program
   .command("init")
   .description("Initialize pipeline configuration")
   .action(async () => {
-    const template = {
-      pipeline: { name: "my-pipeline", version: "1.0.0", tasks: ["example-task"] },
-      tasks: {
-        "example-task": {
-`ingestion`: `export async function ingestion(context) { return { data: "example" }; }`,
-`inference`: `export async function inference(context) { return { output: context.data }; }`
-      }
-    };
-    await fs.mkdir("pipeline-config/tasks/example-task", { recursive: true });
-    await fs.writeFile("pipeline-config/pipeline.json", JSON.stringify(template.pipeline, null, 2));
-    await fs.writeFile("pipeline-config/tasks/index.js", `export default {\n  'example-task': './example-task/index.js'\n};`);
-    await fs.writeFile("pipeline-config/tasks/example-task/index.js", `${template.tasks["example-task"].ingestion}\n\n${template.tasks["example-task"].inference}\n`);
-    console.log("Pipeline configuration initialized");
+    const globalOptions = program.opts();
+    const root = globalOptions.root || path.resolve(process.cwd(), "pipelines");
+
+    // Create directories
+    await fs.mkdir(path.join(root, "pipeline-config"), { recursive: true });
+    await fs.mkdir(path.join(root, "pipeline-data", "pending"), {
+      recursive: true,
+    });
+    await fs.mkdir(path.join(root, "pipeline-data", "current"), {
+      recursive: true,
+    });
+    await fs.mkdir(path.join(root, "pipeline-data", "complete"), {
+      recursive: true,
+    });
+    await fs.mkdir(path.join(root, "pipeline-data", "rejected"), {
+      recursive: true,
+    });
+
+    // Create .gitkeep files
+    await fs.writeFile(
+      path.join(root, "pipeline-data", "pending", ".gitkeep"),
+      ""
+    );
+    await fs.writeFile(
+      path.join(root, "pipeline-data", "current", ".gitkeep"),
+      ""
+    );
+    await fs.writeFile(
+      path.join(root, "pipeline-data", "complete", ".gitkeep"),
+      ""
+    );
+    await fs.writeFile(
+      path.join(root, "pipeline-data", "rejected", ".gitkeep"),
+      ""
+    );
+
+    // Write registry.json with exact required content
+    const registryContent = { pipelines: {} };
+    await fs.writeFile(
+      path.join(root, "pipeline-config", "registry.json"),
+      JSON.stringify(registryContent, null, 2) + "\n"
+    );
+
+    console.log(`Pipeline configuration initialized at ${root}`);
   });
 
 program
   .command("start")
-  .description("Start the pipeline orchestrator")
-  .option("-u, --ui", "Start with UI server")
-  .option("-p, --port <port>", "UI server port", "3000")
-  .action(async (options) => {
-    const orchestrator = new PipelineOrchestrator({ ui: options.ui, uiPort: parseInt(options.port) });
-    await orchestrator.initialize();
-    console.log("Pipeline orchestrator started");
-    process.on("SIGINT", async () => { await orchestrator.stop(); process.exit(0); });
+  .description("Start the pipeline orchestrator with UI server")
+  .action(async () => {
+    const globalOptions = program.opts();
+    let root = globalOptions.root || process.env.PO_ROOT;
+    const port = globalOptions.port || "4000";
+
+    // Resolve absolute root path
+    if (!root) {
+      console.error(
+        "PO_ROOT is required. Use --root or set PO_ROOT to your pipeline root (e.g., ./demo)."
+      );
+      process.exit(1);
+    }
+
+    const absoluteRoot = path.isAbsolute(root)
+      ? root
+      : path.resolve(process.cwd(), root);
+    process.env.PO_ROOT = absoluteRoot;
+
+    console.log(`Using PO_ROOT=${absoluteRoot}`);
+    console.log(`UI port=${port}`);
+
+    let uiChild = null;
+    let orchestratorChild = null;
+    let childrenExited = 0;
+    let exitCode = 0;
+
+    // Cleanup function to kill remaining children
+    const cleanup = () => {
+      if (uiChild && !uiChild.killed) {
+        uiChild.kill("SIGTERM");
+        setTimeout(() => {
+          if (!uiChild.killed) uiChild.kill("SIGKILL");
+        }, 5000);
+      }
+      if (orchestratorChild && !orchestratorChild.killed) {
+        orchestratorChild.kill("SIGTERM");
+        setTimeout(() => {
+          if (!orchestratorChild.killed) orchestratorChild.kill("SIGKILL");
+        }, 5000);
+      }
+    };
+
+    // Handle parent process signals
+    process.on("SIGINT", () => {
+      console.log("\nReceived SIGINT, shutting down...");
+      cleanup();
+      process.exit(exitCode);
+    });
+
+    process.on("SIGTERM", () => {
+      console.log("\nReceived SIGTERM, shutting down...");
+      cleanup();
+      process.exit(exitCode);
+    });
+
+    try {
+      // Step d: Build UI once if dist/ doesn't exist
+      const distPath = path.join(process.cwd(), "dist");
+      try {
+        await fs.access(distPath);
+        console.log("UI build found, skipping build step");
+      } catch {
+        console.log("Building UI...");
+        await new Promise((resolve, reject) => {
+          const vitePath = path.resolve(
+            process.cwd(),
+            "node_modules/vite/bin/vite.js"
+          );
+          const buildChild = spawn("node", [vitePath, "build"], {
+            stdio: "inherit",
+            env: { ...process.env, NODE_ENV: "development" },
+          });
+
+          buildChild.on("exit", (code) => {
+            if (code === 0) {
+              console.log("UI build completed");
+              resolve();
+            } else {
+              reject(new Error(`UI build failed with code ${code}`));
+            }
+          });
+
+          buildChild.on("error", reject);
+        });
+      }
+
+      // Step e: Spawn UI server
+      console.log("Starting UI server...");
+      const uiServerPath = path.resolve(process.cwd(), "src/ui/server.js");
+      uiChild = spawn("node", [uiServerPath], {
+        stdio: "pipe",
+        env: {
+          ...process.env,
+          NODE_ENV: "production",
+          PO_ROOT: absoluteRoot,
+          PO_UI_PORT: port,
+        },
+      });
+
+      // Pipe UI output with prefix
+      uiChild.stdout.on("data", (data) => {
+        console.log(`[ui] ${data.toString().trim()}`);
+      });
+
+      uiChild.stderr.on("data", (data) => {
+        console.error(`[ui] ${data.toString().trim()}`);
+      });
+
+      // Step f: Spawn orchestrator
+      console.log("Starting orchestrator...");
+      const orchestratorPath = path.resolve(
+        process.cwd(),
+        "src/cli/run-orchestrator.js"
+      );
+      orchestratorChild = spawn("node", [orchestratorPath], {
+        stdio: "pipe",
+        env: {
+          ...process.env,
+          NODE_ENV: "production",
+          PO_ROOT: absoluteRoot,
+        },
+      });
+
+      // Pipe orchestrator output with prefix
+      orchestratorChild.stdout.on("data", (data) => {
+        console.log(`[orc] ${data.toString().trim()}`);
+      });
+
+      orchestratorChild.stderr.on("data", (data) => {
+        console.error(`[orc] ${data.toString().trim()}`);
+      });
+
+      // Step h: Kill-others-on-fail behavior
+      const handleChildExit = (child, name) => {
+        return (code, signal) => {
+          console.log(
+            `${name} process exited with code ${code}, signal ${signal}`
+          );
+          childrenExited++;
+
+          if (code !== 0) {
+            exitCode = code;
+            console.log(`${name} failed, terminating other process...`);
+            cleanup();
+          }
+
+          if (childrenExited === 2 || (code !== 0 && childrenExited === 1)) {
+            process.exit(exitCode);
+          }
+        };
+      };
+
+      uiChild.on("exit", handleChildExit(uiChild, "UI"));
+      orchestratorChild.on(
+        "exit",
+        handleChildExit(orchestratorChild, "Orchestrator")
+      );
+
+      // Handle child process errors
+      uiChild.on("error", (error) => {
+        console.error(`UI process error: ${error.message}`);
+        exitCode = 1;
+        cleanup();
+        process.exit(1);
+      });
+
+      orchestratorChild.on("error", (error) => {
+        console.error(`Orchestrator process error: ${error.message}`);
+        exitCode = 1;
+        cleanup();
+        process.exit(1);
+      });
+    } catch (error) {
+      console.error(`Failed to start pipeline: ${error.message}`);
+      cleanup();
+      process.exit(1);
+    }
   });
 
 program
   .command("submit <seed-file>")
   .description("Submit a new job")
   .action(async (seedFile) => {
-    const seed = JSON.parse(await fs.readFile(seedFile, "utf8"));
-    const orchestrator = new PipelineOrchestrator({ autoStart: false });
-    await orchestrator.initialize();
-    const job = await orchestrator.submitJob(seed);
-    console.log(`Job submitted: ${job.name}`);
+    try {
+      const seed = JSON.parse(await fs.readFile(seedFile, "utf8"));
+      const result = await submitJobWithValidation({
+        dataDir: process.cwd(),
+        seedObject: seed,
+      });
+
+      if (result.success) {
+        console.log(`Job submitted: ${result.jobId} (${result.jobName})`);
+      } else {
+        console.error(`Failed to submit job: ${result.message}`);
+        process.exit(1);
+      }
+    } catch (error) {
+      console.error(`Error submitting job: ${error.message}`);
+      process.exit(1);
+    }
   });
 
 program
   .command("status [job-name]")
   .description("Get job status")
   .action(async (jobName) => {
-    const orchestrator = new PipelineOrchestrator({ autoStart: false });
-    await orchestrator.initialize();
+    const orchestrator = await PipelineOrchestrator.create({
+      autoStart: false,
+    });
     if (jobName) {
       const status = await orchestrator.getStatus(jobName);
       console.log(JSON.stringify(status, null, 2));
@@ -67,4 +302,196 @@ program
     }
   });
 
+program
+  .command("add-pipeline <pipeline-slug>")
+  .description("Add a new pipeline configuration")
+  .action(async (pipelineSlug) => {
+    const globalOptions = program.opts();
+    const root = globalOptions.root || path.resolve(process.cwd(), "pipelines");
+
+    // Validate pipeline-slug is kebab-case
+    const kebabCaseRegex = /^[a-z0-9-]+$/;
+    if (!kebabCaseRegex.test(pipelineSlug)) {
+      console.error("Invalid pipeline slug: must be kebab-case (a-z0-9-)");
+      process.exit(1);
+    }
+
+    try {
+      // Ensure directories exist
+      const pipelineConfigDir = path.join(
+        root,
+        "pipeline-config",
+        pipelineSlug
+      );
+      const tasksDir = path.join(pipelineConfigDir, "tasks");
+      await fs.mkdir(tasksDir, { recursive: true });
+
+      // Write pipeline.json
+      const pipelineConfig = {
+        name: pipelineSlug,
+        version: "1.0.0",
+        description: "New pipeline",
+        tasks: [],
+      };
+      await fs.writeFile(
+        path.join(pipelineConfigDir, "pipeline.json"),
+        JSON.stringify(pipelineConfig, null, 2) + "\n"
+      );
+
+      // Write tasks/index.js
+      await fs.writeFile(
+        path.join(tasksDir, "index.js"),
+        "export default {};\n"
+      );
+
+      // Update registry.json
+      const registryPath = path.join(root, "pipeline-config", "registry.json");
+      let registry = { pipelines: {} };
+
+      try {
+        const registryContent = await fs.readFile(registryPath, "utf8");
+        registry = JSON.parse(registryContent);
+        if (!registry.pipelines) {
+          registry.pipelines = {};
+        }
+      } catch (error) {
+        // If registry doesn't exist or is invalid, use empty registry
+        registry = { pipelines: {} };
+      }
+
+      // Add/replace pipeline entry
+      registry.pipelines[pipelineSlug] = {
+        name: pipelineSlug,
+        description: "New pipeline",
+        pipelinePath: `pipeline-config/${pipelineSlug}/pipeline.json`,
+        taskRegistryPath: `pipeline-config/${pipelineSlug}/tasks/index.js`,
+      };
+
+      // Write back registry
+      await fs.writeFile(
+        registryPath,
+        JSON.stringify(registry, null, 2) + "\n"
+      );
+
+      console.log(`Pipeline "${pipelineSlug}" added successfully`);
+    } catch (error) {
+      console.error(`Error adding pipeline: ${error.message}`);
+      process.exit(1);
+    }
+  });
+
+program
+  .command("add-pipeline-task <pipeline-slug> <task-slug>")
+  .description("Add a new task to a pipeline")
+  .action(async (pipelineSlug, taskSlug) => {
+    const globalOptions = program.opts();
+    const root = globalOptions.root || path.resolve(process.cwd(), "pipelines");
+
+    // Validate both slugs are kebab-case
+    const kebabCaseRegex = /^[a-z0-9-]+$/;
+    if (!kebabCaseRegex.test(pipelineSlug)) {
+      console.error("Invalid pipeline slug: must be kebab-case (a-z0-9-)");
+      process.exit(1);
+    }
+    if (!kebabCaseRegex.test(taskSlug)) {
+      console.error("Invalid task slug: must be kebab-case (a-z0-9-)");
+      process.exit(1);
+    }
+
+    // Check if pipeline tasks directory exists
+    const tasksDir = path.join(root, "pipeline-config", pipelineSlug, "tasks");
+    try {
+      await fs.access(tasksDir);
+    } catch (error) {
+      console.error(
+        `Pipeline "${pipelineSlug}" not found. Run add-pipeline first.`
+      );
+      process.exit(1);
+    }
+
+    try {
+      // Create task file with all stage exports
+      const taskFileContent = STAGE_NAMES.map((stageName) => {
+        if (stageName === "ingestion") {
+          return `// Step 1: Ingestion, ${getStagePurpose(stageName)}
+export const ingestion = async ({ io, llm, data: { seed }, meta, flags }) => {
+
+  return { output: {}, flags };
+}`;
+        }
+        const stepNumber = STAGE_NAMES.indexOf(stageName) + 1;
+        return `// Step ${stepNumber}: ${stageName.charAt(0).toUpperCase() + stageName.slice(1)}, ${getStagePurpose(stageName)}
+export const ${stageName} = async ({ io, llm, data, meta, flags }) => {
+
+  return { output: {}, flags };
+}`;
+      }).join("\n\n");
+
+      await fs.writeFile(
+        path.join(tasksDir, `${taskSlug}.js`),
+        taskFileContent + "\n"
+      );
+
+      // Update tasks/index.js
+      const indexFilePath = path.join(tasksDir, "index.js");
+      let taskIndex = {};
+
+      try {
+        const indexContent = await fs.readFile(indexFilePath, "utf8");
+        // Parse the default export from the file
+        const exportMatch = indexContent.match(
+          /export default\s+({[\s\S]*?})\s*;?\s*$/
+        );
+        if (exportMatch) {
+          // Use eval to parse the object (safe in this controlled context)
+          taskIndex = eval(`(${exportMatch[1]})`);
+        }
+      } catch (error) {
+        // If file is missing or invalid, start with empty object
+        taskIndex = {};
+      }
+
+      // Add/replace task mapping
+      taskIndex[taskSlug] = `./${taskSlug}.js`;
+
+      // Sort keys alphabetically for stable output
+      const sortedKeys = Object.keys(taskIndex).sort();
+      const sortedIndex = {};
+      for (const key of sortedKeys) {
+        sortedIndex[key] = taskIndex[key];
+      }
+
+      // Write back the index file with proper formatting
+      const indexContent = `export default ${JSON.stringify(sortedIndex, null, 2)};\n`;
+      await fs.writeFile(indexFilePath, indexContent);
+
+      // Update pipeline.json to include the new task
+      await updatePipelineJson(root, pipelineSlug, taskSlug);
+
+      console.log(`Task "${taskSlug}" added to pipeline "${pipelineSlug}"`);
+    } catch (error) {
+      console.error(`Error adding task: ${error.message}`);
+      process.exit(1);
+    }
+  });
+
+// Helper function to get stage purpose descriptions
+function getStagePurpose(stageName) {
+  const purposes = {
+    ingestion:
+      "load/shape input for downstream stages (no external side-effects required)",
+    preProcessing: "prepare and clean data for main processing",
+    promptTemplating: "generate or format prompts for LLM interaction",
+    inference: "execute LLM calls or other model inference",
+    parsing: "extract and structure results from model outputs",
+    validateStructure: "ensure output meets expected format and schema",
+    validateQuality: "check content quality and completeness",
+    critique: "analyze and evaluate results against criteria",
+    refine: "improve and optimize outputs based on feedback",
+    finalValidation: "perform final checks before completion",
+    integration: "integrate results into downstream systems or workflows",
+  };
+  return purposes[stageName] || "handle stage-specific processing";
+}
+
 program.parse();

package/src/cli/run-orchestrator.js

@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+import { startOrchestrator } from "../core/orchestrator.js";
+
+async function main() {
+  const root = process.env.PO_ROOT;
+
+  if (!root) {
+    console.error(
+      "PO_ROOT environment variable is required. Please set PO_ROOT to your pipeline root directory (e.g., ./demo)."
+    );
+    process.exit(1);
+  }
+
+  try {
+    console.log(`Starting orchestrator with dataDir: ${root}`);
+    const { stop } = await startOrchestrator({ dataDir: root });
+
+    // Handle graceful shutdown
+    process.on("SIGINT", async () => {
+      console.log("\nReceived SIGINT, shutting down orchestrator...");
+      await stop();
+      process.exit(0);
+    });
+
+    process.on("SIGTERM", async () => {
+      console.log("\nReceived SIGTERM, shutting down orchestrator...");
+      await stop();
+      process.exit(0);
+    });
+  } catch (error) {
+    console.error("Orchestrator failed to start:", error.message);
+    process.exit(1);
+  }
+}
+
+main().catch((error) => {
+  console.error("Unhandled error in orchestrator runner:", error);
+  process.exit(1);
+});

package/src/cli/update-pipeline-json.js

@@ -0,0 +1,47 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Updates pipeline.json to include a new task
+ * @param {string} root - The pipeline root directory
+ * @param {string} pipelineSlug - The pipeline slug
+ * @param {string} taskSlug - The task slug to add
+ */
+export async function updatePipelineJson(root, pipelineSlug, taskSlug) {
+  const pipelineConfigPath = path.join(
+    root,
+    "pipeline-config",
+    pipelineSlug,
+    "pipeline.json"
+  );
+  let pipelineConfig = {};
+
+  try {
+    const pipelineContent = await fs.readFile(pipelineConfigPath, "utf8");
+    pipelineConfig = JSON.parse(pipelineContent);
+  } catch (error) {
+    // If file is missing or invalid, create minimal config
+    pipelineConfig = {
+      name: pipelineSlug,
+      version: "1.0.0",
+      description: "New pipeline",
+      tasks: [],
+    };
+  }
+
+  // Ensure tasks array exists
+  if (!Array.isArray(pipelineConfig.tasks)) {
+    pipelineConfig.tasks = [];
+  }
+
+  // Add task to the end of the list if not already present
+  if (!pipelineConfig.tasks.includes(taskSlug)) {
+    pipelineConfig.tasks.push(taskSlug);
+  }
+
+  // Write back pipeline.json
+  await fs.writeFile(
+    pipelineConfigPath,
+    JSON.stringify(pipelineConfig, null, 2) + "\n"
+  );
+}