@ryanfw/prompt-orchestration-pipeline 0.0.1 → 0.4.0

package/src/api/index.js

@@ -1,28 +1,31 @@
-import { Orchestrator } from "../core/orchestrator.js";
+import { startOrchestrator } from "../core/orchestrator.js";
 import path from "node:path";
 import fs from "node:fs/promises";
-import { validateSeedOrThrow } from "../core/validation.js";
+import { validateSeed } from "./validators/seed.js";
+import { atomicWrite, cleanupOnFailure } from "./files.js";
+import { getPipelineConfig } from "../core/config.js";
+import {
+  getPendingSeedPath,
+  resolvePipelinePaths,
+  getJobDirectoryPath,
+  getJobMetadataPath,
+  getJobPipelinePath,
+} from "../config/paths.js";
+import { generateJobId } from "../utils/id-generator.js";
 
 // Pure functional utilities
 const createPaths = (config) => {
-  const {
-    rootDir,
-    dataDir = "pipeline-data",
-    configDir = "pipeline-config",
-  } = config;
+  const { rootDir, dataDir = "pipeline-data" } = config;
   return {
     pending: path.join(rootDir, dataDir, "pending"),
     current: path.join(rootDir, dataDir, "current"),
     complete: path.join(rootDir, dataDir, "complete"),
-    pipeline: path.join(rootDir, configDir, "pipeline.json"),
-    tasks: path.join(rootDir, configDir, "tasks"),
   };
 };
 
 const validateConfig = (options = {}) => ({
   rootDir: options.rootDir || process.cwd(),
   dataDir: options.dataDir || "pipeline-data",
-  configDir: options.configDir || "pipeline-config",
   autoStart: options.autoStart ?? true,
   ui: options.ui ?? false,
   uiPort: options.uiPort || 3000,
@@ -36,22 +39,10 @@ const ensureDirectories = async (paths) => {
   }
 };
 
-const loadPipelineDefinition = async (pipelinePath) => {
-  try {
-    const content = await fs.readFile(pipelinePath, "utf8");
-    const definition = JSON.parse(content);
-    definition.__path = pipelinePath;
-    return definition;
-  } catch (error) {
-    if (error.code === "ENOENT") {
-      throw new Error(`Pipeline definition not found at ${pipelinePath}`);
-    }
-    throw error;
-  }
-};
-
-const createOrchestrator = (paths, pipelineDefinition) =>
-  new Orchestrator({ paths, pipelineDefinition });
+const createOrchestrator = (paths, pipelineDefinition, rootDir) =>
+  // Accept an explicit rootDir (project root) to avoid passing a subpath.
+  // Pipeline is mandatory - see docs/plans/multi-pipeline-backend-plan-PR2.md
+  startOrchestrator({ dataDir: rootDir || paths.pending });
 
 // Main API functions
 export const createPipelineOrchestrator = async (options = {}) => {
@@ -59,40 +50,39 @@ export const createPipelineOrchestrator = async (options = {}) => {
   const paths = createPaths(config);
 
   await ensureDirectories(paths);
-  const pipelineDefinition = await loadPipelineDefinition(paths.pipeline);
-  const orchestrator = createOrchestrator(paths, pipelineDefinition);
+
+  // Pass config.rootDir as the orchestrator dataDir root so the orchestrator resolves
+  // pipeline-data/... correctly (avoids duplicate path segments).
+  const orchestrator = await createOrchestrator(
+    paths,
+    null, // unused - pipeline definition will be loaded per-job
+    config.rootDir
+  );
 
   let uiServer = null;
 
   const state = {
     config,
     paths,
-    pipelineDefinition,
+    pipelineDefinition: undefined, // TODO: multi-pipeline UI will provide per-slug previews from snapshots
     orchestrator,
     uiServer,
   };
 
-  // Auto-start if configured
-  if (config.autoStart) {
-    await orchestrator.start();
-  }
+  // Auto-start if configured (startOrchestrator handles this automatically)
+  // No need to call orchestrator.start() as startOrchestrator auto-starts when autoStart=true
 
   // Start UI if configured
   if (config.ui) {
-    const { createUIServer } = await import("../ui/server.js");
+    const { startServer } = await import("../ui/server.js");
 
-    // Create API object with state injection for UI server
-    const uiApi = {
-      submitJob: (seed) => submitJob(state, seed),
-      getStatus: (jobName) => getStatus(state, jobName),
-      listJobs: (status) => listJobs(state, status),
-    };
-
-    uiServer = createUIServer(uiApi);
-    uiServer.listen(config.uiPort, () => {
-      console.log(`Pipeline UI available at http://localhost:${config.uiPort}`);
+    uiServer = await startServer({
+      dataDir: config.rootDir,
+      port: config.uiPort,
     });
+
     state.uiServer = uiServer;
+    console.log(`Pipeline UI available at ${uiServer.url}`);
   }
 
   return state;
@@ -100,13 +90,120 @@ export const createPipelineOrchestrator = async (options = {}) => {
 
 // Job management functions
 export const submitJob = async (state, seed) => {
-  // Validate seed structure before submitting
-  validateSeedOrThrow(seed);
+  throw new Error(
+    "submitJob is deprecated. Use submitJobWithValidation instead for ID-only job submission."
+  );
+};
+
+/**
+ * Submit a job with comprehensive validation and atomic writes
+ * @param {Object} options - Options object
+ * @param {string} options.dataDir - Base data directory
+ * @param {Object} options.seedObject - Seed object to submit
+ * @returns {Promise<Object>} Result object with success status
+ */
+export const submitJobWithValidation = async ({ dataDir, seedObject }) => {
+  let partialFiles = [];
+
+  try {
+    // Validate the seed object
+    const validatedSeed = await validateSeed(
+      JSON.stringify(seedObject),
+      dataDir
+    );
+
+    // Generate a random job ID
+    const jobId = generateJobId();
+
+    // Get the paths
+    const paths = resolvePipelinePaths(dataDir);
+    const pendingPath = getPendingSeedPath(dataDir, jobId);
+    const currentJobDir = getJobDirectoryPath(dataDir, jobId, "current");
+    const jobMetadataPath = getJobMetadataPath(dataDir, jobId, "current");
+    const jobPipelinePath = getJobPipelinePath(dataDir, jobId, "current");
+
+    // Ensure directories exist
+    await fs.mkdir(paths.pending, { recursive: true });
+    await fs.mkdir(currentJobDir, { recursive: true });
+
+    // Create job metadata
+    const jobMetadata = {
+      id: jobId,
+      name: validatedSeed.name,
+      pipeline: validatedSeed.pipeline, // Include pipeline slug
+      createdAt: new Date().toISOString(),
+      status: "pending",
+    };
 
-  const name = seed.name;
-  const seedPath = path.join(state.paths.pending, `${name}-seed.json`);
-  await fs.writeFile(seedPath, JSON.stringify(seed, null, 2));
-  return { name, seedPath };
+    // Read pipeline configuration for snapshot
+    let pipelineSnapshot = null;
+    try {
+      // Compute snapshot path from the seed-derived slug
+      const pipelineSlug = validatedSeed.pipeline;
+      const { pipelineJsonPath } = getPipelineConfig(pipelineSlug);
+      const pipelineContent = await fs.readFile(pipelineJsonPath, "utf8");
+      pipelineSnapshot = JSON.parse(pipelineContent);
+    } catch (error) {
+      // Handle unknown pipeline slug
+      if (
+        error.message.includes("Unknown pipeline") ||
+        error.message.includes("not found")
+      ) {
+        const errorMessage = "Unknown pipeline slug: " + validatedSeed.pipeline;
+        return {
+          success: false,
+          message: errorMessage,
+        };
+      }
+      // If pipeline config doesn't exist, create a minimal snapshot
+      pipelineSnapshot = {
+        tasks: [],
+        name: validatedSeed.pipeline,
+      };
+    }
+
+    // Write files atomically
+    partialFiles.push(pendingPath);
+    await atomicWrite(pendingPath, JSON.stringify(validatedSeed, null, 2));
+
+    partialFiles.push(jobMetadataPath);
+    await atomicWrite(jobMetadataPath, JSON.stringify(jobMetadata, null, 2));
+
+    partialFiles.push(jobPipelinePath);
+    await atomicWrite(
+      jobPipelinePath,
+      JSON.stringify(pipelineSnapshot, null, 2)
+    );
+
+    return {
+      success: true,
+      jobId,
+      jobName: validatedSeed.name,
+      message: "Seed file uploaded successfully",
+    };
+  } catch (error) {
+    // Clean up any partial files on failure
+    for (const filePath of partialFiles) {
+      try {
+        await cleanupOnFailure(filePath);
+      } catch (cleanupError) {
+        // Ignore cleanup errors
+      }
+    }
+
+    // Map validation errors to appropriate error messages
+    let errorMessage = error.message;
+    if (error.message.includes("Invalid JSON")) {
+      errorMessage = "Invalid JSON";
+    } else if (error.message.includes("required")) {
+      errorMessage = "Required fields missing";
+    }
+
+    return {
+      success: false,
+      message: errorMessage,
+    };
+  }
 };
 
 export const getStatus = async (state, jobName) => {
@@ -166,13 +263,13 @@ export const listJobs = async (state, status = "all") => {
 
 // Control functions
 export const start = async (state) => {
-  await state.orchestrator.start();
+  // startOrchestrator already starts automatically, no need to call start
   return state;
 };
 
 export const stop = async (state) => {
   if (state.uiServer) {
-    await new Promise((resolve) => state.uiServer.close(resolve));
+    await state.uiServer.close();
   }
   await state.orchestrator.stop();
   return state;
@@ -215,6 +312,5 @@ export const PipelineOrchestrator = {
 
 // Export the original functions for direct functional usage
 export { runPipeline } from "../core/task-runner.js";
-export { selectModel } from "../core/task-runner.js";
 
 export default PipelineOrchestrator;

package/src/api/validators/seed.js

@@ -0,0 +1,141 @@
+/**
+ * Seed validation utilities
+ * @module api/validators/seed
+ */
+
+import { promises as fs } from "fs";
+import path from "path";
+import { getPipelineConfig } from "../../core/config.js";
+
+/**
+ * Validate JSON string and parse it
+ * @param {string} jsonString - JSON string to validate
+ * @returns {Object} Parsed JSON object
+ * @throws {Error} With message containing "Invalid JSON" on parse failure
+ */
+function validateAndParseJson(jsonString) {
+  try {
+    return JSON.parse(jsonString);
+  } catch (error) {
+    throw new Error("Invalid JSON");
+  }
+}
+
+/**
+ * Validate required fields in seed object
+ * @param {Object} seedObject - Seed object to validate
+ * @returns {Object} Validated seed object
+ * @throws {Error} With message containing "required" if fields are missing
+ */
+function validateRequiredFields(seedObject) {
+  if (
+    !seedObject.name ||
+    typeof seedObject.name !== "string" ||
+    seedObject.name.trim() === ""
+  ) {
+    throw new Error("name field is required");
+  }
+
+  if (!seedObject.data || typeof seedObject.data !== "object") {
+    throw new Error("data field is required");
+  }
+
+  if (
+    !seedObject.pipeline ||
+    typeof seedObject.pipeline !== "string" ||
+    seedObject.pipeline.trim() === ""
+  ) {
+    throw new Error("pipeline field is required");
+  }
+
+  return seedObject;
+}
+
+/**
+ * Validate name format (alphanumeric + -/_ allowed)
+ * @param {string} name - Name to validate
+ * @returns {string} Validated name
+ * @throws {Error} With message containing "required" if format is invalid
+ */
+function validateNameFormat(name) {
+  // Allow display-friendly names: required non-empty string, trim, length cap
+  if (!name || typeof name !== "string" || name.trim() === "") {
+    throw new Error("name field is required");
+  }
+  const trimmed = name.trim();
+  if (trimmed.length > 120) {
+    throw new Error("name must be 120 characters or less");
+  }
+  // Allow spaces and common punctuation for better UX
+  // Still disallow control characters and path traversal patterns
+  const dangerousPattern = /[\x00-\x1f\x7f-\x9f]/;
+  if (dangerousPattern.test(trimmed)) {
+    throw new Error("name must contain only printable characters");
+  }
+  return trimmed;
+}
+
+/**
+ * Check if a job with the given name already exists
+ * @param {string} baseDir - Base data directory
+ * @param {string} jobName - Job name to check
+ * @returns {Promise<boolean>} True if duplicate exists
+ */
+async function checkDuplicateJob(baseDir, jobName) {
+  const { getPendingSeedPath, getCurrentSeedPath, getCompleteSeedPath } =
+    await import("../../config/paths.js");
+
+  const paths = [
+    getPendingSeedPath(baseDir, jobName),
+    getCurrentSeedPath(baseDir, jobName),
+    getCompleteSeedPath(baseDir, jobName),
+  ];
+
+  for (const filePath of paths) {
+    try {
+      await fs.access(filePath);
+      return true; // File exists, duplicate found
+    } catch (error) {
+      // File doesn't exist, continue checking
+    }
+  }
+
+  return false; // No duplicates found
+}
+
+/**
+ * Comprehensive seed validation
+ * @param {string} jsonString - JSON string to validate
+ * @param {string} baseDir - Base data directory for duplicate checking
+ * @returns {Promise<Object>} Validated seed object
+ * @throws {Error} With appropriate error message
+ */
+async function validateSeed(jsonString, baseDir) {
+  // Step 1: Validate and parse JSON
+  const seedObject = validateAndParseJson(jsonString);
+
+  // Step 2: Validate required fields
+  const validatedObject = validateRequiredFields(seedObject);
+
+  // Step 3: Validate name format
+  validateNameFormat(validatedObject.name);
+
+  // Step 4: Validate pipeline slug against registry
+  getPipelineConfig(validatedObject.pipeline);
+
+  // Step 5: Check for duplicates
+  const isDuplicate = await checkDuplicateJob(baseDir, validatedObject.name);
+  if (isDuplicate) {
+    throw new Error("Job with this name already exists");
+  }
+
+  return validatedObject;
+}
+
+export {
+  validateAndParseJson,
+  validateRequiredFields,
+  validateNameFormat,
+  checkDuplicateJob,
+  validateSeed,
+};