@ryanfw/prompt-orchestration-pipeline 0.11.0 → 0.13.0

package/src/ui/endpoints/task-save-endpoint.js

@@ -0,0 +1,101 @@
+import path from "node:path";
+import { promises as fs } from "node:fs";
+import { getConfig } from "../../core/config.js";
+import { sendJson } from "../utils/http-utils.js";
+
+/**
+ * Handle task creation requests
+ *
+ * POST /api/tasks/create
+ * Body: { pipelineSlug, filename, taskName, code }
+ *
+ * Creates a new task file and updates the pipeline's task registry index.js
+ */
+export async function handleTaskSave(req, res) {
+  try {
+    const { pipelineSlug, filename, taskName, code } = req.body;
+
+    if (!pipelineSlug) {
+      return sendJson(res, 400, { error: "pipelineSlug is required" });
+    }
+    // Validate filename ends with .js
+    if (!filename || !filename.endsWith(".js")) {
+      return sendJson(res, 400, { error: "Filename must end with .js" });
+    }
+
+    // Validate taskName is kebab-case
+    const kebabCaseRegex = /^[a-z][a-z0-9-]*$/;
+    if (!taskName || !kebabCaseRegex.test(taskName)) {
+      return sendJson(res, 400, { error: "TaskName must be kebab-case" });
+    }
+
+    // Get configuration and root directory
+    const config = getConfig();
+    const rootDir = config.paths.root;
+
+    // Read registry.json to find pipeline's taskRegistryPath
+    const registryPath = path.join(rootDir, "pipeline-config", "registry.json");
+    const registryData = JSON.parse(await fs.readFile(registryPath, "utf8"));
+
+    // Look up pipeline in registry
+    const pipelineEntry = registryData.pipelines[pipelineSlug];
+    if (!pipelineEntry) {
+      return sendJson(res, 404, { error: "Pipeline not found" });
+    }
+
+    // Get task registry path (relative to root)
+    const taskRegistryPath = path.join(rootDir, pipelineEntry.taskRegistryPath);
+    const tasksDir = path.dirname(taskRegistryPath);
+
+    // Write task file (prevent path traversal by validating resolved path)
+    const taskFilePath = path.resolve(tasksDir, filename);
+    if (!taskFilePath.startsWith(tasksDir)) {
+      return sendJson(res, 400, { error: "Invalid filename" });
+    }
+    await fs.writeFile(taskFilePath, code, "utf8");
+
+    // Update index.js to export new task
+    const indexPath = taskRegistryPath;
+    let indexContent = await fs.readFile(indexPath, "utf8");
+
+    // Check if task name already exists in the index
+    const taskNamePattern = new RegExp(`^\\s*${taskName}\\s*:`, "m");
+    if (taskNamePattern.test(indexContent)) {
+      return sendJson(res, 400, {
+        error: `Task "${taskName}" already exists in the registry`,
+      });
+    }
+
+    // Find the line containing "export default {"
+    const exportLine = "export default {";
+    const exportLineIndex = indexContent.indexOf(exportLine);
+
+    if (exportLineIndex === -1) {
+      return sendJson(res, 500, {
+        error: "Failed to find export default line in index.js",
+      });
+    }
+
+    // Insert new task entry after the export line
+    const insertPosition = indexContent.indexOf("\n", exportLineIndex) + 1;
+    const newEntry = `  ${taskName}: "./${filename}",\n`;
+
+    indexContent =
+      indexContent.slice(0, insertPosition) +
+      newEntry +
+      indexContent.slice(insertPosition);
+
+    // Write updated index.js
+    await fs.writeFile(indexPath, indexContent, "utf8");
+
+    return sendJson(res, 200, {
+      ok: true,
+      path: taskFilePath,
+    });
+  } catch (error) {
+    console.error("Error creating task:", error);
+    return sendJson(res, 500, {
+      error: error.message || "Failed to create task",
+    });
+  }
+}

package/src/ui/endpoints/upload-endpoints.js

@@ -0,0 +1,406 @@
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { sseRegistry } from "../sse.js";
+import { initializeJobArtifacts } from "../../core/status-writer.js";
+import {
+  resolvePipelinePaths,
+  getPendingSeedPath,
+  getJobDirectoryPath,
+  getJobMetadataPath,
+  getJobPipelinePath,
+} from "../../config/paths.js";
+import { generateJobId } from "../../utils/id-generator.js";
+import { extractSeedZip } from "../zip-utils.js";
+import {
+  sendJson,
+  readRawBody,
+  parseMultipartFormData,
+} from "../utils/http-utils.js";
+
+// Get __dirname equivalent in ES modules
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const DATA_DIR = process.env.PO_ROOT || process.cwd();
+
+const exists = async (p) =>
+  fs.promises
+    .access(p)
+    .then(() => true)
+    .catch(() => false);
+
+/**
+ * Normalize seed upload from various input formats
+ * @param {http.IncomingMessage} req - HTTP request
+ * @param {string} contentTypeHeader - Content-Type header
+ * @returns {Promise<{seedObject: Object, uploadArtifacts: Array<{filename: string, content: Buffer}>}>}
+ */
+async function normalizeSeedUpload({ req, contentTypeHeader }) {
+  // Handle application/json uploads
+  if (contentTypeHeader.includes("application/json")) {
+    const buffer = await readRawBody(req);
+    try {
+      const seedObject = JSON.parse(buffer.toString("utf8") || "{}");
+      return {
+        seedObject,
+        uploadArtifacts: [{ filename: "seed.json", content: buffer }],
+      };
+    } catch (error) {
+      throw new Error("Invalid JSON");
+    }
+  }
+
+  // Handle multipart form data uploads
+  const formData = await parseMultipartFormData(req);
+  if (!formData.contentBuffer) {
+    throw new Error("No file content found");
+  }
+
+  // Check if this is a zip file
+  const isZipFile =
+    formData.contentType === "application/zip" ||
+    formData.filename?.toLowerCase().endsWith(".zip");
+
+  if (isZipFile) {
+    console.log("[UPLOAD] Detected zip upload", {
+      filename: formData.filename,
+      contentType: formData.contentType,
+      bufferSize: formData.contentBuffer.length,
+    });
+
+    // Handle zip upload
+    try {
+      const { seedObject, artifacts } = await extractSeedZip(
+        formData.contentBuffer
+      );
+      console.log("[UPLOAD] Zip extraction completed", {
+        artifactCount: artifacts.length,
+        artifactNames: artifacts.map((a) => a.filename),
+        seedKeys: Object.keys(seedObject),
+      });
+      return {
+        seedObject,
+        uploadArtifacts: artifacts,
+      };
+    } catch (error) {
+      console.log("[UPLOAD] Zip extraction failed", {
+        error: error.message,
+        filename: formData.filename,
+      });
+      // Re-throw zip-specific errors with clear messages
+      throw new Error(error.message);
+    }
+  } else {
+    // Handle regular JSON file upload
+    try {
+      const seedObject = JSON.parse(formData.contentBuffer.toString("utf8"));
+      const filename = formData.filename || "seed.json";
+      return {
+        seedObject,
+        uploadArtifacts: [{ filename, content: formData.contentBuffer }],
+      };
+    } catch (error) {
+      throw new Error("Invalid JSON");
+    }
+  }
+}
+
+/**
+ * Handle seed file upload
+ * @param {http.IncomingMessage} req - HTTP request
+ * @param {http.ServerResponse} res - HTTP response
+ */
+async function handleSeedUpload(req, res) {
+  // Add logging at the very start of the upload handler
+  console.log("[UPLOAD] Incoming seed upload", {
+    method: req.method,
+    url: req.url,
+    contentType: req.headers["content-type"],
+    userAgent: req.headers["user-agent"],
+  });
+
+  try {
+    const ct = req.headers["content-type"] || "";
+
+    // Use the new normalization function to handle all upload formats
+    let normalizedUpload;
+    try {
+      normalizedUpload = await normalizeSeedUpload({
+        req,
+        contentTypeHeader: ct,
+      });
+    } catch (error) {
+      console.log("[UPLOAD] Normalization failed", {
+        error: error.message,
+        contentType: ct,
+      });
+
+      // Handle specific zip-related errors with appropriate messages
+      let errorMessage = error.message;
+      if (error.message === "Invalid JSON") {
+        errorMessage = "Invalid JSON";
+      } else if (error.message === "seed.json not found in zip") {
+        errorMessage = "seed.json not found in zip";
+      }
+
+      res.writeHead(400, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ success: false, message: errorMessage }));
+      return;
+    }
+
+    const { seedObject, uploadArtifacts } = normalizedUpload;
+
+    // Use current PO_ROOT or fallback to DATA_DIR
+    const currentDataDir = process.env.PO_ROOT || DATA_DIR;
+
+    // For test environment, use simplified validation without starting orchestrator
+    if (process.env.NODE_ENV === "test") {
+      // Simplified validation for tests - just write to pending directory
+      const result = await handleSeedUploadDirect(
+        seedObject,
+        currentDataDir,
+        uploadArtifacts
+      );
+
+      // Return appropriate status code based on success
+      if (result.success) {
+        res.writeHead(200, {
+          "Content-Type": "application/json",
+          Connection: "close",
+        });
+        res.end(JSON.stringify(result));
+
+        // Broadcast SSE event for successful upload
+        sseRegistry.broadcast({
+          type: "seed:uploaded",
+          data: { name: result.jobName },
+        });
+      } else {
+        res.writeHead(400, {
+          "Content-Type": "application/json",
+          Connection: "close",
+        });
+        res.end(JSON.stringify(result));
+      }
+      return;
+    }
+
+    // Submit job with validation (for production)
+    // Dynamically import only in non-test mode
+    if (process.env.NODE_ENV !== "test") {
+      const { submitJobWithValidation } = await import("../../api/index.js");
+      const result = await submitJobWithValidation({
+        dataDir: currentDataDir,
+        seedObject,
+        uploadArtifacts,
+      });
+
+      // Send appropriate response
+      if (result.success) {
+        res.writeHead(200, { "Content-Type": "application/json" });
+        res.end(JSON.stringify(result));
+
+        // Broadcast SSE event for successful upload
+        sseRegistry.broadcast({
+          type: "seed:uploaded",
+          data: { name: result.jobName },
+        });
+      } else {
+        res.writeHead(400, { "Content-Type": "application/json" });
+        res.end(JSON.stringify(result));
+      }
+    } else {
+      // In test mode, we should never reach here, but handle gracefully
+      res.writeHead(500, { "Content-Type": "application/json" });
+      res.end(
+        JSON.stringify({
+          success: false,
+          message:
+            "Test environment error - should not reach production code path",
+        })
+      );
+    }
+  } catch (error) {
+    console.error("Upload error:", error);
+    res.writeHead(500, { "Content-Type": "application/json" });
+    res.end(
+      JSON.stringify({
+        success: false,
+        message: "Internal server error",
+      })
+    );
+  }
+}
+
+/**
+ * Handle seed upload directly without starting orchestrator (for test environment)
+ * @param {Object} seedObject - Seed object to upload
+ * @param {string} dataDir - Base data directory
+ * @param {Array} uploadArtifacts - Array of {filename, content} objects
+ * @returns {Promise<Object>} Result object
+ */
+async function handleSeedUploadDirect(
+  seedObject,
+  dataDir,
+  uploadArtifacts = []
+) {
+  let partialFiles = [];
+
+  try {
+    // Basic validation
+    if (
+      !seedObject.name ||
+      typeof seedObject.name !== "string" ||
+      seedObject.name.trim() === ""
+    ) {
+      return {
+        success: false,
+        message: "Required fields missing",
+      };
+    }
+
+    const hasValidPayload = (seed) => {
+      if (!seed || typeof seed !== "object") return false;
+      const hasData = seed.data && typeof seed.data === "object";
+      const hasPipelineParams =
+        typeof seed.pipeline === "string" &&
+        seed.params &&
+        typeof seed.params === "object";
+      return hasData || hasPipelineParams;
+    };
+
+    if (!hasValidPayload(seedObject)) {
+      return { success: false, message: "Required fields missing" };
+    }
+
+    // Validate name format using the same logic as seed validator
+    if (
+      !seedObject.name ||
+      typeof seedObject.name !== "string" ||
+      seedObject.name.trim() === ""
+    ) {
+      return {
+        success: false,
+        message: "name field is required",
+      };
+    }
+
+    const trimmedName = seedObject.name.trim();
+    if (trimmedName.length > 120) {
+      return {
+        success: false,
+        message: "name must be 120 characters or less",
+      };
+    }
+
+    // Allow spaces and common punctuation for better UX
+    // Still disallow control characters and path traversal patterns
+    const dangerousPattern = /[\x00-\x1f\x7f-\x9f]/;
+    if (dangerousPattern.test(trimmedName)) {
+      return {
+        success: false,
+        message: "name must contain only printable characters",
+      };
+    }
+
+    // Update seedObject with validated trimmed name
+    seedObject.name = trimmedName;
+
+    // Generate a random job ID
+    const jobId = generateJobId();
+
+    // Get the paths
+    const paths = resolvePipelinePaths(dataDir);
+    const pendingPath = getPendingSeedPath(dataDir, jobId);
+    const currentJobDir = getJobDirectoryPath(dataDir, jobId, "current");
+    const jobMetadataPath = getJobMetadataPath(dataDir, jobId, "current");
+    const jobPipelinePath = getJobPipelinePath(dataDir, jobId, "current");
+
+    // Ensure directories exist
+    await fs.promises.mkdir(paths.pending, { recursive: true });
+    await fs.promises.mkdir(currentJobDir, { recursive: true });
+
+    // Create job metadata
+    const jobMetadata = {
+      id: jobId,
+      name: seedObject.name,
+      pipeline: seedObject.pipeline || "default",
+      createdAt: new Date().toISOString(),
+      status: "pending",
+    };
+
+    // Read pipeline configuration for snapshot
+    let pipelineSnapshot = null;
+    try {
+      const pipelineConfigPath = path.join(
+        dataDir,
+        "pipeline-config",
+        "pipeline.json"
+      );
+      const pipelineContent = await fs.promises.readFile(
+        pipelineConfigPath,
+        "utf8"
+      );
+      pipelineSnapshot = JSON.parse(pipelineContent);
+    } catch (error) {
+      // If pipeline config doesn't exist, create a minimal snapshot
+      pipelineSnapshot = {
+        tasks: [],
+        name: seedObject.pipeline || "default",
+      };
+    }
+
+    // Write files atomically
+    partialFiles.push(pendingPath);
+    await fs.promises.writeFile(
+      pendingPath,
+      JSON.stringify(seedObject, null, 2)
+    );
+
+    partialFiles.push(jobMetadataPath);
+    await fs.promises.writeFile(
+      jobMetadataPath,
+      JSON.stringify(jobMetadata, null, 2)
+    );
+
+    partialFiles.push(jobPipelinePath);
+    await fs.promises.writeFile(
+      jobPipelinePath,
+      JSON.stringify(pipelineSnapshot, null, 2)
+    );
+
+    // Initialize job artifacts if any provided
+    if (uploadArtifacts.length > 0) {
+      try {
+        await initializeJobArtifacts(currentJobDir, uploadArtifacts);
+      } catch (artifactError) {
+        // Don't fail the upload if artifact initialization fails, just log the error
+        console.error("Failed to initialize job artifacts:", artifactError);
+      }
+    }
+
+    return {
+      success: true,
+      jobId,
+      jobName: seedObject.name,
+      message: "Seed file uploaded successfully",
+    };
+  } catch (error) {
+    // Clean up any partial files on failure
+    for (const filePath of partialFiles) {
+      try {
+        await fs.promises.unlink(filePath);
+      } catch (cleanupError) {
+        // Ignore cleanup errors
+      }
+    }
+
+    return {
+      success: false,
+      message: error.message || "Internal server error",
+    };
+  }
+}
+
+export { handleSeedUpload, normalizeSeedUpload, handleSeedUploadDirect };