@ryanfw/prompt-orchestration-pipeline 0.10.0 → 0.12.0

package/src/ui/endpoints/upload-endpoints.js

@@ -0,0 +1,406 @@
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { sseRegistry } from "../sse.js";
+import { initializeJobArtifacts } from "../../core/status-writer.js";
+import {
+  resolvePipelinePaths,
+  getPendingSeedPath,
+  getJobDirectoryPath,
+  getJobMetadataPath,
+  getJobPipelinePath,
+} from "../../config/paths.js";
+import { generateJobId } from "../../utils/id-generator.js";
+import { extractSeedZip } from "../zip-utils.js";
+import {
+  sendJson,
+  readRawBody,
+  parseMultipartFormData,
+} from "../utils/http-utils.js";
+
+// Get __dirname equivalent in ES modules
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const DATA_DIR = process.env.PO_ROOT || process.cwd();
+
+const exists = async (p) =>
+  fs.promises
+    .access(p)
+    .then(() => true)
+    .catch(() => false);
+
+/**
+ * Normalize seed upload from various input formats
+ * @param {http.IncomingMessage} req - HTTP request
+ * @param {string} contentTypeHeader - Content-Type header
+ * @returns {Promise<{seedObject: Object, uploadArtifacts: Array<{filename: string, content: Buffer}>}>}
+ */
+async function normalizeSeedUpload({ req, contentTypeHeader }) {
+  // Handle application/json uploads
+  if (contentTypeHeader.includes("application/json")) {
+    const buffer = await readRawBody(req);
+    try {
+      const seedObject = JSON.parse(buffer.toString("utf8") || "{}");
+      return {
+        seedObject,
+        uploadArtifacts: [{ filename: "seed.json", content: buffer }],
+      };
+    } catch (error) {
+      throw new Error("Invalid JSON");
+    }
+  }
+
+  // Handle multipart form data uploads
+  const formData = await parseMultipartFormData(req);
+  if (!formData.contentBuffer) {
+    throw new Error("No file content found");
+  }
+
+  // Check if this is a zip file
+  const isZipFile =
+    formData.contentType === "application/zip" ||
+    formData.filename?.toLowerCase().endsWith(".zip");
+
+  if (isZipFile) {
+    console.log("[UPLOAD] Detected zip upload", {
+      filename: formData.filename,
+      contentType: formData.contentType,
+      bufferSize: formData.contentBuffer.length,
+    });
+
+    // Handle zip upload
+    try {
+      const { seedObject, artifacts } = await extractSeedZip(
+        formData.contentBuffer
+      );
+      console.log("[UPLOAD] Zip extraction completed", {
+        artifactCount: artifacts.length,
+        artifactNames: artifacts.map((a) => a.filename),
+        seedKeys: Object.keys(seedObject),
+      });
+      return {
+        seedObject,
+        uploadArtifacts: artifacts,
+      };
+    } catch (error) {
+      console.log("[UPLOAD] Zip extraction failed", {
+        error: error.message,
+        filename: formData.filename,
+      });
+      // Re-throw zip-specific errors with clear messages
+      throw new Error(error.message);
+    }
+  } else {
+    // Handle regular JSON file upload
+    try {
+      const seedObject = JSON.parse(formData.contentBuffer.toString("utf8"));
+      const filename = formData.filename || "seed.json";
+      return {
+        seedObject,
+        uploadArtifacts: [{ filename, content: formData.contentBuffer }],
+      };
+    } catch (error) {
+      throw new Error("Invalid JSON");
+    }
+  }
+}
+
+/**
+ * Handle seed file upload
+ * @param {http.IncomingMessage} req - HTTP request
+ * @param {http.ServerResponse} res - HTTP response
+ */
+async function handleSeedUpload(req, res) {
+  // Add logging at the very start of the upload handler
+  console.log("[UPLOAD] Incoming seed upload", {
+    method: req.method,
+    url: req.url,
+    contentType: req.headers["content-type"],
+    userAgent: req.headers["user-agent"],
+  });
+
+  try {
+    const ct = req.headers["content-type"] || "";
+
+    // Use the new normalization function to handle all upload formats
+    let normalizedUpload;
+    try {
+      normalizedUpload = await normalizeSeedUpload({
+        req,
+        contentTypeHeader: ct,
+      });
+    } catch (error) {
+      console.log("[UPLOAD] Normalization failed", {
+        error: error.message,
+        contentType: ct,
+      });
+
+      // Handle specific zip-related errors with appropriate messages
+      let errorMessage = error.message;
+      if (error.message === "Invalid JSON") {
+        errorMessage = "Invalid JSON";
+      } else if (error.message === "seed.json not found in zip") {
+        errorMessage = "seed.json not found in zip";
+      }
+
+      res.writeHead(400, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ success: false, message: errorMessage }));
+      return;
+    }
+
+    const { seedObject, uploadArtifacts } = normalizedUpload;
+
+    // Use current PO_ROOT or fallback to DATA_DIR
+    const currentDataDir = process.env.PO_ROOT || DATA_DIR;
+
+    // For test environment, use simplified validation without starting orchestrator
+    if (process.env.NODE_ENV === "test") {
+      // Simplified validation for tests - just write to pending directory
+      const result = await handleSeedUploadDirect(
+        seedObject,
+        currentDataDir,
+        uploadArtifacts
+      );
+
+      // Return appropriate status code based on success
+      if (result.success) {
+        res.writeHead(200, {
+          "Content-Type": "application/json",
+          Connection: "close",
+        });
+        res.end(JSON.stringify(result));
+
+        // Broadcast SSE event for successful upload
+        sseRegistry.broadcast({
+          type: "seed:uploaded",
+          data: { name: result.jobName },
+        });
+      } else {
+        res.writeHead(400, {
+          "Content-Type": "application/json",
+          Connection: "close",
+        });
+        res.end(JSON.stringify(result));
+      }
+      return;
+    }
+
+    // Submit job with validation (for production)
+    // Dynamically import only in non-test mode
+    if (process.env.NODE_ENV !== "test") {
+      const { submitJobWithValidation } = await import("../../api/index.js");
+      const result = await submitJobWithValidation({
+        dataDir: currentDataDir,
+        seedObject,
+        uploadArtifacts,
+      });
+
+      // Send appropriate response
+      if (result.success) {
+        res.writeHead(200, { "Content-Type": "application/json" });
+        res.end(JSON.stringify(result));
+
+        // Broadcast SSE event for successful upload
+        sseRegistry.broadcast({
+          type: "seed:uploaded",
+          data: { name: result.jobName },
+        });
+      } else {
+        res.writeHead(400, { "Content-Type": "application/json" });
+        res.end(JSON.stringify(result));
+      }
+    } else {
+      // In test mode, we should never reach here, but handle gracefully
+      res.writeHead(500, { "Content-Type": "application/json" });
+      res.end(
+        JSON.stringify({
+          success: false,
+          message:
+            "Test environment error - should not reach production code path",
+        })
+      );
+    }
+  } catch (error) {
+    console.error("Upload error:", error);
+    res.writeHead(500, { "Content-Type": "application/json" });
+    res.end(
+      JSON.stringify({
+        success: false,
+        message: "Internal server error",
+      })
+    );
+  }
+}
+
+/**
+ * Handle seed upload directly without starting orchestrator (for test environment)
+ * @param {Object} seedObject - Seed object to upload
+ * @param {string} dataDir - Base data directory
+ * @param {Array} uploadArtifacts - Array of {filename, content} objects
+ * @returns {Promise<Object>} Result object
+ */
+async function handleSeedUploadDirect(
+  seedObject,
+  dataDir,
+  uploadArtifacts = []
+) {
+  let partialFiles = [];
+
+  try {
+    // Basic validation
+    if (
+      !seedObject.name ||
+      typeof seedObject.name !== "string" ||
+      seedObject.name.trim() === ""
+    ) {
+      return {
+        success: false,
+        message: "Required fields missing",
+      };
+    }
+
+    const hasValidPayload = (seed) => {
+      if (!seed || typeof seed !== "object") return false;
+      const hasData = seed.data && typeof seed.data === "object";
+      const hasPipelineParams =
+        typeof seed.pipeline === "string" &&
+        seed.params &&
+        typeof seed.params === "object";
+      return hasData || hasPipelineParams;
+    };
+
+    if (!hasValidPayload(seedObject)) {
+      return { success: false, message: "Required fields missing" };
+    }
+
+    // Validate name format using the same logic as seed validator
+    if (
+      !seedObject.name ||
+      typeof seedObject.name !== "string" ||
+      seedObject.name.trim() === ""
+    ) {
+      return {
+        success: false,
+        message: "name field is required",
+      };
+    }
+
+    const trimmedName = seedObject.name.trim();
+    if (trimmedName.length > 120) {
+      return {
+        success: false,
+        message: "name must be 120 characters or less",
+      };
+    }
+
+    // Allow spaces and common punctuation for better UX
+    // Still disallow control characters and path traversal patterns
+    const dangerousPattern = /[\x00-\x1f\x7f-\x9f]/;
+    if (dangerousPattern.test(trimmedName)) {
+      return {
+        success: false,
+        message: "name must contain only printable characters",
+      };
+    }
+
+    // Update seedObject with validated trimmed name
+    seedObject.name = trimmedName;
+
+    // Generate a random job ID
+    const jobId = generateJobId();
+
+    // Get the paths
+    const paths = resolvePipelinePaths(dataDir);
+    const pendingPath = getPendingSeedPath(dataDir, jobId);
+    const currentJobDir = getJobDirectoryPath(dataDir, jobId, "current");
+    const jobMetadataPath = getJobMetadataPath(dataDir, jobId, "current");
+    const jobPipelinePath = getJobPipelinePath(dataDir, jobId, "current");
+
+    // Ensure directories exist
+    await fs.promises.mkdir(paths.pending, { recursive: true });
+    await fs.promises.mkdir(currentJobDir, { recursive: true });
+
+    // Create job metadata
+    const jobMetadata = {
+      id: jobId,
+      name: seedObject.name,
+      pipeline: seedObject.pipeline || "default",
+      createdAt: new Date().toISOString(),
+      status: "pending",
+    };
+
+    // Read pipeline configuration for snapshot
+    let pipelineSnapshot = null;
+    try {
+      const pipelineConfigPath = path.join(
+        dataDir,
+        "pipeline-config",
+        "pipeline.json"
+      );
+      const pipelineContent = await fs.promises.readFile(
+        pipelineConfigPath,
+        "utf8"
+      );
+      pipelineSnapshot = JSON.parse(pipelineContent);
+    } catch (error) {
+      // If pipeline config doesn't exist, create a minimal snapshot
+      pipelineSnapshot = {
+        tasks: [],
+        name: seedObject.pipeline || "default",
+      };
+    }
+
+    // Write files atomically
+    partialFiles.push(pendingPath);
+    await fs.promises.writeFile(
+      pendingPath,
+      JSON.stringify(seedObject, null, 2)
+    );
+
+    partialFiles.push(jobMetadataPath);
+    await fs.promises.writeFile(
+      jobMetadataPath,
+      JSON.stringify(jobMetadata, null, 2)
+    );
+
+    partialFiles.push(jobPipelinePath);
+    await fs.promises.writeFile(
+      jobPipelinePath,
+      JSON.stringify(pipelineSnapshot, null, 2)
+    );
+
+    // Initialize job artifacts if any provided
+    if (uploadArtifacts.length > 0) {
+      try {
+        await initializeJobArtifacts(currentJobDir, uploadArtifacts);
+      } catch (artifactError) {
+        // Don't fail the upload if artifact initialization fails, just log the error
+        console.error("Failed to initialize job artifacts:", artifactError);
+      }
+    }
+
+    return {
+      success: true,
+      jobId,
+      jobName: seedObject.name,
+      message: "Seed file uploaded successfully",
+    };
+  } catch (error) {
+    // Clean up any partial files on failure
+    for (const filePath of partialFiles) {
+      try {
+        await fs.promises.unlink(filePath);
+      } catch (cleanupError) {
+        // Ignore cleanup errors
+      }
+    }
+
+    return {
+      success: false,
+      message: error.message || "Internal server error",
+    };
+  }
+}
+
+export { handleSeedUpload, normalizeSeedUpload, handleSeedUploadDirect };

package/src/ui/express-app.js

@@ -0,0 +1,182 @@
+import express from "express";
+import path from "path";
+import { fileURLToPath } from "url";
+import { sseRegistry } from "./sse.js";
+import { handleApiState } from "./endpoints/state-endpoint.js";
+import { handleSeedUpload } from "./endpoints/upload-endpoints.js";
+import {
+  handleJobListRequest,
+  handleJobDetailRequest,
+} from "./endpoints/job-endpoints.js";
+import {
+  handleJobRescan,
+  handleJobRestart,
+  handleJobStop,
+  handleTaskStart,
+} from "./endpoints/job-control-endpoints.js";
+import {
+  handleTaskFileListRequest,
+  handleTaskFileRequest,
+} from "./endpoints/file-endpoints.js";
+import { sendJson } from "./utils/http-utils.js";
+import { PROVIDER_FUNCTIONS } from "../config/models.js";
+
+// Get __dirname equivalent in ES modules
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+/**
+ * Build Express application with API routes, SSE, and static serving
+ * @param {Object} params - Configuration parameters
+ * @param {string} params.dataDir - Base data directory
+ * @param {Object} [params.viteServer] - Vite dev server instance (optional)
+ * @returns {express.Application} Configured Express app
+ */
+export function buildExpressApp({ dataDir, viteServer }) {
+  const app = express();
+
+  // API guard middleware mounted on /api
+  app.use("/api", (req, res, next) => {
+    // Set CORS headers
+    res.setHeader("Access-Control-Allow-Origin", "*");
+    res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
+    res.setHeader("Access-Control-Allow-Headers", "Content-Type");
+
+    // Handle OPTIONS preflight
+    if (req.method === "OPTIONS") {
+      return res.status(204).end();
+    }
+
+    // Set Connection: close for non-SSE requests
+    const isSSE = req.path === "/events" || req.path === "/sse";
+    if (!isSSE) {
+      res.setHeader("Connection", "close");
+    }
+
+    next();
+  });
+
+  // SSE routes
+  app.get(["/api/events", "/api/sse"], (req, res) => {
+    // Set SSE headers
+    res.setHeader("Content-Type", "text/event-stream");
+    res.setHeader("Cache-Control", "no-cache");
+    res.setHeader("Connection", "keep-alive");
+    res.setHeader("X-Accel-Buffering", "no");
+
+    // Flush headers if available
+    if (typeof res.flushHeaders === "function") {
+      res.flushHeaders();
+    }
+
+    const jobId = req.query.jobId;
+    sseRegistry.addClient(res, { jobId });
+
+    req.on("close", () => {
+      sseRegistry.removeClient(res);
+    });
+  });
+
+  // REST routes
+
+  // GET /api/state
+  app.get("/api/state", async (req, res) => {
+    await handleApiState(req, res);
+  });
+
+  // POST /api/upload/seed
+  app.post("/api/upload/seed", async (req, res) => {
+    await handleSeedUpload(req, res);
+  });
+
+  // GET /api/llm/functions
+  app.get("/api/llm/functions", (req, res) => {
+    try {
+      sendJson(res, 200, { ok: true, data: PROVIDER_FUNCTIONS });
+    } catch (error) {
+      console.error("Error serving LLM functions:", error);
+      sendJson(res, 500, {
+        ok: false,
+        error: "internal_error",
+        message: "Failed to load LLM functions",
+      });
+    }
+  });
+
+  // GET /api/jobs
+  app.get("/api/jobs", async (req, res) => {
+    await handleJobListRequest(req, res);
+  });
+
+  // GET /api/jobs/:jobId
+  app.get("/api/jobs/:jobId", async (req, res) => {
+    const { jobId } = req.params;
+    await handleJobDetailRequest(req, res, jobId);
+  });
+
+  // POST /api/jobs/:jobId/rescan
+  app.post("/api/jobs/:jobId/rescan", async (req, res) => {
+    const { jobId } = req.params;
+    await handleJobRescan(req, res, jobId, dataDir, sendJson);
+  });
+
+  // POST /api/jobs/:jobId/restart
+  app.post("/api/jobs/:jobId/restart", async (req, res) => {
+    const { jobId } = req.params;
+    await handleJobRestart(req, res, jobId, dataDir, sendJson);
+  });
+
+  // POST /api/jobs/:jobId/stop
+  app.post("/api/jobs/:jobId/stop", async (req, res) => {
+    const { jobId } = req.params;
+    await handleJobStop(req, res, jobId, dataDir, sendJson);
+  });
+
+  // POST /api/jobs/:jobId/tasks/:taskId/start
+  app.post("/api/jobs/:jobId/tasks/:taskId/start", async (req, res) => {
+    const { jobId, taskId } = req.params;
+    await handleTaskStart(req, res, jobId, taskId, dataDir, sendJson);
+  });
+
+  // GET /api/jobs/:jobId/tasks/:taskId/files
+  app.get("/api/jobs/:jobId/tasks/:taskId/files", async (req, res) => {
+    const { jobId, taskId } = req.params;
+    const { type } = req.query;
+    await handleTaskFileListRequest(req, res, {
+      jobId,
+      taskId,
+      type,
+      dataDir,
+    });
+  });
+
+  // GET /api/jobs/:jobId/tasks/:taskId/file
+  app.get("/api/jobs/:jobId/tasks/:taskId/file", async (req, res) => {
+    const { jobId, taskId } = req.params;
+    const { type, filename } = req.query;
+    await handleTaskFileRequest(req, res, {
+      jobId,
+      taskId,
+      type,
+      filename,
+      dataDir,
+    });
+  });
+
+  // Dev middleware (mount after all API routes)
+  if (viteServer && viteServer.middlewares) {
+    app.use(viteServer.middlewares);
+  } else {
+    // Production static serving
+    app.use("/public", express.static(path.join(__dirname, "public")));
+    app.use("/assets", express.static(path.join(__dirname, "dist", "assets")));
+    app.use(express.static(path.join(__dirname, "dist")));
+
+    // SPA fallback
+    app.get("*", (_, res) => {
+      res.sendFile(path.join(__dirname, "dist", "index.html"));
+    });
+  }
+
+  return app;
+}