@ryanfw/prompt-orchestration-pipeline 0.10.0 → 0.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ryanfw/prompt-orchestration-pipeline",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "A Prompt-orchestration pipeline (POP) is a framework for building, running, and experimenting with complex chains of LLM tasks.",
   "type": "module",
   "main": "src/ui/server.js",
@@ -43,6 +43,7 @@
     "chokidar": "^3.5.3",
     "commander": "^14.0.2",
     "dotenv": "^17.2.3",
+    "fflate": "^0.8.2",
     "lucide-react": "^0.544.0",
     "openai": "^5.23.1",
     "react": "^19.2.0",

package/src/api/index.js

@@ -12,6 +12,7 @@ import {
   getJobPipelinePath,
 } from "../config/paths.js";
 import { generateJobId } from "../utils/id-generator.js";
+import { initializeJobArtifacts } from "../core/status-writer.js";
 
 // Pure functional utilities
 const createPaths = (config) => {
@@ -100,17 +101,28 @@ export const submitJob = async (state, seed) => {
  * @param {Object} options - Options object
  * @param {string} options.dataDir - Base data directory
  * @param {Object} options.seedObject - Seed object to submit
+ * @param {Array} [options.uploadArtifacts] - Array of {filename, content} objects
  * @returns {Promise<Object>} Result object with success status
  */
-export const submitJobWithValidation = async ({ dataDir, seedObject }) => {
+export const submitJobWithValidation = async ({
+  dataDir,
+  seedObject,
+  uploadArtifacts = [],
+}) => {
   let partialFiles = [];
 
   try {
     // Validate the seed object
+    console.log("[DEBUG] submitJobWithValidation: validating seed", {
+      seedName: seedObject.name,
+      seedPipeline: seedObject.pipeline,
+      hasData: !!seedObject.data,
+    });
     const validatedSeed = await validateSeed(
       JSON.stringify(seedObject),
       dataDir
     );
+    console.log("[DEBUG] submitJobWithValidation: seed validation passed");
 
     // Generate a random job ID
     const jobId = generateJobId();
@@ -175,6 +187,24 @@ export const submitJobWithValidation = async ({ dataDir, seedObject }) => {
       JSON.stringify(pipelineSnapshot, null, 2)
     );
 
+    // Initialize job artifacts if any provided
+    if (uploadArtifacts.length > 0) {
+      console.log("[DEBUG] submitJobWithValidation: initializing artifacts", {
+        artifactCount: uploadArtifacts.length,
+        artifactNames: uploadArtifacts.map((a) => a.filename),
+        currentJobDir,
+      });
+      try {
+        await initializeJobArtifacts(currentJobDir, uploadArtifacts);
+        console.log(
+          "[DEBUG] submitJobWithValidation: artifacts initialized successfully"
+        );
+      } catch (artifactError) {
+        // Don't fail the upload if artifact initialization fails, just log the error
+        console.error("Failed to initialize job artifacts:", artifactError);
+      }
+    }
+
     return {
       success: true,
       jobId,
@@ -199,6 +229,13 @@ export const submitJobWithValidation = async ({ dataDir, seedObject }) => {
       errorMessage = "Required fields missing";
     }
 
+    console.error("[DEBUG] submitJobWithValidation: validation failed", {
+      errorMessage,
+      originalError: error.message,
+      seedName: seedObject.name,
+      seedPipeline: seedObject.pipeline,
+    });
+
     return {
       success: false,
       message: errorMessage,

package/src/components/DAGGrid.jsx

@@ -87,6 +87,22 @@ const canShowRestart = (status) => {
   return status === TaskState.FAILED || status === TaskState.DONE;
 };
 
+// Custom comparison function for TaskCard memoization
+const areEqualTaskCardProps = (prevProps, nextProps) => {
+  return (
+    prevProps.item === nextProps.item &&
+    prevProps.idx === nextProps.idx &&
+    prevProps.status === nextProps.status &&
+    prevProps.isActive === nextProps.isActive &&
+    prevProps.canRestart === nextProps.canRestart &&
+    prevProps.isSubmitting === nextProps.isSubmitting &&
+    prevProps.disabledReason === nextProps.disabledReason &&
+    prevProps.onClick === nextProps.onClick &&
+    prevProps.onKeyDown === nextProps.onKeyDown &&
+    prevProps.handleRestartClick === nextProps.handleRestartClick
+  );
+};
+
 // Memoized card component to prevent unnecessary re-renders
 const TaskCard = memo(function TaskCard({
   item,
@@ -96,7 +112,7 @@ const TaskCard = memo(function TaskCard({
   isActive,
   canRestart,
   isSubmitting,
-  getRestartDisabledReason,
+  disabledReason,
   onClick,
   onKeyDown,
   handleRestartClick,
@@ -190,9 +206,7 @@ const TaskCard = memo(function TaskCard({
               disabled={!canRestart || isSubmitting}
               className="text-xs cursor-pointer disabled:cursor-not-allowed"
               title={
-                !canRestart
-                  ? getRestartDisabledReason()
-                  : `Restart job from ${item.id}`
+                !canRestart ? disabledReason : `Restart job from ${item.id}`
               }
             >
               Restart
@@ -202,7 +216,7 @@ const TaskCard = memo(function TaskCard({
       </div>
     </div>
   );
-});
+}, areEqualTaskCardProps);
 
 /**
  * DAGGrid component for visualizing pipeline tasks with connectors and slide-over details
@@ -222,6 +236,7 @@ function DAGGrid({
   activeIndex = 0,
   jobId,
   filesByTypeForItem = () => createEmptyTaskFiles(),
+  taskById = {},
 }) {
   const overlayRef = useRef(null);
   const gridRef = useRef(null);
@@ -441,7 +456,7 @@ function DAGGrid({
         cancelAnimationFrame(rafRef.current);
       }
     };
-  }, [items, effectiveCols, visualOrder]);
+  }, [items.length, effectiveCols, visualOrder]);
 
   // Get status for a given item index with fallback to activeIndex
   const getStatus = (index) => {
@@ -679,6 +694,7 @@ function DAGGrid({
           const status = getStatus(idx);
           const isActive = idx === activeIndex;
           const canRestart = isRestartEnabled();
+          const restartDisabledReason = getRestartDisabledReason();
 
           return (
             <TaskCard
@@ -689,7 +705,7 @@ function DAGGrid({
               isActive={isActive}
               canRestart={canRestart}
               isSubmitting={isSubmitting}
-              getRestartDisabledReason={getRestartDisabledReason}
+              disabledReason={restartDisabledReason}
               onClick={() => {
                 setOpenIdx(idx);
               }}
@@ -715,6 +731,7 @@ function DAGGrid({
           jobId={jobId}
           taskId={items[openIdx]?.id || `task-${openIdx}`}
           taskBody={items[openIdx]?.body || null}
+          taskError={taskById[items[openIdx]?.id]?.error || null}
           filesByTypeForItem={filesByTypeForItem}
           task={items[openIdx]}
           taskIndex={openIdx}

package/src/components/JobDetail.jsx

@@ -129,6 +129,16 @@ export default function JobDetail({ job, pipeline }) {
       return item;
     });
 
+    // Check if all entries were reused and lengths match
+    const allReused = newItems.every(
+      (item, index) => item === prevItems[index]
+    );
+
+    if (allReused && prevItems.length === newItems.length) {
+      // All items reused, preserve array reference
+      return prevItems;
+    }
+
     prevDagItemsRef.current = newItems;
     return newItems;
   }, [stableDagItems]);
@@ -156,6 +166,7 @@ export default function JobDetail({ job, pipeline }) {
         activeIndex={activeIndex}
         jobId={job.id}
         filesByTypeForItem={filesByTypeForItem}
+        taskById={taskById}
       />
     </div>
   );

package/src/components/TaskDetailSidebar.jsx

@@ -23,6 +23,7 @@ export function TaskDetailSidebar({
   jobId,
   taskId,
   taskBody,
+  taskError,
   filesByTypeForItem = () => ({ artifacts: [], logs: [], tmp: [] }),
   task,
   onClose,
@@ -32,6 +33,7 @@ export function TaskDetailSidebar({
   const [filePaneType, setFilePaneType] = useState("artifacts");
   const [filePaneOpen, setFilePaneOpen] = useState(false);
   const [filePaneFilename, setFilePaneFilename] = useState(null);
+  const [showStack, setShowStack] = useState(false);
   const closeButtonRef = useRef(null);
 
   // Get CSS classes for card header based on status (mirrored from DAGGrid)
@@ -120,14 +122,36 @@ export function TaskDetailSidebar({
       </div>
 
       <div className="p-6 space-y-8 overflow-y-auto h-full">
-        {/* Error Callout - shown when task has error status and body */}
-        {status === TaskState.FAILED && taskBody && (
+        {/* Error Callout - shown when task has error status */}
+        {status === TaskState.FAILED && (taskError?.message || taskBody) && (
           <section aria-label="Error">
             <Callout.Root role="alert" aria-live="assertive">
               <Callout.Text className="whitespace-pre-wrap break-words">
-                {taskBody}
+                {taskError?.message || taskBody}
               </Callout.Text>
             </Callout.Root>
+
+            {/* Stack trace toggle */}
+            {taskError?.stack && (
+              <div className="mt-3">
+                <button
+                  onClick={() => setShowStack(!showStack)}
+                  className="text-sm text-blue-600 hover:text-blue-800 underline"
+                  aria-expanded={showStack}
+                  aria-controls="error-stack"
+                >
+                  {showStack ? "Hide stack" : "Show stack"}
+                </button>
+                {showStack && (
+                  <pre
+                    id="error-stack"
+                    className="mt-2 p-2 bg-gray-50 border rounded text-xs font-mono max-h-64 overflow-auto whitespace-pre-wrap"
+                  >
+                    {taskError.stack}
+                  </pre>
+                )}
+              </div>
+            )}
           </section>
         )}
 

package/src/components/UploadSeed.jsx

@@ -152,14 +152,14 @@ export default function UploadSeed({ onUploadSuccess }) {
             <span className="font-medium text-gray-900">Click to upload</span>{" "}
             or drag and drop
           </div>
-          <p className="text-xs text-gray-500">JSON files only</p>
+          <p className="text-xs text-gray-500">JSON or zip files only</p>
         </div>
       </div>
 
       <input
         ref={fileInputRef}
         type="file"
-        accept=".json"
+        accept=".json,.zip"
         className="hidden"
         onChange={handleFileChange}
         data-testid="file-input"

package/src/core/orchestrator.js

@@ -154,6 +154,38 @@ export async function startOrchestrator(opts) {
         tasks: {}, // Initialize empty tasks object for pipeline runner
       };
       await fs.writeFile(statusPath, JSON.stringify(status, null, 2));
+
+      // Initialize status from artifacts if any exist
+      try {
+        const { initializeStatusFromArtifacts } = await import(
+          "./status-initializer.js"
+        );
+        const pipelineConfig = getPipelineConfig(seed?.pipeline || "default");
+        const pipelineSnapshot = JSON.parse(
+          await fs.readFile(pipelineConfig.pipelineJsonPath, "utf8")
+        );
+
+        const applyArtifacts = await initializeStatusFromArtifacts({
+          jobDir: workDir,
+          pipeline: pipelineSnapshot,
+        });
+
+        // Apply artifact initialization to the status
+        const updatedStatus = applyArtifacts(status);
+        await fs.writeFile(statusPath, JSON.stringify(updatedStatus, null, 2));
+
+        logger.log("Initialized status from upload artifacts", {
+          jobId,
+          pipeline: seed?.pipeline,
+          artifactsCount: updatedStatus.files?.artifacts?.length || 0,
+        });
+      } catch (artifactError) {
+        // Don't fail job startup if artifact initialization fails, just log
+        logger.warn("Failed to initialize status from artifacts", {
+          jobId,
+          error: artifactError.message,
+        });
+      }
     }
     // Create fileIO for orchestrator-level logging
     const fileIO = createTaskFileIO({

package/src/core/status-initializer.js

@@ -0,0 +1,155 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Initialize status snapshot from artifacts in the filesystem
+ * @param {Object} options - Options object
+ * @param {string} options.jobDir - Job directory path
+ * @param {Object} options.pipeline - Pipeline configuration object
+ * @returns {Promise<Function>} Function that applies artifact initialization to a snapshot
+ */
+export async function initializeStatusFromArtifacts({ jobDir, pipeline }) {
+  if (!jobDir || typeof jobDir !== "string") {
+    throw new Error("jobDir must be a non-empty string");
+  }
+
+  if (!pipeline || typeof pipeline !== "object") {
+    throw new Error("pipeline must be an object");
+  }
+
+  const artifactsDir = path.join(jobDir, "files", "artifacts");
+  let artifactFilenames = [];
+
+  try {
+    // Read artifacts directory
+    const entries = await fs.readdir(artifactsDir, { withFileTypes: true });
+
+    // Collect filenames for regular files only
+    artifactFilenames = entries
+      .filter((entry) => entry.isFile())
+      .map((entry) => entry.name);
+
+    console.log("[STATUS_INIT] Found artifacts in directory", {
+      artifactsDir,
+      artifactCount: artifactFilenames.length,
+      artifactNames: artifactFilenames,
+    });
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      // Directory doesn't exist, no artifacts to initialize
+      console.log(
+        "[STATUS_INIT] Artifacts directory does not exist, skipping initialization",
+        {
+          artifactsDir,
+        }
+      );
+    } else {
+      console.error("[STATUS_INIT] Failed to read artifacts directory", {
+        artifactsDir,
+        error: error.message,
+      });
+    }
+    // Return a no-op function for non-existent or unreadable directory
+    return (snapshot) => snapshot;
+  }
+
+  // Determine first task ID from pipeline
+  const firstTaskId = Array.isArray(pipeline.tasks) ? pipeline.tasks[0] : null;
+  console.log("[STATUS_INIT] Determined first task", {
+    firstTaskId,
+    hasTasks: Array.isArray(pipeline.tasks),
+    taskCount: pipeline.tasks?.length || 0,
+  });
+
+  // Return function that applies the artifact initialization to a snapshot
+  return function apply(snapshot) {
+    console.log("[STATUS_INIT] Applying artifact initialization to snapshot", {
+      existingArtifacts: snapshot.files?.artifacts?.length || 0,
+      newArtifacts: artifactFilenames.length,
+      firstTaskId,
+    });
+
+    // Ensure files object exists with proper structure
+    if (!snapshot.files || typeof snapshot.files !== "object") {
+      snapshot.files = { artifacts: [], logs: [], tmp: [] };
+    } else {
+      // Ensure each files array exists
+      for (const type of ["artifacts", "logs", "tmp"]) {
+        if (!Array.isArray(snapshot.files[type])) {
+          snapshot.files[type] = [];
+        }
+      }
+    }
+
+    // Add artifact filenames to root level (deduplicate)
+    const existingArtifacts = new Set(snapshot.files.artifacts || []);
+    for (const filename of artifactFilenames) {
+      if (!existingArtifacts.has(filename)) {
+        snapshot.files.artifacts.push(filename);
+        existingArtifacts.add(filename);
+      }
+    }
+
+    // Add artifact filenames to first task (if it exists)
+    if (firstTaskId) {
+      // Ensure tasks object exists
+      if (!snapshot.tasks || typeof snapshot.tasks !== "object") {
+        snapshot.tasks = {};
+      }
+
+      // Ensure first task exists
+      if (!snapshot.tasks[firstTaskId]) {
+        snapshot.tasks[firstTaskId] = {};
+      }
+
+      // Ensure task files object exists with proper structure
+      if (
+        !snapshot.tasks[firstTaskId].files ||
+        typeof snapshot.tasks[firstTaskId].files !== "object"
+      ) {
+        snapshot.tasks[firstTaskId].files = {
+          artifacts: [],
+          logs: [],
+          tmp: [],
+        };
+      } else {
+        // Ensure each task files array exists
+        for (const type of ["artifacts", "logs", "tmp"]) {
+          if (!Array.isArray(snapshot.tasks[firstTaskId].files[type])) {
+            snapshot.tasks[firstTaskId].files[type] = [];
+          }
+        }
+      }
+
+      // Add artifact filenames to first task (deduplicate)
+      const existingTaskArtifacts = new Set(
+        snapshot.tasks[firstTaskId].files.artifacts || []
+      );
+      for (const filename of artifactFilenames) {
+        if (!existingTaskArtifacts.has(filename)) {
+          snapshot.tasks[firstTaskId].files.artifacts.push(filename);
+          existingTaskArtifacts.add(filename);
+        }
+      }
+
+      console.log("[STATUS_INIT] Added artifacts to first task", {
+        firstTaskId,
+        taskArtifactCount: snapshot.tasks[firstTaskId].files.artifacts.length,
+        artifactNames: artifactFilenames,
+      });
+    }
+
+    console.log("[STATUS_INIT] Final snapshot state", {
+      rootArtifacts: snapshot.files.artifacts.length,
+      rootArtifactNames: snapshot.files.artifacts,
+      firstTaskArtifacts: firstTaskId
+        ? snapshot.tasks[firstTaskId].files.artifacts.length
+        : 0,
+      firstTaskArtifactNames: firstTaskId
+        ? snapshot.tasks[firstTaskId].files.artifacts
+        : [],
+    });
+
+    return snapshot;
+  };
+}

package/src/core/status-writer.js

@@ -281,22 +281,71 @@ export async function updateTaskStatus(jobDir, taskId, taskUpdateFn) {
     throw new Error("taskUpdateFn must be a function");
   }
 
-  return writeJobStatus(jobDir, (snapshot) => {
-    // Ensure task exists
-    if (!snapshot.tasks[taskId]) {
-      snapshot.tasks[taskId] = {};
-    }
+  const jobId = path.basename(jobDir);
+  const logger = createJobLogger("StatusWriter", jobId);
 
-    const task = snapshot.tasks[taskId];
+  // Get or create the write queue for this job directory
+  const prev = writeQueues.get(jobDir) || Promise.resolve();
+  let resultSnapshot;
 
-    // Apply task updates
-    const result = taskUpdateFn(task);
-    if (result !== undefined) {
-      snapshot.tasks[taskId] = result;
-    }
+  const next = prev
+    .then(async () => {
+      logger.group("Task Status Update Operation");
+      logger.log(`Updating task ${taskId} for job: ${jobId}`);
 
-    return snapshot;
-  });
+      const statusPath = path.join(jobDir, "tasks-status.json");
+
+      // Read existing status or create default
+      const current = await readStatusFile(statusPath, jobId);
+      const validated = validateStatusSnapshot(current);
+
+      // Ensure task exists
+      if (!validated.tasks[taskId]) {
+        validated.tasks[taskId] = {};
+      }
+
+      const task = validated.tasks[taskId];
+
+      // Apply task updates
+      const result = taskUpdateFn(task);
+      if (result !== undefined) {
+        validated.tasks[taskId] = result;
+      }
+
+      validated.lastUpdated = new Date().toISOString();
+
+      // Atomic write
+      await atomicWrite(statusPath, validated);
+      logger.log("Task status file written successfully");
+
+      // Emit task:updated SSE event after successful write
+      try {
+        const eventData = {
+          jobId,
+          taskId,
+          task: validated.tasks[taskId],
+        };
+        await logger.sse("task:updated", eventData);
+        logger.log("task:updated SSE event broadcasted successfully");
+      } catch (error) {
+        // Don't fail the write if SSE emission fails
+        logger.error("Failed to emit task:updated SSE event:", error);
+      }
+
+      logger.groupEnd();
+      resultSnapshot = validated;
+    })
+    .catch((e) => {
+      throw e;
+    });
+
+  // Store the promise chain and set up cleanup
+  writeQueues.set(
+    jobDir,
+    next.finally(() => {})
+  );
+
+  return next.then(() => resultSnapshot);
 }
 
 /**
@@ -435,3 +484,92 @@ export async function resetJobToCleanSlate(
     return snapshot;
   });
 }
+
+/**
+ * Consolidated path jail security validation with generic error messages
+ * @param {string} filename - Filename to validate
+ * @returns {Object|null} Validation result or null if valid
+ */
+function validateFilePath(filename) {
+  // Check for path traversal patterns
+  if (filename.includes("..")) {
+    console.error("Path security: path traversal detected", { filename });
+    return {
+      allowed: false,
+      message: "Path validation failed",
+    };
+  }
+
+  // Check for absolute paths (POSIX, Windows, backslashes, ~)
+  if (
+    path.isAbsolute(filename) ||
+    /^[a-zA-Z]:/.test(filename) ||
+    filename.includes("\\") ||
+    filename.startsWith("~")
+  ) {
+    console.error("Path security: absolute path detected", { filename });
+    return {
+      allowed: false,
+      message: "Path validation failed",
+    };
+  }
+
+  // Check for empty filename
+  if (!filename || filename.trim() === "") {
+    console.error("Path security: empty filename detected");
+    return {
+      allowed: false,
+      message: "Path validation failed",
+    };
+  }
+
+  // Path is valid
+  return null;
+}
+
+/**
+ * Initialize job-level artifact index and copy artifacts to job directory
+ * @param {string} jobDir - Job directory path
+ * @param {Array} uploadArtifacts - Array of {filename, content} objects
+ * @returns {Promise<void>}
+ */
+export async function initializeJobArtifacts(jobDir, uploadArtifacts = []) {
+  if (!jobDir || typeof jobDir !== "string") {
+    throw new Error("jobDir must be a non-empty string");
+  }
+
+  if (!Array.isArray(uploadArtifacts)) {
+    throw new Error("uploadArtifacts must be an array");
+  }
+
+  if (uploadArtifacts.length === 0) {
+    return;
+  }
+
+  const jobFilesDir = path.join(jobDir, "files");
+  const jobArtifactsDir = path.join(jobFilesDir, "artifacts");
+
+  await fs.mkdir(jobFilesDir, { recursive: true });
+  await fs.mkdir(jobArtifactsDir, { recursive: true });
+
+  for (const artifact of uploadArtifacts) {
+    const { filename, content } = artifact || {};
+
+    if (!filename || typeof filename !== "string") {
+      continue; // Skip invalid entries rather than throwing
+    }
+
+    // Validate filename using the consolidated function
+    const validation = validateFilePath(filename);
+    if (validation) {
+      console.error("Path security: skipping invalid artifact", {
+        filename,
+        reason: validation.message,
+      });
+      continue; // Skip invalid filenames rather than throwing
+    }
+
+    const artifactPath = path.join(jobArtifactsDir, filename);
+    await fs.writeFile(artifactPath, content);
+  }
+}