@ryanfw/prompt-orchestration-pipeline 0.9.1 → 0.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ryanfw/prompt-orchestration-pipeline",
-  "version": "0.9.1",
+  "version": "0.11.0",
   "description": "A Prompt-orchestration pipeline (POP) is a framework for building, running, and experimenting with complex chains of LLM tasks.",
   "type": "module",
   "main": "src/ui/server.js",
@@ -43,6 +43,7 @@
     "chokidar": "^3.5.3",
     "commander": "^14.0.2",
     "dotenv": "^17.2.3",
+    "fflate": "^0.8.2",
     "lucide-react": "^0.544.0",
     "openai": "^5.23.1",
     "react": "^19.2.0",

package/src/api/index.js

@@ -12,6 +12,7 @@ import {
   getJobPipelinePath,
 } from "../config/paths.js";
 import { generateJobId } from "../utils/id-generator.js";
+import { initializeJobArtifacts } from "../core/status-writer.js";
 
 // Pure functional utilities
 const createPaths = (config) => {
@@ -100,17 +101,28 @@ export const submitJob = async (state, seed) => {
  * @param {Object} options - Options object
  * @param {string} options.dataDir - Base data directory
  * @param {Object} options.seedObject - Seed object to submit
+ * @param {Array} [options.uploadArtifacts] - Array of {filename, content} objects
  * @returns {Promise<Object>} Result object with success status
  */
-export const submitJobWithValidation = async ({ dataDir, seedObject }) => {
+export const submitJobWithValidation = async ({
+  dataDir,
+  seedObject,
+  uploadArtifacts = [],
+}) => {
   let partialFiles = [];
 
   try {
     // Validate the seed object
+    console.log("[DEBUG] submitJobWithValidation: validating seed", {
+      seedName: seedObject.name,
+      seedPipeline: seedObject.pipeline,
+      hasData: !!seedObject.data,
+    });
     const validatedSeed = await validateSeed(
       JSON.stringify(seedObject),
       dataDir
     );
+    console.log("[DEBUG] submitJobWithValidation: seed validation passed");
 
     // Generate a random job ID
     const jobId = generateJobId();
@@ -175,6 +187,24 @@ export const submitJobWithValidation = async ({ dataDir, seedObject }) => {
       JSON.stringify(pipelineSnapshot, null, 2)
     );
 
+    // Initialize job artifacts if any provided
+    if (uploadArtifacts.length > 0) {
+      console.log("[DEBUG] submitJobWithValidation: initializing artifacts", {
+        artifactCount: uploadArtifacts.length,
+        artifactNames: uploadArtifacts.map((a) => a.filename),
+        currentJobDir,
+      });
+      try {
+        await initializeJobArtifacts(currentJobDir, uploadArtifacts);
+        console.log(
+          "[DEBUG] submitJobWithValidation: artifacts initialized successfully"
+        );
+      } catch (artifactError) {
+        // Don't fail the upload if artifact initialization fails, just log the error
+        console.error("Failed to initialize job artifacts:", artifactError);
+      }
+    }
+
     return {
       success: true,
       jobId,
@@ -199,6 +229,13 @@ export const submitJobWithValidation = async ({ dataDir, seedObject }) => {
       errorMessage = "Required fields missing";
     }
 
+    console.error("[DEBUG] submitJobWithValidation: validation failed", {
+      errorMessage,
+      originalError: error.message,
+      seedName: seedObject.name,
+      seedPipeline: seedObject.pipeline,
+    });
+
     return {
       success: false,
       message: errorMessage,

package/src/components/DAGGrid.jsx

@@ -87,6 +87,22 @@ const canShowRestart = (status) => {
   return status === TaskState.FAILED || status === TaskState.DONE;
 };
 
+// Custom comparison function for TaskCard memoization
+const areEqualTaskCardProps = (prevProps, nextProps) => {
+  return (
+    prevProps.item === nextProps.item &&
+    prevProps.idx === nextProps.idx &&
+    prevProps.status === nextProps.status &&
+    prevProps.isActive === nextProps.isActive &&
+    prevProps.canRestart === nextProps.canRestart &&
+    prevProps.isSubmitting === nextProps.isSubmitting &&
+    prevProps.disabledReason === nextProps.disabledReason &&
+    prevProps.onClick === nextProps.onClick &&
+    prevProps.onKeyDown === nextProps.onKeyDown &&
+    prevProps.handleRestartClick === nextProps.handleRestartClick
+  );
+};
+
 // Memoized card component to prevent unnecessary re-renders
 const TaskCard = memo(function TaskCard({
   item,
@@ -96,7 +112,7 @@ const TaskCard = memo(function TaskCard({
   isActive,
   canRestart,
   isSubmitting,
-  getRestartDisabledReason,
+  disabledReason,
   onClick,
   onKeyDown,
   handleRestartClick,
@@ -190,9 +206,7 @@ const TaskCard = memo(function TaskCard({
               disabled={!canRestart || isSubmitting}
               className="text-xs cursor-pointer disabled:cursor-not-allowed"
               title={
-                !canRestart
-                  ? getRestartDisabledReason()
-                  : `Restart job from ${item.id}`
+                !canRestart ? disabledReason : `Restart job from ${item.id}`
               }
             >
               Restart
@@ -202,7 +216,7 @@ const TaskCard = memo(function TaskCard({
       </div>
     </div>
   );
-});
+}, areEqualTaskCardProps);
 
 /**
  * DAGGrid component for visualizing pipeline tasks with connectors and slide-over details
@@ -222,6 +236,7 @@ function DAGGrid({
   activeIndex = 0,
   jobId,
   filesByTypeForItem = () => createEmptyTaskFiles(),
+  taskById = {},
 }) {
   const overlayRef = useRef(null);
   const gridRef = useRef(null);
@@ -441,7 +456,7 @@ function DAGGrid({
         cancelAnimationFrame(rafRef.current);
       }
     };
-  }, [items, effectiveCols, visualOrder]);
+  }, [items.length, effectiveCols, visualOrder]);
 
   // Get status for a given item index with fallback to activeIndex
   const getStatus = (index) => {
@@ -679,6 +694,7 @@ function DAGGrid({
           const status = getStatus(idx);
           const isActive = idx === activeIndex;
           const canRestart = isRestartEnabled();
+          const restartDisabledReason = getRestartDisabledReason();
 
           return (
             <TaskCard
@@ -689,7 +705,7 @@ function DAGGrid({
               isActive={isActive}
               canRestart={canRestart}
               isSubmitting={isSubmitting}
-              getRestartDisabledReason={getRestartDisabledReason}
+              disabledReason={restartDisabledReason}
               onClick={() => {
                 setOpenIdx(idx);
               }}
@@ -715,6 +731,7 @@ function DAGGrid({
           jobId={jobId}
           taskId={items[openIdx]?.id || `task-${openIdx}`}
           taskBody={items[openIdx]?.body || null}
+          taskError={taskById[items[openIdx]?.id]?.error || null}
           filesByTypeForItem={filesByTypeForItem}
           task={items[openIdx]}
           taskIndex={openIdx}

package/src/components/JobDetail.jsx

@@ -129,6 +129,16 @@ export default function JobDetail({ job, pipeline }) {
       return item;
     });
 
+    // Check if all entries were reused and lengths match
+    const allReused = newItems.every(
+      (item, index) => item === prevItems[index]
+    );
+
+    if (allReused && prevItems.length === newItems.length) {
+      // All items reused, preserve array reference
+      return prevItems;
+    }
+
     prevDagItemsRef.current = newItems;
     return newItems;
   }, [stableDagItems]);
@@ -156,6 +166,7 @@ export default function JobDetail({ job, pipeline }) {
         activeIndex={activeIndex}
         jobId={job.id}
         filesByTypeForItem={filesByTypeForItem}
+        taskById={taskById}
       />
     </div>
   );

package/src/components/TaskDetailSidebar.jsx

@@ -23,6 +23,7 @@ export function TaskDetailSidebar({
   jobId,
   taskId,
   taskBody,
+  taskError,
   filesByTypeForItem = () => ({ artifacts: [], logs: [], tmp: [] }),
   task,
   onClose,
@@ -32,6 +33,7 @@ export function TaskDetailSidebar({
   const [filePaneType, setFilePaneType] = useState("artifacts");
   const [filePaneOpen, setFilePaneOpen] = useState(false);
   const [filePaneFilename, setFilePaneFilename] = useState(null);
+  const [showStack, setShowStack] = useState(false);
   const closeButtonRef = useRef(null);
 
   // Get CSS classes for card header based on status (mirrored from DAGGrid)
@@ -120,14 +122,36 @@ export function TaskDetailSidebar({
       </div>
 
       <div className="p-6 space-y-8 overflow-y-auto h-full">
-        {/* Error Callout - shown when task has error status and body */}
-        {status === TaskState.FAILED && taskBody && (
+        {/* Error Callout - shown when task has error status */}
+        {status === TaskState.FAILED && (taskError?.message || taskBody) && (
           <section aria-label="Error">
             <Callout.Root role="alert" aria-live="assertive">
               <Callout.Text className="whitespace-pre-wrap break-words">
-                {taskBody}
+                {taskError?.message || taskBody}
               </Callout.Text>
             </Callout.Root>
+
+            {/* Stack trace toggle */}
+            {taskError?.stack && (
+              <div className="mt-3">
+                <button
+                  onClick={() => setShowStack(!showStack)}
+                  className="text-sm text-blue-600 hover:text-blue-800 underline"
+                  aria-expanded={showStack}
+                  aria-controls="error-stack"
+                >
+                  {showStack ? "Hide stack" : "Show stack"}
+                </button>
+                {showStack && (
+                  <pre
+                    id="error-stack"
+                    className="mt-2 p-2 bg-gray-50 border rounded text-xs font-mono max-h-64 overflow-auto whitespace-pre-wrap"
+                  >
+                    {taskError.stack}
+                  </pre>
+                )}
+              </div>
+            )}
           </section>
         )}
 

package/src/components/UploadSeed.jsx

@@ -152,14 +152,14 @@ export default function UploadSeed({ onUploadSuccess }) {
             <span className="font-medium text-gray-900">Click to upload</span>{" "}
             or drag and drop
           </div>
-          <p className="text-xs text-gray-500">JSON files only</p>
+          <p className="text-xs text-gray-500">JSON or zip files only</p>
         </div>
       </div>
 
       <input
         ref={fileInputRef}
         type="file"
-        accept=".json"
+        accept=".json,.zip"
         className="hidden"
         onChange={handleFileChange}
         data-testid="file-input"

package/src/config/log-events.js

@@ -0,0 +1,77 @@
+/**
+ * Canonical log event constants and file extensions for the prompt orchestration pipeline.
+ * This module serves as the single source of truth for all log-related naming conventions.
+ */
+
+// Log event types for different stages and events in the pipeline
+export const LogEvent = Object.freeze({
+  START: "start",
+  COMPLETE: "complete",
+  ERROR: "error",
+  CONTEXT: "context",
+  DEBUG: "debug",
+  METRICS: "metrics",
+  PIPELINE_START: "pipeline-start",
+  PIPELINE_COMPLETE: "pipeline-complete",
+  PIPELINE_ERROR: "pipeline-error",
+  EXECUTION_LOGS: "execution-logs",
+  FAILURE_DETAILS: "failure-details",
+});
+
+// File extensions for different log types
+export const LogFileExtension = Object.freeze({
+  TEXT: "log",
+  JSON: "json",
+});
+
+// Validation sets for ensuring consistency
+export const VALID_LOG_EVENTS = new Set(Object.values(LogEvent));
+export const VALID_LOG_FILE_EXTENSIONS = new Set(
+  Object.values(LogFileExtension)
+);
+
+/**
+ * Validates a log event string.
+ * @param {string} event - Log event to validate
+ * @returns {boolean} True if valid, false otherwise
+ */
+export function isValidLogEvent(event) {
+  return VALID_LOG_EVENTS.has(event);
+}
+
+/**
+ * Validates a log file extension string.
+ * @param {string} ext - File extension to validate
+ * @returns {boolean} True if valid, false otherwise
+ */
+export function isValidLogFileExtension(ext) {
+  return VALID_LOG_FILE_EXTENSIONS.has(ext);
+}
+
+/**
+ * Normalizes a log event string to canonical form.
+ * @param {string} event - Raw log event
+ * @returns {string|null} Canonical log event or null if invalid
+ */
+export function normalizeLogEvent(event) {
+  if (typeof event !== "string") {
+    return null;
+  }
+
+  const normalized = event.toLowerCase().trim();
+  return isValidLogEvent(normalized) ? normalized : null;
+}
+
+/**
+ * Normalizes a log file extension string to canonical form.
+ * @param {string} ext - Raw file extension
+ * @returns {string|null} Canonical file extension or null if invalid
+ */
+export function normalizeLogFileExtension(ext) {
+  if (typeof ext !== "string") {
+    return null;
+  }
+
+  const normalized = ext.toLowerCase().trim().replace(/^\./, "");
+  return isValidLogFileExtension(normalized) ? normalized : null;
+}

package/src/core/file-io.js

@@ -1,6 +1,13 @@
 import fs from "node:fs/promises";
+import fsSync from "node:fs";
 import path from "node:path";
 import { writeJobStatus } from "./status-writer.js";
+import {
+  LogEvent,
+  LogFileExtension,
+  isValidLogEvent,
+  isValidLogFileExtension,
+} from "../config/log-events.js";
 
 /**
  * Creates a task-scoped file I/O interface that manages file operations
@@ -18,7 +25,17 @@ async function ensureDir(dir) {
   await fs.mkdir(dir, { recursive: true });
 }
 
-export function createTaskFileIO({ workDir, taskName, getStage, statusPath }) {
+function ensureDirSync(dir) {
+  fsSync.mkdir(dir, { recursive: true });
+}
+
+export function createTaskFileIO({
+  workDir,
+  taskName,
+  getStage,
+  statusPath,
+  trackTaskFiles = true,
+}) {
   const taskDir = path.join(workDir, "tasks", taskName);
 
   // New directory structure: {workDir}/files/{type}
@@ -34,18 +51,21 @@ export function createTaskFileIO({ workDir, taskName, getStage, statusPath }) {
     const jobDir = path.dirname(statusPath);
     await writeJobStatus(jobDir, (snapshot) => {
       snapshot.files ||= { artifacts: [], logs: [], tmp: [] };
-      snapshot.tasks ||= {};
-      snapshot.tasks[taskName] ||= {};
-      snapshot.tasks[taskName].files ||= { artifacts: [], logs: [], tmp: [] };
 
       const jobArray = snapshot.files[fileType];
       if (!jobArray.includes(fileName)) {
         jobArray.push(fileName);
       }
 
-      const taskArray = snapshot.tasks[taskName].files[fileType];
-      if (!taskArray.includes(fileName)) {
-        taskArray.push(fileName);
+      if (trackTaskFiles) {
+        snapshot.tasks ||= {};
+        snapshot.tasks[taskName] ||= {};
+        snapshot.tasks[taskName].files ||= { artifacts: [], logs: [], tmp: [] };
+
+        const taskArray = snapshot.tasks[taskName].files[fileType];
+        if (!taskArray.includes(fileName)) {
+          taskArray.push(fileName);
+        }
       }
 
       return snapshot;
@@ -61,6 +81,15 @@ export function createTaskFileIO({ workDir, taskName, getStage, statusPath }) {
     await fs.rename(tmpPath, filePath);
   }
 
+  /**
+   * Synchronous atomic write helper
+   */
+  function atomicWriteSync(filePath, data) {
+    const tmpPath = filePath + ".tmp";
+    fsSync.writeFileSync(tmpPath, data);
+    fsSync.renameSync(tmpPath, filePath);
+  }
+
   /**
    * Generic write function that handles different modes
    */
@@ -85,6 +114,54 @@ export function createTaskFileIO({ workDir, taskName, getStage, statusPath }) {
     return await fs.readFile(filePath, "utf8");
   }
 
+  /**
+   * Synchronous status writer for critical paths
+   * @param {string} jobDir - Directory containing tasks-status.json
+   * @param {Function} updater - Function that mutates and returns the snapshot
+   */
+  function writeJobStatusSync(jobDir, updater) {
+    const statusPath = path.join(jobDir, "tasks-status.json");
+    let snapshot;
+    try {
+      const raw = fsSync.readFileSync(statusPath, "utf8");
+      snapshot = JSON.parse(raw);
+    } catch {
+      snapshot = { files: { artifacts: [], logs: [], tmp: [] }, tasks: {} };
+    }
+    const updated = updater(snapshot);
+    fsSync.writeFileSync(statusPath, JSON.stringify(updated, null, 2));
+  }
+
+  /**
+   * Synchronous status update with file tracking and metadata
+   * @param {string} fileType - "logs", "artifacts", or "tmp"
+   * @param {string} fileName - Name of the file
+   */
+  function updateStatusWithFilesSync(fileType, fileName) {
+    const jobDir = path.dirname(statusPath);
+    writeJobStatusSync(jobDir, (snapshot) => {
+      snapshot.files ||= { artifacts: [], logs: [], tmp: [] };
+
+      const jobArray = snapshot.files[fileType];
+      if (!jobArray.includes(fileName)) {
+        jobArray.push(fileName);
+      }
+
+      if (trackTaskFiles) {
+        snapshot.tasks ||= {};
+        snapshot.tasks[taskName] ||= {};
+        snapshot.tasks[taskName].files ||= { artifacts: [], logs: [], tmp: [] };
+
+        const taskArray = snapshot.tasks[taskName].files[fileType];
+        if (!taskArray.includes(fileName)) {
+          taskArray.push(fileName);
+        }
+      }
+
+      return snapshot;
+    });
+  }
+
   // Return curried functions for each file type
   return {
     /**
@@ -113,6 +190,12 @@ export function createTaskFileIO({ workDir, taskName, getStage, statusPath }) {
      * @param {string} options.mode - "append" (default) or "replace"
      */
     async writeLog(name, content, options = {}) {
+      if (!validateLogName(name)) {
+        throw new Error(
+          `Invalid log filename "${name}". Must follow format {taskName}-{stage}-{event}.{ext}`
+        );
+      }
+
       const filePath = await writeFile(
         logsDir,
         name,
@@ -176,6 +259,33 @@ export function createTaskFileIO({ workDir, taskName, getStage, statusPath }) {
       return taskDir;
     },
 
+    /**
+     * Write a log file synchronously (critical path only)
+     * @param {string} name - File name
+     * @param {string} content - Log content
+     * @param {Object} options - Options object
+     * @param {string} options.mode - "replace" (default) or "append"
+     */
+    writeLogSync(name, content, options = {}) {
+      if (!validateLogName(name)) {
+        throw new Error(
+          `Invalid log filename "${name}". Must follow format {taskName}-{stage}-{event}.{ext}`
+        );
+      }
+
+      ensureDirSync(logsDir);
+      const filePath = path.join(logsDir, name);
+
+      if (options.mode === "append") {
+        fsSync.appendFileSync(filePath, content);
+      } else {
+        atomicWriteSync(filePath, content);
+      }
+
+      updateStatusWithFilesSync("logs", name);
+      return filePath;
+    },
+
     /**
      * Get the current stage name
      * @returns {string} Current stage name
@@ -185,3 +295,88 @@ export function createTaskFileIO({ workDir, taskName, getStage, statusPath }) {
     },
   };
 }
+
+/**
+ * Generates a standardized log filename following the convention {taskName}-{stage}-{event}.{ext}
+ * @param {string} taskName - Name of the task
+ * @param {string} stage - Stage name or identifier
+ * @param {string} event - Event type from LogEvent constants
+ * @param {string} ext - File extension from LogFileExtension constants
+ * @returns {string} Formatted log filename
+ */
+export function generateLogName(
+  taskName,
+  stage,
+  event,
+  ext = LogFileExtension.TEXT
+) {
+  if (!taskName || !stage || !event || !ext) {
+    throw new Error(
+      "All parameters (taskName, stage, event, ext) are required for generateLogName"
+    );
+  }
+  if (!isValidLogEvent(event)) {
+    throw new Error(
+      `Invalid log event "${event}". Use a value from LogEvent: ${Object.values(
+        LogEvent
+      ).join(", ")}`
+    );
+  }
+  if (!isValidLogFileExtension(ext)) {
+    throw new Error(
+      `Invalid log file extension "${ext}". Use a value from LogFileExtension: ${Object.values(
+        LogFileExtension
+      ).join(", ")}`
+    );
+  }
+  return `${taskName}-${stage}-${event}.${ext}`;
+}
+
+/**
+ * Parses a log filename to extract taskName, stage, event, and extension
+ * @param {string} fileName - Log filename to parse
+ * @returns {Object|null} Parsed components or null if invalid format
+ */
+export function parseLogName(fileName) {
+  if (typeof fileName !== "string") {
+    return null;
+  }
+
+  // Match pattern: taskName-stage-event.ext
+  // Split on first two hyphens: taskName-stage-event.ext
+  const match = fileName.match(
+    /^(?<taskName>[^-]+)-(?<stage>[^-]+)-(?<event>[^.]+)\.(?<ext>.+)$/
+  );
+  if (!match) {
+    return null;
+  }
+
+  const { taskName, stage, event, ext } = match.groups;
+  return { taskName, stage, event, ext };
+}
+
+/**
+ * Generates a glob pattern for matching log files with specific components
+ * @param {string} taskName - Task name (optional, use "*" for wildcard)
+ * @param {string} stage - Stage name (optional, use "*" for wildcard)
+ * @param {string} event - Event type (optional, use "*" for wildcard)
+ * @param {string} ext - File extension (optional, use "*" for wildcard)
+ * @returns {string} Glob pattern for file matching
+ */
+export function getLogPattern(
+  taskName = "*",
+  stage = "*",
+  event = "*",
+  ext = "*"
+) {
+  return `${taskName}-${stage}-${event}.${ext}`;
+}
+
+/**
+ * Validates that a log filename follows the standardized naming convention
+ * @param {string} fileName - Log filename to validate
+ * @returns {boolean} True if valid, false otherwise
+ */
+export function validateLogName(fileName) {
+  return parseLogName(fileName) !== null;
+}