@ryanfw/prompt-orchestration-pipeline 0.8.0 → 0.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ryanfw/prompt-orchestration-pipeline",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "A Prompt-orchestration pipeline (POP) is a framework for building, running, and experimenting with complex chains of LLM tasks.",
   "type": "module",
   "main": "src/ui/server.js",

package/src/core/logger.js

@@ -0,0 +1,219 @@
+/**
+ * Centralized logging utility for the prompt orchestration pipeline
+ *
+ * Provides consistent, structured, and context-aware logging across all core files
+ * with SSE integration capabilities and multiple log levels.
+ */
+
+// Lazy import SSE registry to avoid circular dependencies
+let sseRegistry = null;
+async function getSSERegistry() {
+  if (!sseRegistry) {
+    try {
+      const module = await import("../ui/sse.js");
+      sseRegistry = module.sseRegistry;
+    } catch (error) {
+      // SSE not available in all environments
+      return null;
+    }
+  }
+  return sseRegistry;
+}
+
+/**
+ * Creates a logger instance with component name and optional context
+ *
+ * @param {string} componentName - Name of the component (e.g., 'Orchestrator', 'TaskRunner')
+ * @param {Object} context - Optional context object (e.g., { jobId, taskName })
+ * @returns {Object} Logger instance with methods for different log levels
+ */
+export function createLogger(componentName, context = {}) {
+  // Build context string for log prefixes
+  const contextParts = [];
+  if (context.jobId) contextParts.push(context.jobId);
+  if (context.taskName) contextParts.push(context.taskName);
+  if (context.stage) contextParts.push(context.stage);
+
+  const contextString =
+    contextParts.length > 0 ? `|${contextParts.join("|")}` : "";
+  const prefix = `[${componentName}${contextString}]`;
+
+  /**
+   * Formats data for consistent JSON output
+   * @param {*} data - Data to format
+   * @returns {string|null} Formatted JSON string or null if data is null/undefined
+   */
+  function formatData(data) {
+    if (data === null || data === undefined) {
+      return null;
+    }
+    if (typeof data === "object") {
+      try {
+        return JSON.stringify(data, null, 2);
+      } catch (error) {
+        return `{ "serialization_error": "${error.message}" }`;
+      }
+    }
+    return data;
+  }
+
+  /**
+   * Broadcasts SSE event if registry is available
+   * @param {string} eventType - Type of SSE event
+   * @param {*} eventData - Data to broadcast
+   */
+  async function broadcastSSE(eventType, eventData) {
+    const registry = await getSSERegistry().catch(() => null);
+    if (registry) {
+      try {
+        const payload = {
+          type: eventType,
+          data: eventData,
+          component: componentName,
+          timestamp: new Date().toISOString(),
+          ...context,
+        };
+        registry.broadcast(payload);
+      } catch (error) {
+        // Don't fail logging if SSE broadcast fails
+        console.warn(
+          `${prefix} Failed to broadcast SSE event: ${error.message}`
+        );
+      }
+    }
+  }
+
+  return {
+    /**
+     * Debug level logging
+     * @param {string} message - Log message
+     * @param {*} data - Optional data to log
+     */
+    debug: (message, data = null) => {
+      if (process.env.NODE_ENV !== "production" || process.env.DEBUG) {
+        console.debug(`${prefix} ${message}`, formatData(data) || "");
+      }
+    },
+
+    /**
+     * Info level logging
+     * @param {string} message - Log message
+     * @param {*} data - Optional data to log
+     */
+    log: (message, data = null) => {
+      console.log(`${prefix} ${message}`, formatData(data) || "");
+    },
+
+    /**
+     * Warning level logging
+     * @param {string} message - Log message
+     * @param {*} data - Optional data to log
+     */
+    warn: (message, data = null) => {
+      console.warn(`${prefix} ${message}`, formatData(data) || "");
+    },
+
+    /**
+     * Error level logging with enhanced error context
+     * @param {string} message - Log message
+     * @param {*} data - Optional data to log
+     */
+    error: (message, data = null) => {
+      let enhancedData = data;
+
+      // Enhance error objects with additional context
+      if (data && data instanceof Error) {
+        enhancedData = {
+          name: data.name,
+          message: data.message,
+          stack: data.stack,
+          component: componentName,
+          timestamp: new Date().toISOString(),
+          ...context,
+        };
+      } else if (
+        data &&
+        typeof data === "object" &&
+        data.error instanceof Error
+      ) {
+        enhancedData = {
+          ...data,
+          error: {
+            name: data.error.name,
+            message: data.error.message,
+            stack: data.error.stack,
+          },
+          component: componentName,
+          timestamp: new Date().toISOString(),
+          ...context,
+        };
+      }
+
+      console.error(`${prefix} ${message}`, formatData(enhancedData) || "");
+    },
+
+    /**
+     * Console group management
+     * @param {string} label - Group label
+     * @param {*} data - Optional data to log
+     */
+    group: (label, data = null) => {
+      console.group(`${prefix} ${label}`);
+      if (data !== null && data !== undefined) {
+        const formattedData = formatData(data);
+        if (formattedData !== null) {
+          console.log(formattedData);
+        }
+      }
+    },
+
+    /**
+     * End console group
+     */
+    groupEnd: () => console.groupEnd(),
+
+    /**
+     * SSE event broadcasting
+     * @param {string} eventType - Type of SSE event
+     * @param {*} eventData - Data to broadcast
+     */
+    sse: async (eventType, eventData) => {
+      // Log SSE broadcast with styling for visibility
+      console.log(
+        `%c${prefix} SSE Broadcast: ${eventType}`,
+        "color: #cc6600; font-weight: bold;",
+        formatData(eventData) || ""
+      );
+
+      await broadcastSSE(eventType, eventData);
+    },
+  };
+}
+
+/**
+ * Creates a logger with job context (convenience function)
+ * @param {string} componentName - Component name
+ * @param {string} jobId - Job ID
+ * @param {Object} additionalContext - Additional context
+ * @returns {Object} Logger instance with job context
+ */
+export function createJobLogger(componentName, jobId, additionalContext = {}) {
+  return createLogger(componentName, { jobId, ...additionalContext });
+}
+
+/**
+ * Creates a logger with task context (convenience function)
+ * @param {string} componentName - Component name
+ * @param {string} jobId - Job ID
+ * @param {string} taskName - Task name
+ * @param {Object} additionalContext - Additional context
+ * @returns {Object} Logger instance with task context
+ */
+export function createTaskLogger(
+  componentName,
+  jobId,
+  taskName,
+  additionalContext = {}
+) {
+  return createLogger(componentName, { jobId, taskName, ...additionalContext });
+}

package/src/core/orchestrator.js

@@ -4,6 +4,7 @@ import path from "node:path";
 import chokidar from "chokidar";
 import { spawn as defaultSpawn } from "node:child_process";
 import { getConfig, getPipelineConfig } from "./config.js";
+import { createLogger } from "./logger.js";
 
 /**
  * Resolve canonical pipeline directories for the given data root.
@@ -70,6 +71,8 @@ export async function startOrchestrator(opts) {
   const watcherFactory = opts?.watcherFactory ?? chokidar.watch;
   const testMode = !!opts?.testMode;
 
+  const logger = createLogger("Orchestrator");
+
   const dirs = resolveDirs(dataDir);
   await ensureDir(dirs.pending);
   await ensureDir(dirs.current);
@@ -94,7 +97,7 @@ export async function startOrchestrator(opts) {
     const base = path.basename(filePath);
     const match = base.match(/^([A-Za-z0-9-_]+)-seed\.json$/);
     if (!match) {
-      console.warn("Rejecting non-id seed file:", base);
+      logger.warn("Rejecting non-id seed file:", { filename: base });
       return;
     }
     const jobId = match[1];
@@ -118,12 +121,16 @@ export async function startOrchestrator(opts) {
     } catch {}
 
     // Move seed to current/{jobId}/seed.json
-    console.log(`[Orchestrator] Moving file from ${filePath} to ${dest}`);
+    logger.log("Moving file", { from: filePath, to: dest });
     try {
       await moveFile(filePath, dest);
-      console.log(`[Orchestrator] ✓ Successfully moved file to ${dest}`);
+      logger.log("Successfully moved file", { destination: dest });
     } catch (error) {
-      console.log(`[Orchestrator] ✗ Failed to move file: ${error.message}`);
+      logger.error("Failed to move file", {
+        from: filePath,
+        to: dest,
+        error: error.message,
+      });
       throw error; // Re-throw to see the actual error
     }
 
@@ -147,14 +154,22 @@ export async function startOrchestrator(opts) {
       await fs.writeFile(statusPath, JSON.stringify(status, null, 2));
     }
     // Spawn runner for this job
-    const child = spawnRunner(jobId, dirs, running, spawn, testMode, seed);
+    const child = spawnRunner(
+      logger,
+      jobId,
+      dirs,
+      running,
+      spawn,
+      testMode,
+      seed
+    );
     // child registered inside spawnRunner
     return child;
   }
 
   // Watch pending directory for seeds
   const watchPattern = path.join(dirs.pending, "*.json");
-  console.log("Orchestrator watching pattern:", watchPattern);
+  logger.log("Watching pattern", { pattern: watchPattern });
   const watcher = watcherFactory(watchPattern, {
     ignoreInitial: false,
     awaitWriteFinish: false, // Disable awaitWriteFinish for faster detection
@@ -164,19 +179,19 @@ export async function startOrchestrator(opts) {
   // Wait for watcher to be ready before resolving
   await new Promise((resolve, reject) => {
     watcher.on("ready", () => {
-      console.log("Orchestrator watcher is ready");
+      logger.log("Watcher is ready");
       resolve();
     });
 
     watcher.on("error", (error) => {
-      console.log("Orchestrator watcher error:", error);
+      logger.error("Watcher error", error);
       reject(error);
     });
   });
 
   watcher.on("add", (file) => {
-    console.log("Orchestrator detected file add:", file);
-    // Return the promise so tests awaiting the add handler block until processing completes
+    logger.log("Detected file add", { file });
+    // Return promise so tests awaiting the add handler block until processing completes
     return handleSeedAdd(file);
   });
 
@@ -212,6 +227,7 @@ export async function startOrchestrator(opts) {
  * Spawn a pipeline runner. In testMode we still call spawn() so tests can assert,
  * but we resolve immediately and let tests drive the lifecycle (emit 'exit', etc.).
  *
+ * @param {Object} logger - Logger instance for orchestrator logging
  * @param {string} jobId
  * @param {{dataDir:string,pending:string,current:string,complete:string}} dirs
  * @param {Map<string, import('node:child_process').ChildProcess>} running
@@ -219,7 +235,7 @@ export async function startOrchestrator(opts) {
  * @param {boolean} testMode
  * @param {Object} seed - Seed data containing pipeline information
  */
-function spawnRunner(jobId, dirs, running, spawn, testMode, seed) {
+function spawnRunner(logger, jobId, dirs, running, spawn, testMode, seed) {
   // Use path relative to this file to avoid process.cwd() issues
   const orchestratorDir = path.dirname(new URL(import.meta.url).pathname);
   const runnerPath = path.join(orchestratorDir, "pipeline-runner.js");
@@ -234,7 +250,7 @@ function spawnRunner(jobId, dirs, running, spawn, testMode, seed) {
     const availablePipelines = Object.keys(configSnapshot?.pipelines ?? {});
     const pipelineSlug = seed?.pipeline;
 
-    console.log("[Orchestrator] spawnRunner invoked", {
+    logger.log("spawnRunner invoked", {
       jobId,
       pipelineSlug: pipelineSlug ?? null,
       availablePipelines,
@@ -242,22 +258,19 @@ function spawnRunner(jobId, dirs, running, spawn, testMode, seed) {
     });
 
     if (!availablePipelines.length) {
-      console.warn(
-        "[Orchestrator] No pipelines registered in config() when spawnRunner invoked"
+      logger.warn(
+        "No pipelines registered in config() when spawnRunner invoked"
       );
     } else if (!availablePipelines.includes(pipelineSlug)) {
-      console.warn(
-        "[Orchestrator] Requested pipeline slug missing from registry snapshot",
-        {
-          jobId,
-          pipelineSlug,
-          availablePipelines,
-        }
-      );
+      logger.warn("Requested pipeline slug missing from registry snapshot", {
+        jobId,
+        pipelineSlug,
+        availablePipelines,
+      });
     }
 
     if (!pipelineSlug) {
-      console.error("[Orchestrator] Missing pipeline slug in seed", {
+      logger.error("Missing pipeline slug in seed", {
         jobId,
         seed,
         availablePipelines,
@@ -271,7 +284,7 @@ function spawnRunner(jobId, dirs, running, spawn, testMode, seed) {
     try {
       pipelineConfig = getPipelineConfig(pipelineSlug);
     } catch (error) {
-      console.error("[Orchestrator] Pipeline lookup failed", {
+      logger.error("Pipeline lookup failed", {
         jobId,
         pipelineSlug,
         availablePipelines,

package/src/core/pipeline-runner.js

@@ -9,6 +9,7 @@ import { TaskState } from "../config/statuses.js";
 import { ensureTaskSymlinkBridge } from "./symlink-bridge.js";
 import { cleanupTaskSymlinks } from "./symlink-utils.js";
 import { createTaskFileIO } from "./file-io.js";
+import { createJobLogger } from "./logger.js";
 
 const ROOT = process.env.PO_ROOT || process.cwd();
 const DATA_DIR = path.join(ROOT, process.env.PO_DATA_DIR || "pipeline-data");
@@ -20,6 +21,8 @@ const COMPLETE_DIR =
 const jobId = process.argv[2];
 if (!jobId) throw new Error("runner requires jobId as argument");
 
+const logger = createJobLogger("PipelineRunner", jobId);
+
 const workDir = path.join(CURRENT_DIR, jobId);
 
 const startFromTask = process.env.PO_START_FROM_TASK;
@@ -66,12 +69,23 @@ const seed = JSON.parse(
 
 let pipelineArtifacts = {};
 
+logger.group("Pipeline execution", {
+  jobId,
+  pipelineSlug,
+  totalTasks: pipeline.tasks.length,
+  startFromTask: startFromTask || null,
+});
+
 for (const taskName of pipeline.tasks) {
   // Skip tasks before startFromTask when targeting a specific restart point
   if (
     startFromTask &&
     pipeline.tasks.indexOf(taskName) < pipeline.tasks.indexOf(startFromTask)
   ) {
+    logger.log("Skipping task before restart point", {
+      taskName,
+      startFromTask,
+    });
     continue;
   }
 
@@ -80,10 +94,14 @@ for (const taskName of pipeline.tasks) {
       const outputPath = path.join(workDir, "tasks", taskName, "output.json");
       const output = JSON.parse(await fs.readFile(outputPath, "utf8"));
       pipelineArtifacts[taskName] = output;
-    } catch {}
+      logger.log("Task already completed", { taskName });
+    } catch {
+      logger.warn("Failed to read completed task output", { taskName });
+    }
     continue;
   }
 
+  logger.log("Starting task", { taskName });
   await updateStatus(taskName, {
     state: TaskState.RUNNING,
     startedAt: now(),
@@ -130,9 +148,17 @@ for (const taskName of pipeline.tasks) {
       statusPath: tasksStatusPath,
     });
 
+    logger.log("Running task", { taskName, modulePath: absoluteModulePath });
     const result = await runPipeline(relocatedEntry, ctx);
 
     if (!result.ok) {
+      logger.error("Task failed", {
+        taskName,
+        failedStage: result.failedStage,
+        error: result.error,
+        refinementAttempts: result.refinementAttempts || 0,
+      });
+
       // Persist execution-logs.json and failure-details.json on task failure via IO
       if (result.logs) {
         await fileIO.writeLog(
@@ -180,6 +206,13 @@ for (const taskName of pipeline.tasks) {
       process.exit(1);
     }
 
+    logger.log("Task completed successfully", {
+      taskName,
+      executionTimeMs:
+        result.logs?.reduce((total, log) => total + (log.ms || 0), 0) || 0,
+      refinementAttempts: result.refinementAttempts || 0,
+    });
+
     // The file I/O system automatically handles writing outputs and updating tasks-status.json
     // No need to manually write output.json or enumerate artifacts
 
@@ -211,6 +244,20 @@ for (const taskName of pipeline.tasks) {
 
 await fs.mkdir(COMPLETE_DIR, { recursive: true });
 const dest = path.join(COMPLETE_DIR, jobId);
+
+logger.log("Pipeline completed", {
+  jobId,
+  totalExecutionTime: Object.values(status.tasks).reduce(
+    (total, t) => total + (t.executionTimeMs || 0),
+    0
+  ),
+  totalRefinementAttempts: Object.values(status.tasks).reduce(
+    (total, t) => total + (t.refinementAttempts || 0),
+    0
+  ),
+  finalArtifacts: Object.keys(pipelineArtifacts),
+});
+
 await fs.rename(workDir, dest);
 await appendLine(
   path.join(COMPLETE_DIR, "runs.jsonl"),
@@ -233,6 +280,8 @@ await appendLine(
 // Clean up task symlinks to avoid dangling links in archives
 await cleanupTaskSymlinks(dest);
 
+logger.groupEnd();
+
 function now() {
   return new Date().toISOString();
 }

package/src/core/status-writer.js

@@ -1,50 +1,11 @@
 import fs from "node:fs/promises";
 import path from "node:path";
 import { TaskState } from "../config/statuses.js";
-
-// Lazy import SSE registry to avoid circular dependencies
-let sseRegistry = null;
-async function getSSERegistry() {
-  if (!sseRegistry) {
-    try {
-      const module = await import("../ui/sse.js");
-      sseRegistry = module.sseRegistry;
-    } catch (error) {
-      // SSE not available in all environments
-      return null;
-    }
-  }
-  return sseRegistry;
-}
+import { createJobLogger } from "./logger.js";
 
 // Per-job write queues to serialize writes to tasks-status.json
 const writeQueues = new Map(); // Map<string jobDir, Promise<any>>
 
-// Instrumentation helper for status writer
-const createStatusWriterLogger = (jobId) => {
-  const prefix = `[StatusWriter:${jobId || "unknown"}]`;
-  return {
-    log: (message, data = null) => {
-      console.log(`${prefix} ${message}`, data ? data : "");
-    },
-    warn: (message, data = null) => {
-      console.warn(`${prefix} ${message}`, data ? data : "");
-    },
-    error: (message, data = null) => {
-      console.error(`${prefix} ${message}`, data ? data : "");
-    },
-    group: (label) => console.group(`${prefix} ${label}`),
-    groupEnd: () => console.groupEnd(),
-    sse: (eventType, eventData) => {
-      console.log(
-        `%c${prefix} SSE Broadcast: ${eventType}`,
-        "color: #cc6600; font-weight: bold;",
-        eventData
-      );
-    },
-  };
-};
-
 /**
  * Atomic status writer utility for tasks-status.json
  *
@@ -195,7 +156,7 @@ export async function writeJobStatus(jobDir, updateFn) {
 
   const statusPath = path.join(jobDir, "tasks-status.json");
   const jobId = path.basename(jobDir);
-  const logger = createStatusWriterLogger(jobId);
+  const logger = createJobLogger("StatusWriter", jobId);
 
   // Get or create the write queue for this job directory
   const prev = writeQueues.get(jobDir) || Promise.resolve();
@@ -219,7 +180,7 @@ export async function writeJobStatus(jobDir, updateFn) {
       try {
         maybeUpdated = updateFn(validated);
       } catch (error) {
-        console.error(`[${jobId}] Error executing update function:`, error);
+        logger.error("Error executing update function:", error);
         throw new Error(`Update function failed: ${error.message}`);
       }
       const snapshot = validateStatusSnapshot(
@@ -233,28 +194,18 @@ export async function writeJobStatus(jobDir, updateFn) {
       await atomicWrite(statusPath, snapshot);
       logger.log("Status file written successfully");
 
-      // Emit SSE event for tasks-status.json change
-      const registry = (await getSSERegistry().catch(() => null)) || null;
-      if (registry) {
-        try {
-          const eventData = {
-            type: "state:change",
-            data: {
-              path: path.join(jobDir, "tasks-status.json"),
-              id: jobId,
-              jobId,
-            },
-          };
-          registry.broadcast(eventData);
-          logger.sse("state:change", eventData.data);
-          logger.log("SSE event broadcasted successfully");
-        } catch (error) {
-          // Don't fail the write if SSE emission fails
-          logger.error("Failed to emit SSE event:", error);
-          console.warn(`Failed to emit SSE event: ${error.message}`);
-        }
-      } else {
-        logger.warn("SSE registry not available - no event broadcasted");
+      // Emit SSE event for tasks-status.json change using logger
+      try {
+        const eventData = {
+          path: path.join(jobDir, "tasks-status.json"),
+          id: jobId,
+          jobId,
+        };
+        await logger.sse("state:change", eventData);
+        logger.log("SSE event broadcasted successfully");
+      } catch (error) {
+        // Don't fail the write if SSE emission fails
+        logger.error("Failed to emit SSE event:", error);
       }
 
       logger.groupEnd();

package/src/core/symlink-utils.js

@@ -1,5 +1,8 @@
 import fs from "node:fs/promises";
 import path from "node:path";
+import { createLogger } from "./logger.js";
+
+const logger = createLogger("SymlinkUtils");
 
 /**
  * Creates an idempotent symlink, safely handling existing files/symlinks.
@@ -87,8 +90,9 @@ export async function cleanupTaskSymlinks(completedJobDir) {
     }
   } catch (error) {
     // Log but don't fail - cleanup is optional
-    console.warn(
-      `Warning: Failed to cleanup task symlinks in ${completedJobDir}: ${error.message}`
-    );
+    logger.warn("Failed to cleanup task symlinks", {
+      directory: completedJobDir,
+      error: error.message,
+    });
   }
 }

package/src/core/task-runner.js

@@ -9,6 +9,7 @@ import { writeJobStatus } from "./status-writer.js";
 import { computeDeterministicProgress } from "./progress.js";
 import { TaskState } from "../config/statuses.js";
 import { validateWithSchema } from "../api/validators/json.js";
+import { createJobLogger } from "./logger.js";
 
 /**
  * Derives model key and token counts from LLM metric event.
@@ -341,6 +342,11 @@ const PIPELINE_STAGES = [
  * Runs a pipeline by loading a module that exports functions keyed by stage name.
  */
 export async function runPipeline(modulePath, initialContext = {}) {
+  const logger = createJobLogger(
+    "TaskRunner",
+    initialContext.jobId || "unknown"
+  );
+
   if (!initialContext.envLoaded) {
     await loadEnvironment();
     initialContext.envLoaded = true;
@@ -477,6 +483,10 @@ export async function runPipeline(modulePath, initialContext = {}) {
 
     // Skip stages when skipIf predicate returns true
     if (stageConfig.skipIf && stageConfig.skipIf(context.flags)) {
+      logger.log("Skipping stage", {
+        stage: stageName,
+        reason: "skipIf predicate returned true",
+      });
       context.logs.push({
         stage: stageName,
         action: "skipped",
@@ -488,6 +498,7 @@ export async function runPipeline(modulePath, initialContext = {}) {
 
     // Skip if handler is not available (not implemented)
     if (typeof stageHandler !== "function") {
+      logger.log("Stage not available, skipping", { stage: stageName });
       logs.push({
         stage: stageName,
         skipped: true,
@@ -510,6 +521,11 @@ export async function runPipeline(modulePath, initialContext = {}) {
     // Set current stage before execution
     context.currentStage = stageName;
 
+    logger.log("Starting stage execution", {
+      stage: stageName,
+      taskName: context.meta.taskName,
+    });
+
     // Write stage start status using writeJobStatus
     if (context.meta.workDir && context.meta.taskName) {
       try {
@@ -527,7 +543,10 @@ export async function runPipeline(modulePath, initialContext = {}) {
         });
       } catch (error) {
         // Don't fail the pipeline if status write fails
-        console.warn(`Failed to write stage start status: ${error.message}`);
+        logger.error("Failed to write stage start status", {
+          stage: stageName,
+          error: error.message,
+        });
       }
     }
 
@@ -678,6 +697,12 @@ export async function runPipeline(modulePath, initialContext = {}) {
       }
 
       const ms = +(performance.now() - start).toFixed(2);
+      logger.log("Stage completed successfully", {
+        stage: stageName,
+        executionTimeMs: ms,
+        outputType: typeof stageResult.output,
+        flagKeys: Object.keys(stageResult.flags),
+      });
       logs.push({
         stage: stageName,
         ok: true,
@@ -688,6 +713,14 @@ export async function runPipeline(modulePath, initialContext = {}) {
       const ms = +(performance.now() - start).toFixed(2);
       const errInfo = normalizeError(error);
 
+      logger.error("Stage execution failed", {
+        stage: stageName,
+        taskName: context.meta.taskName,
+        executionTimeMs: ms,
+        error: errInfo,
+        previousStage: lastExecutedStageName,
+      });
+
       // Attach debug metadata to the error envelope for richer diagnostics
       errInfo.debug = {
         stage: stageName,
@@ -756,6 +789,13 @@ export async function runPipeline(modulePath, initialContext = {}) {
 
   llmEvents.off("llm:request:complete", onLLMComplete);
 
+  logger.log("Pipeline completed successfully", {
+    taskName: context.meta.taskName,
+    totalStages: PIPELINE_STAGES.length,
+    executedStages: logs.filter((l) => l.ok).length,
+    llmMetricsCount: llmMetrics.length,
+  });
+
   // Write final status with currentStage: null to indicate completion
   if (context.meta.workDir && context.meta.taskName) {
     try {
@@ -775,7 +815,10 @@ export async function runPipeline(modulePath, initialContext = {}) {
       });
     } catch (error) {
       // Don't fail the pipeline if final status write fails
-      console.warn(`Failed to write final status: ${error.message}`);
+      logger.error("Failed to write final status", {
+        taskName: context.meta.taskName,
+        error: error.message,
+      });
     }
   }
 

package/src/ui/job-reader.js

@@ -30,6 +30,7 @@ export async function readJob(jobId) {
 
   // Locations in precedence order
   const locations = ["current", "complete"];
+  const attemptedLocations = [];
 
   for (const location of locations) {
     console.log(`readJob: checking location ${location} for ${jobId}`);
@@ -70,11 +71,8 @@ export async function readJob(jobId) {
     const result = await readFileWithRetry(tasksPath);
 
     if (!result.ok) {
-      // Log a warning for failed reads of tasks-status.json in this location
-      console.warn(
-        `Failed to read tasks-status.json for job ${jobId} in ${location}`,
-        result
-      );
+      // Track attempted location for final warning if needed
+      attemptedLocations.push(location);
 
       // If not found, continue to next location
       if (result.code === configBridge.Constants.ERROR_CODES.NOT_FOUND) {
@@ -101,6 +99,9 @@ export async function readJob(jobId) {
   }
 
   // If we reach here, job not found in any location
+  console.warn(
+    `Job ${jobId} not found in any location. Searched: ${attemptedLocations.join(", ")}`
+  );
   return configBridge.createErrorResponse(
     configBridge.Constants.ERROR_CODES.JOB_NOT_FOUND,
     "Job not found",