@ryanfw/prompt-orchestration-pipeline 0.15.0 → 0.16.0

package/src/core/batch-runner.js

@@ -0,0 +1,277 @@
+/**
+ * Batch Runner - Concurrent job processing with SQLite state management
+ */
+
+import crypto from "node:crypto";
+import pLimit from "p-limit";
+
+/**
+ * Creates the batch_jobs table and index if they don't exist
+ * @param {import('better-sqlite3').Database} db - SQLite database instance
+ */
+export function ensureBatchSchema(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS batch_jobs (
+      id TEXT PRIMARY KEY,
+      batch_id TEXT NOT NULL,
+      status TEXT NOT NULL DEFAULT 'pending',
+      input TEXT NOT NULL,
+      output TEXT,
+      error TEXT,
+      retry_count INTEGER NOT NULL DEFAULT 0,
+      started_at TEXT,
+      completed_at TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_batch_jobs_batch_status ON batch_jobs(batch_id, status);
+  `);
+}
+
+/**
+ * Inserts jobs into the batch_jobs table
+ * @param {import('better-sqlite3').Database} db - SQLite database instance
+ * @param {string} batchId - Unique batch identifier
+ * @param {Array<Object>} jobs - Array of job objects
+ * @returns {string[]} Array of job IDs inserted
+ */
+export function insertJobs(db, batchId, jobs) {
+  const insertStmt = db.prepare(
+    `INSERT OR IGNORE INTO batch_jobs (id, batch_id, status, input) VALUES (?, ?, 'pending', ?)`
+  );
+  const selectStatusStmt = db.prepare(
+    `SELECT status FROM batch_jobs WHERE id = ? AND batch_id = ?`
+  );
+
+  const insertMany = db.transaction((jobList) => {
+    const ids = [];
+    for (const job of jobList) {
+      const id = job.id ?? crypto.randomUUID();
+      const input = JSON.stringify(job);
+      const result = insertStmt.run(id, batchId, input);
+
+      // If no row was inserted, the job already exists. Validate its state.
+      if (result.changes === 0) {
+        const existing = selectStatusStmt.get(id, batchId);
+        if (existing && (existing.status === "complete" || existing.status === "permanently_failed")) {
+          throw new Error(
+            `Cannot re-insert job "${id}" for batch "${batchId}": existing job is in terminal state "${existing.status}".`
+          );
+        }
+      }
+      ids.push(id);
+    }
+    return ids;
+  });
+
+  return insertMany(jobs);
+}
+
+/**
+ * Marks a job as processing
+ * @param {import('better-sqlite3').Database} db - SQLite database instance
+ * @param {string} jobId - Job identifier
+ */
+export function markProcessing(db, jobId) {
+  const stmt = db.prepare(
+    `UPDATE batch_jobs SET status = 'processing', started_at = datetime('now') WHERE id = ?`
+  );
+  stmt.run(jobId);
+}
+
+/**
+ * Marks a job as complete with output
+ * @param {import('better-sqlite3').Database} db - SQLite database instance
+ * @param {string} jobId - Job identifier
+ * @param {*} output - Job output (will be JSON serialized)
+ */
+export function markComplete(db, jobId, output) {
+  const stmt = db.prepare(
+    `UPDATE batch_jobs SET status = 'complete', output = ?, completed_at = datetime('now') WHERE id = ?`
+  );
+  stmt.run(JSON.stringify(output), jobId);
+}
+
+/**
+ * Marks a job as failed and increments retry count
+ * @param {import('better-sqlite3').Database} db - SQLite database instance
+ * @param {string} jobId - Job identifier
+ * @param {string} error - Error message
+ */
+export function markFailed(db, jobId, error) {
+  const stmt = db.prepare(
+    `UPDATE batch_jobs SET status = 'failed', error = ?, retry_count = retry_count + 1 WHERE id = ?`
+  );
+  stmt.run(error, jobId);
+}
+
+/**
+ * Gets pending and failed jobs that are under the retry limit
+ * @param {import('better-sqlite3').Database} db - SQLite database instance
+ * @param {string} batchId - Unique batch identifier
+ * @param {number} maxRetries - Maximum retry attempts
+ * @returns {Array<{id: string, input: Object, retryCount: number}>} Array of pending jobs
+ */
+export function getPendingJobs(db, batchId, maxRetries) {
+  const stmt = db.prepare(
+    `SELECT id, input, retry_count FROM batch_jobs WHERE batch_id = ? AND status IN ('pending', 'failed') AND retry_count < ? ORDER BY id`
+  );
+  const rows = stmt.all(batchId, maxRetries);
+  return rows.map((row) => ({
+    id: row.id,
+    input: JSON.parse(row.input),
+    retryCount: row.retry_count,
+  }));
+}
+
+/**
+ * Recovers jobs stuck in 'processing' state (from process crash)
+ * @param {import('better-sqlite3').Database} db - SQLite database instance
+ * @param {string} batchId - Unique batch identifier
+ * @returns {number} Number of jobs recovered
+ */
+export function recoverStaleJobs(db, batchId) {
+  const stmt = db.prepare(
+    `UPDATE batch_jobs SET status = 'pending' WHERE batch_id = ? AND status = 'processing'`
+  );
+  const result = stmt.run(batchId);
+  return result.changes;
+}
+
+/**
+ * Gets completed jobs for a batch
+ * @param {import('better-sqlite3').Database} db - SQLite database instance
+ * @param {string} batchId - Unique batch identifier
+ * @returns {Array<{id: string, input: Object, output: *}>} Array of completed jobs
+ */
+function getCompletedJobs(db, batchId) {
+  const stmt = db.prepare(
+    `SELECT id, input, output FROM batch_jobs WHERE batch_id = ? AND status = 'complete'`
+  );
+  const rows = stmt.all(batchId);
+  return rows.map((row) => ({
+    id: row.id,
+    input: JSON.parse(row.input),
+    output: JSON.parse(row.output),
+  }));
+}
+
+/**
+ * Gets failed jobs for a batch (those that exhausted retries)
+ * @param {import('better-sqlite3').Database} db - SQLite database instance
+ * @param {string} batchId - Unique batch identifier
+ * @param {number} maxRetries - Maximum retry attempts
+ * @returns {Array<{id: string, input: Object, error: string, retryCount: number}>} Array of failed jobs
+ */
+function getFailedJobs(db, batchId, maxRetries) {
+  const stmt = db.prepare(
+    `SELECT id, input, error, retry_count FROM batch_jobs WHERE batch_id = ? AND status = 'failed' AND retry_count >= ?`
+  );
+  const rows = stmt.all(batchId, maxRetries);
+  return rows.map((row) => ({
+    id: row.id,
+    input: JSON.parse(row.input),
+    error: row.error,
+    retryCount: row.retry_count,
+  }));
+}
+
+/**
+ * Processes a single job with try/catch and status updates
+ * @param {import('better-sqlite3').Database} db - SQLite database instance
+ * @param {Object} job - Job object with id, input, retryCount
+ * @param {Function} processor - async (input, ctx) => result
+ * @param {string} batchId - Unique batch identifier
+ */
+async function processOneJob(db, job, processor, batchId) {
+  markProcessing(db, job.id);
+  try {
+    const output = await processor(job.input, {
+      attempt: job.retryCount + 1,
+      batchId,
+      db,
+    });
+    markComplete(db, job.id, output);
+  } catch (err) {
+    markFailed(db, job.id, err.message || String(err));
+  }
+}
+
+/**
+ * Validates batch options and throws with descriptive errors if invalid
+ * @param {Object} options - Batch options to validate
+ * @throws {Error} If options are invalid
+ */
+export function validateBatchOptions(options) {
+  if (!options || typeof options !== "object") {
+    throw new Error(
+      `runBatch: options must be an object, got: ${typeof options}`
+    );
+  }
+  if (!Array.isArray(options.jobs)) {
+    throw new Error(
+      `runBatch: jobs must be an array, got: ${typeof options.jobs}`
+    );
+  }
+  if (options.jobs.length === 0) {
+    throw new Error("runBatch: jobs must be a non-empty array");
+  }
+  if (typeof options.processor !== "function") {
+    throw new Error(
+      `runBatch: processor must be a function, got: ${typeof options.processor}`
+    );
+  }
+  if (options.concurrency !== undefined) {
+    if (!Number.isInteger(options.concurrency) || options.concurrency < 1) {
+      throw new Error(
+        `runBatch: concurrency must be a positive integer, got: ${options.concurrency}`
+      );
+    }
+  }
+  if (options.maxRetries !== undefined) {
+    if (!Number.isInteger(options.maxRetries) || options.maxRetries < 0) {
+      throw new Error(
+        `runBatch: maxRetries must be a non-negative integer, got: ${options.maxRetries}`
+      );
+    }
+  }
+}
+
+/**
+ * Executes a batch of jobs concurrently with retry support
+ * @param {import('better-sqlite3').Database} db - SQLite database instance
+ * @param {Object} options - Batch options
+ * @param {Array<Object>} options.jobs - Array of job objects
+ * @param {Function} options.processor - async (input, ctx) => result
+ * @param {number} [options.concurrency=10] - Max concurrent jobs
+ * @param {number} [options.maxRetries=3] - Max retry attempts per job
+ * @param {string} [options.batchId] - Unique batch identifier (auto-generated if omitted)
+ * @returns {Promise<{completed: Array, failed: Array}>} Batch results
+ */
+export async function executeBatch(db, options) {
+  const {
+    jobs,
+    processor,
+    concurrency = 10,
+    maxRetries = 3,
+    batchId = crypto.randomUUID(),
+  } = options;
+
+  ensureBatchSchema(db);
+  recoverStaleJobs(db, batchId);
+  insertJobs(db, batchId, jobs);
+
+  const limit = pLimit(concurrency);
+
+  let pending = getPendingJobs(db, batchId, maxRetries);
+  while (pending.length > 0) {
+    const promises = pending.map((job) =>
+      limit(() => processOneJob(db, job, processor, batchId))
+    );
+    await Promise.allSettled(promises);
+    pending = getPendingJobs(db, batchId, maxRetries);
+  }
+
+  return {
+    completed: getCompletedJobs(db, batchId),
+    failed: getFailedJobs(db, batchId, maxRetries),
+  };
+}

package/src/core/file-io.js

@@ -8,6 +8,8 @@ import {
   isValidLogEvent,
   isValidLogFileExtension,
 } from "../config/log-events.js";
+import Database from "better-sqlite3";
+import { executeBatch, validateBatchOptions } from "./batch-runner.js";
 
 /**
  * Creates a task-scoped file I/O interface that manages file operations
@@ -26,7 +28,7 @@ async function ensureDir(dir) {
 }
 
 function ensureDirSync(dir) {
-  fsSync.mkdir(dir, { recursive: true });
+  fsSync.mkdirSync(dir, { recursive: true });
 }
 
 export function createTaskFileIO({
@@ -293,6 +295,40 @@ export function createTaskFileIO({
     getCurrentStage() {
       return getStage();
     },
+
+    /**
+     * Get a SQLite database instance for this job run
+     * @param {Object} options - better-sqlite3 options
+     * @returns {Database} better-sqlite3 Database instance
+     */
+    getDB(options = {}) {
+      ensureDirSync(artifactsDir);
+      const dbPath = path.join(artifactsDir, "run.db");
+      const db = new Database(dbPath, options);
+      db.pragma("journal_mode = WAL");
+      updateStatusWithFilesSync("artifacts", "run.db");
+      return db;
+    },
+
+    /**
+     * Execute a batch of jobs concurrently with SQLite state management
+     * @param {Object} options - Batch options
+     * @param {Array<Object>} options.jobs - Array of job objects
+     * @param {Function} options.processor - async (input, ctx) => result
+     * @param {number} [options.concurrency=10] - Max concurrent jobs
+     * @param {number} [options.maxRetries=3] - Max retry attempts per job
+     * @param {string} [options.batchId] - Unique batch identifier (auto-generated if omitted)
+     * @returns {Promise<{completed: Array, failed: Array}>} Batch results
+     */
+    async runBatch(options) {
+      validateBatchOptions(options);
+      const db = this.getDB();
+      try {
+        return await executeBatch(db, options);
+      } finally {
+        db.close();
+      }
+    },
   };
 }
 

package/src/core/orchestrator.js

@@ -123,10 +123,8 @@ export async function startOrchestrator(opts) {
     } catch {}
 
     // Move seed to current/{jobId}/seed.json
-    logger.log("Moving file", { from: filePath, to: dest });
     try {
       await moveFile(filePath, dest);
-      logger.log("Successfully moved file", { destination: dest });
     } catch (error) {
       logger.error("Failed to move file", {
         from: filePath,
@@ -173,12 +171,6 @@ export async function startOrchestrator(opts) {
         // Apply artifact initialization to the status
         const updatedStatus = applyArtifacts(status);
         await fs.writeFile(statusPath, JSON.stringify(updatedStatus, null, 2));
-
-        logger.log("Initialized status from upload artifacts", {
-          jobId,
-          pipeline: seed?.pipeline,
-          artifactsCount: updatedStatus.files?.artifacts?.length || 0,
-        });
       } catch (artifactError) {
         // Don't fail job startup if artifact initialization fails, just log
         logger.warn("Failed to initialize status from artifacts", {
@@ -233,7 +225,6 @@ export async function startOrchestrator(opts) {
 
   // Watch pending directory for seeds
   const watchPattern = path.join(dirs.pending, "*.json");
-  logger.log("Watching pattern", { pattern: watchPattern });
   const watcher = watcherFactory(watchPattern, {
     ignoreInitial: false,
     awaitWriteFinish: false, // Disable awaitWriteFinish for faster detection
@@ -243,7 +234,6 @@ export async function startOrchestrator(opts) {
   // Wait for watcher to be ready before resolving
   await new Promise((resolve, reject) => {
     watcher.on("ready", () => {
-      logger.log("Watcher is ready");
       resolve();
     });
 
@@ -254,7 +244,6 @@ export async function startOrchestrator(opts) {
   });
 
   watcher.on("add", (file) => {
-    logger.log("Detected file add", { file });
     // Return promise so tests awaiting the add handler block until processing completes
     return handleSeedAdd(file);
   });
@@ -330,13 +319,6 @@ function spawnRunner(
     const availablePipelines = Object.keys(configSnapshot?.pipelines ?? {});
     const pipelineSlug = seed?.pipeline;
 
-    logger.log("spawnRunner invoked", {
-      jobId,
-      pipelineSlug: pipelineSlug ?? null,
-      availablePipelines,
-      seedKeys: seed ? Object.keys(seed) : null,
-    });
-
     if (!availablePipelines.length) {
       logger.warn(
         "No pipelines registered in config() when spawnRunner invoked"

package/src/core/pipeline-runner.js

@@ -143,10 +143,6 @@ try {
       startFromTask &&
       taskNames.indexOf(taskName) < taskNames.indexOf(startFromTask)
     ) {
-      logger.log("Skipping task before restart point", {
-        taskName,
-        startFromTask,
-      });
       continue;
     }
 
@@ -155,7 +151,6 @@ try {
         const outputPath = path.join(workDir, "tasks", taskName, "output.json");
         const output = JSON.parse(await fs.readFile(outputPath, "utf8"));
         pipelineArtifacts[taskName] = output;
-        logger.log("Task already completed", { taskName });
       } catch {
         logger.warn("Failed to read completed task output", { taskName });
       }
@@ -190,7 +185,6 @@ try {
       }
     }
 
-    logger.log("Starting task", { taskName });
     await updateStatus(taskName, {
       state: TaskState.RUNNING,
       startedAt: now(),
@@ -265,13 +259,6 @@ try {
           process.exitCode = 1;
           process.exit(1);
         }
-
-        logger.log("Task symlinks repaired successfully", {
-          taskName,
-          taskDir,
-          repairDuration: repairResult.duration,
-          relocatedEntry: repairResult.relocatedEntry,
-        });
       } else {
         logger.debug("Task symlinks validation passed", {
           taskName,
@@ -295,7 +282,6 @@ try {
         statusPath: tasksStatusPath,
       });
 
-      logger.log("Running task", { taskName, modulePath: absoluteModulePath });
       const result = await runPipeline(relocatedEntry, ctx);
 
       if (!result.ok) {
@@ -363,13 +349,6 @@ try {
         process.exit(1);
       }
 
-      logger.log("Task completed successfully", {
-        taskName,
-        executionTimeMs:
-          result.logs?.reduce((total, log) => total + (log.ms || 0), 0) || 0,
-        refinementAttempts: result.refinementAttempts || 0,
-      });
-
       // The file I/O system automatically handles writing outputs and updating tasks-status.json
       // No need to manually write output.json or enumerate artifacts
 
@@ -396,7 +375,6 @@ try {
 
       // Check if this is a single task run and we've completed the target task
       if (runSingleTask && taskName === startFromTask) {
-        logger.log("Stopping after single task execution", { taskName });
         break;
       }
     } catch (err) {
@@ -415,19 +393,6 @@ try {
     await fs.mkdir(COMPLETE_DIR, { recursive: true });
     const dest = path.join(COMPLETE_DIR, jobId);
 
-    logger.log("Pipeline completed", {
-      jobId,
-      totalExecutionTime: Object.values(status.tasks).reduce(
-        (total, t) => total + (t.executionTimeMs || 0),
-        0
-      ),
-      totalRefinementAttempts: Object.values(status.tasks).reduce(
-        (total, t) => total + (t.refinementAttempts || 0),
-        0
-      ),
-      finalArtifacts: Object.keys(pipelineArtifacts),
-    });
-
     await fs.rename(workDir, dest);
     await appendLine(
       path.join(COMPLETE_DIR, "runs.jsonl"),
@@ -449,8 +414,6 @@ try {
 
     // Clean up task symlinks to avoid dangling links in archives
     await cleanupTaskSymlinks(dest);
-  } else {
-    logger.log("Single task run completed, job remains in current", { jobId });
   }
 } catch (error) {
   throw error;

package/src/core/status-writer.js

@@ -164,13 +164,8 @@ export async function writeJobStatus(jobDir, updateFn) {
 
   const next = prev
     .then(async () => {
-      logger.group("Status Write Operation");
-      logger.log(`Updating status for job: ${jobId}`);
-      logger.log(`Status file path: ${statusPath}`);
-
       // Read existing status or create default
       const current = await readStatusFile(statusPath, jobId);
-      logger.log("Current status snapshot:", current);
 
       // Validate basic structure
       const validated = validateStatusSnapshot(current);
@@ -188,11 +183,9 @@ export async function writeJobStatus(jobDir, updateFn) {
       );
 
       snapshot.lastUpdated = new Date().toISOString();
-      logger.log("Status after update function:", snapshot);
 
       // Atomic write
       await atomicWrite(statusPath, snapshot);
-      logger.log("Status file written successfully");
 
       // Emit SSE event for tasks-status.json change using logger
       try {
@@ -202,7 +195,6 @@ export async function writeJobStatus(jobDir, updateFn) {
           jobId,
         };
         await logger.sse("state:change", eventData);
-        logger.log("SSE event broadcasted successfully");
       } catch (error) {
         // Don't fail the write if SSE emission fails
         logger.error("Failed to emit SSE event:", error);
@@ -218,10 +210,6 @@ export async function writeJobStatus(jobDir, updateFn) {
             reason: snapshot.lifecycleBlockReason,
           };
           await logger.sse("lifecycle_block", lifecycleEventData);
-          logger.log(
-            "lifecycle_block SSE event broadcasted successfully",
-            lifecycleEventData
-          );
         } catch (error) {
           // Don't fail the write if SSE emission fails
           logger.error("Failed to emit lifecycle_block SSE event:", error);
@@ -310,9 +298,6 @@ export async function updateTaskStatus(jobDir, taskId, taskUpdateFn) {
 
   const next = prev
     .then(async () => {
-      logger.group("Task Status Update Operation");
-      logger.log(`Updating task ${taskId} for job: ${jobId}`);
-
       const statusPath = path.join(jobDir, "tasks-status.json");
 
       // Read existing status or create default
@@ -336,7 +321,6 @@ export async function updateTaskStatus(jobDir, taskId, taskUpdateFn) {
 
       // Atomic write
       await atomicWrite(statusPath, validated);
-      logger.log("Task status file written successfully");
 
       // Emit task:updated SSE event after successful write
       try {
@@ -346,13 +330,11 @@ export async function updateTaskStatus(jobDir, taskId, taskUpdateFn) {
           task: validated.tasks[taskId],
         };
         await logger.sse("task:updated", eventData);
-        logger.log("task:updated SSE event broadcasted successfully");
       } catch (error) {
         // Don't fail the write if SSE emission fails
         logger.error("Failed to emit task:updated SSE event:", error);
       }
 
-      logger.groupEnd();
       resultSnapshot = validated;
     })
     .catch((e) => {

package/src/core/symlink-utils.js

@@ -196,12 +196,6 @@ export async function repairTaskSymlinks(taskDir, poRoot, taskModulePath) {
   const startTime = Date.now();
 
   try {
-    logger.log("Repairing task symlinks", {
-      taskDir,
-      poRoot,
-      taskModulePath,
-    });
-
     // Use existing ensureTaskSymlinkBridge for repairs
     const relocatedEntry = await ensureTaskSymlinkBridge({
       taskDir,
@@ -211,12 +205,6 @@ export async function repairTaskSymlinks(taskDir, poRoot, taskModulePath) {
 
     const duration = Date.now() - startTime;
 
-    logger.log("Task symlinks repaired successfully", {
-      taskDir,
-      duration,
-      relocatedEntry,
-    });
-
     return {
       success: true,
       relocatedEntry,

package/src/core/task-runner.js

@@ -484,10 +484,6 @@ export async function runPipeline(modulePath, initialContext = {}) {
 
     // Skip stages when skipIf predicate returns true
     if (stageConfig.skipIf && stageConfig.skipIf(context.flags)) {
-      logger.log("Skipping stage", {
-        stage: stageName,
-        reason: "skipIf predicate returned true",
-      });
       context.logs.push({
         stage: stageName,
         action: "skipped",
@@ -499,7 +495,6 @@ export async function runPipeline(modulePath, initialContext = {}) {
 
     // Skip if handler is not available (not implemented)
     if (typeof stageHandler !== "function") {
-      logger.log("Stage not available, skipping", { stage: stageName });
       logs.push({
         stage: stageName,
         skipped: true,
@@ -526,11 +521,6 @@ export async function runPipeline(modulePath, initialContext = {}) {
     // Set current stage before execution
     context.currentStage = stageName;
 
-    logger.log("Starting stage execution", {
-      stage: stageName,
-      taskName: context.meta.taskName,
-    });
-
     // Write stage start status using writeJobStatus
     if (context.meta.workDir && context.meta.taskName) {
       try {
@@ -719,12 +709,6 @@ export async function runPipeline(modulePath, initialContext = {}) {
       );
 
       const ms = +(performance.now() - start).toFixed(2);
-      logger.log("Stage completed successfully", {
-        stage: stageName,
-        executionTimeMs: ms,
-        outputType: typeof stageResult.output,
-        flagKeys: Object.keys(stageResult.flags),
-      });
       logs.push({
         stage: stageName,
         ok: true,
@@ -819,13 +803,6 @@ export async function runPipeline(modulePath, initialContext = {}) {
 
   llmEvents.off("llm:request:complete", onLLMComplete);
 
-  logger.log("Pipeline completed successfully", {
-    taskName: context.meta.taskName,
-    totalStages: PIPELINE_STAGES.length,
-    executedStages: logs.filter((l) => l.ok).length,
-    llmMetricsCount: llmMetrics.length,
-  });
-
   // Write final status with currentStage: null to indicate completion
   if (context.meta.workDir && context.meta.taskName) {
     try {