builderman 1.4.0 → 1.5.0

package/README.md

@@ -19,7 +19,8 @@ It is designed for monorepos, long-running development processes, and CI/CD pipe
 >   - [Environment Variables](#environment-variables)
 >   - [Dependencies](#dependencies)
 >   - [Pipelines](#pipelines)
->   - [Pipeline Composition](#pipeline-composition)
+>     - [Concurrency Control](#concurrency-control)
+>     - [Pipeline Composition](#pipeline-composition)
 > - [Error Handling Guarantees](#error-handling-guarantees)
 > - [Cancellation](#cancellation)
 > - [Teardown](#teardown)
@@ -256,6 +257,31 @@ const result = await pipeline([libTask, consumerTask]).run({
 })
 ```
 
+#### Concurrency Control
+
+By default, pipelines run as many tasks concurrently as possible (limited only by dependencies). You can limit concurrent execution using `maxConcurrency`:
+
+```ts
+const result = await pipeline([task1, task2, task3, task4, task5]).run({
+  maxConcurrency: 2, // At most 2 tasks will run simultaneously
+})
+```
+
+When `maxConcurrency` is set:
+
+- Tasks that are ready to run (dependencies satisfied) will start up to the limit
+- As tasks complete, new ready tasks will start to maintain the concurrency limit
+- Dependencies are still respected — a task won't start until its dependencies complete
+
+This is useful for:
+
+- Limiting resource usage (CPU, memory, network)
+- Controlling database connection pools
+- Managing API rate limits
+- Reducing system load in CI environments
+
+If `maxConcurrency` is not specified, there is no limit (tasks run concurrently as dependencies allow).
+
 ---
 
 ### Pipeline Composition
@@ -306,16 +332,19 @@ if (!result.ok) {
       console.error("Pipeline was cancelled")
       break
     case PipelineError.TaskFailed:
-      console.error(`Task failed: ${result.error.message}`)
+      console.error("Task failed:", result.error.message)
+      break
+    case PipelineError.TaskReadyTimeout:
+      console.error("Task was not ready in time:", result.error.message)
+      break
+    case PipelineError.TaskCompletedTimeout:
+      console.error("Task did not complete in time:", result.error.message)
       break
     case PipelineError.ProcessTerminated:
-      console.error("Process was terminated")
+      console.error("Process terminated:", result.error.message)
       break
     case PipelineError.InvalidTask:
-      console.error(`Invalid task configuration: ${result.error.message}`)
-      break
-    case PipelineError.InvalidSignal:
-      console.error("Invalid abort signal")
+      console.error("Invalid task configuration:", result.error.message)
       break
   }
 }

package/dist/errors.d.ts

@@ -0,0 +1,12 @@
+export type PipelineErrorCode = typeof PipelineError.Aborted | typeof PipelineError.ProcessTerminated | typeof PipelineError.TaskFailed | typeof PipelineError.TaskCompletedTimeout | typeof PipelineError.TaskReadyTimeout | typeof PipelineError.InvalidTask;
+export declare class PipelineError extends Error {
+    readonly code: PipelineErrorCode;
+    readonly taskName?: string;
+    constructor(message: string, code: PipelineErrorCode, taskName?: string);
+    static Aborted: "aborted";
+    static ProcessTerminated: "process-terminated";
+    static TaskFailed: "task-failed";
+    static TaskReadyTimeout: "task-ready-timeout";
+    static TaskCompletedTimeout: "task-completed-timeout";
+    static InvalidTask: "invalid-task";
+}

package/dist/{pipeline-error.js → errors.js}

@@ -22,29 +22,35 @@ Object.defineProperty(PipelineError, "Aborted", {
     enumerable: true,
     configurable: true,
     writable: true,
-    value: 0
+    value: "aborted"
 });
 Object.defineProperty(PipelineError, "ProcessTerminated", {
     enumerable: true,
     configurable: true,
     writable: true,
-    value: 1
+    value: "process-terminated"
 });
 Object.defineProperty(PipelineError, "TaskFailed", {
     enumerable: true,
     configurable: true,
     writable: true,
-    value: 2
+    value: "task-failed"
 });
-Object.defineProperty(PipelineError, "InvalidSignal", {
+Object.defineProperty(PipelineError, "TaskReadyTimeout", {
     enumerable: true,
     configurable: true,
     writable: true,
-    value: 3
+    value: "task-ready-timeout"
+});
+Object.defineProperty(PipelineError, "TaskCompletedTimeout", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: "task-completed-timeout"
 });
 Object.defineProperty(PipelineError, "InvalidTask", {
     enumerable: true,
     configurable: true,
     writable: true,
-    value: 4
+    value: "invalid-task"
 });

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 export { task } from "./task.js";
 export { pipeline } from "./pipeline.js";
-export { PipelineError, type PipelineErrorCode } from "./pipeline-error.js";
+export { PipelineError, type PipelineErrorCode } from "./errors.js";
 export type { Task, Pipeline, TaskConfig, Command, CommandConfig, Commands, PipelineRunConfig, PipelineTaskConfig, RunResult, PipelineStats, TaskStats, TaskStatus, } from "./types.js";

package/dist/index.js

@@ -1,3 +1,3 @@
 export { task } from "./task.js";
 export { pipeline } from "./pipeline.js";
-export { PipelineError } from "./pipeline-error.js";
+export { PipelineError } from "./errors.js";

package/dist/internal/constants.d.ts

@@ -0,0 +1,2 @@
+export declare const $TASK_INTERNAL: unique symbol;
+export declare const $PIPELINE_INTERNAL: unique symbol;

package/dist/internal/constants.js

@@ -0,0 +1,2 @@
+export const $TASK_INTERNAL = Symbol("task-internal");
+export const $PIPELINE_INTERNAL = Symbol("pipeline-internal");

package/dist/internal/execution-context.d.ts

@@ -0,0 +1,33 @@
+import type { PipelineRunConfig, TaskStats } from "../types.js";
+import type { TeardownManager } from "./teardown-manager.js";
+import type { TimeoutManager } from "./timeout-manager.js";
+import type { QueueManager } from "./queue-manager.js";
+/**
+ * Represents a task execution in progress
+ */
+export interface TaskExecution {
+    taskId: string;
+    taskName: string;
+    process?: import("node:child_process").ChildProcess;
+    startedAt: number;
+    readyAt?: number;
+}
+/**
+ * Centralized execution context that replaces scattered configuration parameters
+ * and provides unified access to execution state and helper methods
+ */
+export interface ExecutionContext {
+    config: PipelineRunConfig;
+    signal?: AbortSignal;
+    spawn: typeof import("node:child_process").spawn;
+    teardownManager: TeardownManager;
+    timeoutManager: TimeoutManager;
+    queueManager: QueueManager;
+    taskStats: Map<string, TaskStats>;
+    updateTaskStatus: (taskId: string, updates: Partial<TaskStats>) => void;
+    isAborted: () => boolean;
+}
+/**
+ * Creates an execution context for pipeline execution
+ */
+export declare function createExecutionContext(config: PipelineRunConfig, teardownManager: TeardownManager, timeoutManager: TimeoutManager, queueManager: QueueManager, taskStats: Map<string, TaskStats>): ExecutionContext;

package/dist/internal/execution-context.js

@@ -0,0 +1,30 @@
+import { spawn } from "node:child_process";
+/**
+ * Creates an execution context for pipeline execution
+ */
+export function createExecutionContext(config, teardownManager, timeoutManager, queueManager, taskStats) {
+    // Use dynamic import only if spawn is not provided
+    const spawnFn = config.spawn ?? spawn;
+    return {
+        config,
+        signal: config.signal,
+        spawn: spawnFn,
+        teardownManager,
+        timeoutManager,
+        queueManager,
+        taskStats,
+        updateTaskStatus(taskId, updates) {
+            const currentStats = taskStats.get(taskId);
+            const updatedStats = { ...currentStats, ...updates };
+            // Calculate duration if both start and finish times are available
+            if (updatedStats.startedAt && updatedStats.finishedAt) {
+                updatedStats.durationMs =
+                    updatedStats.finishedAt - updatedStats.startedAt;
+            }
+            taskStats.set(taskId, updatedStats);
+        },
+        isAborted() {
+            return config.signal?.aborted ?? false;
+        },
+    };
+}

package/dist/{graph.d.ts → internal/graph.d.ts}

@@ -1,2 +1,2 @@
-import type { TaskGraph, Task } from "./types.js";
+import type { TaskGraph, Task } from "../types.js";
 export declare function createTaskGraph(tasks: Task[]): TaskGraph;

package/dist/internal/queue-manager.d.ts

@@ -0,0 +1,24 @@
+import type { TaskGraph } from "../types.js";
+import type { TaskExecution } from "./execution-context.js";
+/**
+ * Queue manager interface returned by createQueueManager
+ */
+export interface QueueManager {
+    getNextReadyTask(): string | null;
+    markRunningTaskReady(taskId: string): void;
+    markTaskComplete(taskId: string): void;
+    markTaskFailed(taskId: string): void;
+    markTaskSkipped(taskId: string): void;
+    markTaskRunning(taskId: string, execution: TaskExecution): void;
+    canExecuteMore(): boolean;
+    isComplete(): boolean;
+    hasFailed(): boolean;
+    getRunningTasks(): Map<string, TaskExecution>;
+    clearQueues(): void;
+    abortAllRunningTasks(): void;
+}
+/**
+ * Creates a queue manager that replaces the generator-based scheduler
+ * with explicit queue-based execution state management
+ */
+export declare function createQueueManager(graph: TaskGraph, maxConcurrency?: number): QueueManager;

package/dist/internal/queue-manager.js

@@ -0,0 +1,182 @@
+/**
+ * Creates a queue manager that replaces the generator-based scheduler
+ * with explicit queue-based execution state management
+ */
+export function createQueueManager(graph, maxConcurrency) {
+    const readyQueue = [];
+    const waitingQueue = new Map();
+    const runningTasks = new Map();
+    const completedTasks = new Set();
+    const failedTasks = new Set();
+    const skippedTasks = new Set();
+    const maxConcurrencyLimit = maxConcurrency ?? Infinity;
+    let status = "running";
+    for (const [taskId, node] of graph.nodes) {
+        const depCount = node.dependencies.size;
+        if (depCount === 0) {
+            readyQueue.push(taskId);
+        }
+        else {
+            waitingQueue.set(taskId, depCount);
+        }
+    }
+    const updateDependentTasks = (completedTaskId) => {
+        const node = graph.nodes.get(completedTaskId);
+        if (!node)
+            return;
+        for (const dependentId of node.dependents) {
+            const currentCount = waitingQueue.get(dependentId);
+            if (currentCount === undefined)
+                continue;
+            const newCount = currentCount - 1;
+            if (newCount > 0) {
+                waitingQueue.set(dependentId, newCount);
+                continue;
+            }
+            waitingQueue.delete(dependentId);
+            readyQueue.push(dependentId);
+        }
+    };
+    const allTasksFinished = () => {
+        const totalTasks = graph.nodes.size;
+        const finishedTasks = completedTasks.size + failedTasks.size + skippedTasks.size;
+        return finishedTasks === totalTasks;
+    };
+    /**
+     * Update execution status based on current queue state
+     */
+    const updateExecutionStatus = () => {
+        if (status === "failed" || status === "aborted") {
+            return; // Don't change from terminal states
+        }
+        if (allTasksFinished()) {
+            status = "completed";
+        }
+    };
+    return {
+        /**
+         * Get the next ready task for execution, respecting concurrency limits
+         * Returns null if pipeline has failed or been aborted
+         */
+        getNextReadyTask() {
+            // Don't return tasks if pipeline has failed or been aborted
+            if (status === "failed" || status === "aborted") {
+                return null;
+            }
+            if (readyQueue.length === 0)
+                return null;
+            if (runningTasks.size >= maxConcurrencyLimit)
+                return null;
+            // Additional check: if we have failed tasks, don't process new ones
+            if (failedTasks.size > 0) {
+                return null;
+            }
+            const taskId = readyQueue.shift() ?? null;
+            if (taskId) {
+                // Final check before returning - prevent race conditions
+                // Check failed tasks to prevent race conditions (status check already done above)
+                if (failedTasks.size > 0) {
+                    // Put it back if we detected failure
+                    readyQueue.unshift(taskId);
+                    return null;
+                }
+            }
+            return taskId;
+        },
+        /**
+         * Mark a running task as ready (via readyWhen) and update dependent tasks
+         */
+        markRunningTaskReady(taskId) {
+            if (runningTasks.has(taskId)) {
+                updateDependentTasks(taskId);
+            }
+        },
+        /**
+         * Mark a task as complete and update dependent tasks
+         */
+        markTaskComplete(taskId) {
+            runningTasks.delete(taskId);
+            completedTasks.add(taskId);
+            updateDependentTasks(taskId);
+            updateExecutionStatus();
+        },
+        /**
+         * Mark a task as failed
+         * When a task fails, we clear the ready queue immediately to prevent dependent tasks from starting.
+         * Note: This clears only the ready queue. The full cleanup (including waiting queue) happens
+         * in failPipeline via clearQueues(). This immediate ready queue clearing is critical to prevent
+         * race conditions where dependent tasks might be in the ready queue when a dependency fails.
+         * Failed tasks don't update dependents as they block the pipeline.
+         */
+        markTaskFailed(taskId) {
+            runningTasks.delete(taskId);
+            failedTasks.add(taskId);
+            status = "failed";
+            // Clear ready queue immediately to prevent any dependent tasks from starting
+            // This is critical to prevent race conditions where dependent tasks might
+            // be in the ready queue when a dependency fails
+            readyQueue.length = 0;
+        },
+        /**
+         * Mark a task as skipped and update dependent tasks
+         */
+        markTaskSkipped(taskId) {
+            runningTasks.delete(taskId);
+            skippedTasks.add(taskId);
+            updateDependentTasks(taskId);
+            updateExecutionStatus();
+        },
+        /**
+         * Mark a task as running
+         */
+        markTaskRunning(taskId, execution) {
+            runningTasks.set(taskId, execution);
+        },
+        /**
+         * Check if there are more tasks that can be executed
+         */
+        canExecuteMore() {
+            return readyQueue.length > 0 && runningTasks.size < maxConcurrencyLimit;
+        },
+        /**
+         * Check if all tasks are complete (either completed, failed, or skipped)
+         */
+        isComplete: allTasksFinished,
+        /**
+         * Check if any tasks have failed
+         */
+        hasFailed() {
+            return failedTasks.size > 0;
+        },
+        /**
+         * Get current running tasks
+         */
+        getRunningTasks() {
+            return new Map(runningTasks);
+        },
+        /**
+         * Clear all queues (used during cancellation)
+         * This prevents any pending tasks from starting, but preserves their status
+         * as "pending" in the task stats (they are not marked as aborted)
+         */
+        clearQueues() {
+            readyQueue.length = 0;
+            waitingQueue.clear();
+            // Keep running tasks for cleanup, but mark them for termination
+            // Note: Tasks in waitingQueue remain with "pending" status - they are not
+            // moved to failed/aborted state since they never started
+        },
+        /**
+         * Mark all running tasks as aborted and clear them from running tasks
+         * Used during cancellation to ensure proper state consistency
+         */
+        abortAllRunningTasks() {
+            // Move all running tasks to failed state
+            for (const taskId of runningTasks.keys()) {
+                failedTasks.add(taskId);
+            }
+            runningTasks.clear();
+            status = "aborted";
+        },
+    };
+}

package/dist/{modules → internal}/signal-handler.js

@@ -21,9 +21,12 @@ export function createSignalHandler({ abortSignal, onAborted, onProcessTerminate
     // Handle abort signal if provided
     let signalCleanup = null;
     if (abortSignal) {
-        abortSignal.addEventListener("abort", onAborted);
+        const handleAbort = () => {
+            onAborted();
+        };
+        abortSignal.addEventListener("abort", handleAbort);
         signalCleanup = () => {
-            abortSignal.removeEventListener("abort", onAborted);
+            abortSignal.removeEventListener("abort", handleAbort);
         };
     }
     return {

package/dist/internal/task-executor.d.ts

@@ -0,0 +1,33 @@
+import type { Task } from "../types.js";
+import type { ExecutionContext } from "./execution-context.js";
+/**
+ * Callbacks for task execution events that replace scheduler coordination.
+ * These callbacks are invoked by the task executor to notify the queue manager
+ * of task state changes, enabling queue-based execution flow.
+ */
+export interface TaskExecutionCallbacks {
+    /**
+     * Called when a task becomes ready (e.g., via readyWhen condition).
+     * This allows dependent tasks to start executing.
+     */
+    onTaskReady: (taskId: string) => void;
+    /**
+     * Called when a task completes successfully.
+     * Updates dependent task dependency counts and moves ready tasks to execution queue.
+     */
+    onTaskComplete: (taskId: string) => void;
+    /**
+     * Called when a task fails.
+     * Triggers pipeline failure and cleanup.
+     */
+    onTaskFailed: (taskId: string, error: Error) => void;
+    /**
+     * Called when a task is skipped (e.g., missing command in non-strict mode).
+     * Treated similarly to completion for dependency resolution.
+     */
+    onTaskSkipped: (taskId: string) => void;
+}
+/**
+ * Executes a task (either a regular task or a nested pipeline).
+ */
+export declare function executeTask(task: Task, context: ExecutionContext, callbacks: TaskExecutionCallbacks): void;