builderman 1.3.0 → 1.5.0

package/README.md

@@ -10,28 +10,30 @@ It is designed for monorepos, long-running development processes, and CI/CD pipe
 
 ## Table of Contents
 
-- [Key Features](#key-features)
-- [Installation](#installation)
-- [Quick Start](#quick-start)
-- [Core Concepts](#core-concepts)
-  - [Tasks](#tasks)
-  - [Commands & Modes](#commands--modes)
-  - [Dependencies](#dependencies)
-  - [Pipelines](#pipelines)
-  - [Pipeline Composition](#pipeline-composition)
-- [Error Handling Guarantees](#error-handling-guarantees)
-- [Cancellation](#cancellation)
-- [Teardown](#teardown)
-  - [Basic Teardown](#basic-teardown)
-  - [Teardown Callbacks](#teardown-callbacks)
-  - [Teardown Execution Rules](#teardown-execution-rules)
-- [Skipping Tasks](#skipping-tasks)
-  - [Strict Mode](#strict-mode)
-  - [Task-Level Skip Override](#task-level-skip-override)
-- [Execution Statistics](#execution-statistics)
-  - [Pipeline Statistics](#pipeline-statistics)
-  - [Task Statistics](#task-statistics)
-- [When Should I Use builderman?](#when-should-i-use-builderman)
+> - [Key Features](#key-features)
+> - [Installation](#installation)
+> - [Quick Start](#quick-start)
+> - [Core Concepts](#core-concepts)
+>   - [Tasks](#tasks)
+>   - [Commands & Modes](#commands--modes)
+>   - [Environment Variables](#environment-variables)
+>   - [Dependencies](#dependencies)
+>   - [Pipelines](#pipelines)
+>     - [Concurrency Control](#concurrency-control)
+>     - [Pipeline Composition](#pipeline-composition)
+> - [Error Handling Guarantees](#error-handling-guarantees)
+> - [Cancellation](#cancellation)
+> - [Teardown](#teardown)
+>   - [Basic Teardown](#basic-teardown)
+>   - [Teardown Callbacks](#teardown-callbacks)
+>   - [Teardown Execution Rules](#teardown-execution-rules)
+> - [Skipping Tasks](#skipping-tasks)
+>   - [Strict Mode](#strict-mode)
+>   - [Task-Level Skip Override](#task-level-skip-override)
+> - [Execution Statistics](#execution-statistics)
+>   - [Pipeline Statistics](#pipeline-statistics)
+>   - [Task Statistics](#task-statistics)
+> - [When Should I Use builderman?](#when-should-i-use-builderman)
 
 ## Key Features
 
@@ -62,17 +64,19 @@ import { task, pipeline } from "builderman"
 const build = task({
   name: "build",
   commands: { build: "tsc" },
-  cwd: ".",
+  cwd: "packages/my-package", // Optional: defaults to "."
 })
 
 const test = task({
   name: "test",
   commands: { build: "npm test" },
-  cwd: ".",
   dependencies: [build],
+  cwd: "packages/my-package",
 })
 
-const result = await pipeline([build, test]).run()
+const result = await pipeline([build, test]).run({
+  command: "build",
+})
 
 if (!result.ok) {
   console.error("Pipeline failed:", result.error.message)
@@ -93,6 +97,7 @@ A **task** represents a unit of work. Each task:
 - Defines commands for one or more modes
 - May depend on other tasks
 - May register teardown logic
+- Has an optional working directory (`cwd`, defaults to `"."`)
 
 ```ts
 import { task } from "builderman"
@@ -130,6 +135,89 @@ Commands may be:
   - `run`: the command to execute
   - `readyWhen`: a predicate that marks the task as ready
   - `teardown`: cleanup logic to run after completion
+  - `env`: environment variables specific to this command
+
+---
+
+### Environment Variables
+
+Environment variables can be provided at multiple levels, with more specific levels overriding less specific ones:
+
+**Precedence order (highest to lowest):**
+
+1. Command-level `env` (in command config)
+2. Task-level `env` (in task config)
+3. Pipeline-level `env` (in `pipeline.run()`)
+4. Process environment variables
+
+#### Command-Level Environment Variables
+
+```ts
+const apiTask = task({
+  name: "api",
+  commands: {
+    dev: {
+      run: "npm run dev",
+      env: {
+        PORT: "3000",
+        NODE_ENV: "development",
+      },
+    },
+  },
+})
+```
+
+#### Task-Level Environment Variables
+
+```ts
+const apiTask = task({
+  name: "api",
+  commands: {
+    dev: "npm run dev",
+    build: "npm run build",
+  },
+  env: {
+    API_URL: "http://localhost:3000",
+    LOG_LEVEL: "debug",
+  },
+})
+```
+
+#### Pipeline-Level Environment Variables
+
+```ts
+const result = await pipeline([apiTask]).run({
+  env: {
+    DATABASE_URL: "postgres://localhost/mydb",
+    REDIS_URL: "redis://localhost:6379",
+  },
+})
+```
+
+#### Nested Pipeline Environment Variables
+
+When converting a pipeline to a task, you can provide environment variables that will be merged with the outer pipeline's environment:
+
+```ts
+const innerPipeline = pipeline([
+  /* ... */
+])
+const innerTask = innerPipeline.toTask({
+  name: "inner",
+  env: {
+    INNER_VAR: "inner-value",
+  },
+})
+
+const outerPipeline = pipeline([innerTask])
+const result = await outerPipeline.run({
+  env: {
+    OUTER_VAR: "outer-value",
+  },
+})
+```
+
+In this example, tasks in `innerPipeline` will receive both `INNER_VAR` and `OUTER_VAR`, with `INNER_VAR` taking precedence if there's a conflict.
 
 ---
 
@@ -169,6 +257,31 @@ const result = await pipeline([libTask, consumerTask]).run({
 })
 ```
 
+#### Concurrency Control
+
+By default, pipelines run as many tasks concurrently as possible (limited only by dependencies). You can limit concurrent execution using `maxConcurrency`:
+
+```ts
+const result = await pipeline([task1, task2, task3, task4, task5]).run({
+  maxConcurrency: 2, // At most 2 tasks will run simultaneously
+})
+```
+
+When `maxConcurrency` is set:
+
+- Tasks that are ready to run (dependencies satisfied) will start up to the limit
+- As tasks complete, new ready tasks will start to maintain the concurrency limit
+- Dependencies are still respected — a task won't start until its dependencies complete
+
+This is useful for:
+
+- Limiting resource usage (CPU, memory, network)
+- Controlling database connection pools
+- Managing API rate limits
+- Reducing system load in CI environments
+
+If `maxConcurrency` is not specified, there is no limit (tasks run concurrently as dependencies allow).
+
 ---
 
 ### Pipeline Composition
@@ -187,7 +300,11 @@ const deploy = pipeline([
 ])
 
 const buildTask = build.toTask({ name: "build" })
-const testTask = test.toTask({ name: "test", dependencies: [buildTask] })
+const testTask = test.toTask({
+  name: "test",
+  dependencies: [buildTask],
+  env: { TEST_ENV: "test-value" }, // Optional: env for nested pipeline
+})
 const deployTask = deploy.toTask({ name: "deploy", dependencies: [testTask] })
 
 const ci = pipeline([buildTask, testTask, deployTask])
@@ -215,16 +332,19 @@ if (!result.ok) {
       console.error("Pipeline was cancelled")
       break
     case PipelineError.TaskFailed:
-      console.error(`Task failed: ${result.error.message}`)
+      console.error("Task failed:", result.error.message)
+      break
+    case PipelineError.TaskReadyTimeout:
+      console.error("Task was not ready in time:", result.error.message)
+      break
+    case PipelineError.TaskCompletedTimeout:
+      console.error("Task did not complete in time:", result.error.message)
       break
     case PipelineError.ProcessTerminated:
-      console.error("Process was terminated")
+      console.error("Process terminated:", result.error.message)
       break
     case PipelineError.InvalidTask:
-      console.error(`Invalid task configuration: ${result.error.message}`)
-      break
-    case PipelineError.InvalidSignal:
-      console.error("Invalid abort signal")
+      console.error("Invalid task configuration:", result.error.message)
       break
   }
 }
@@ -278,7 +398,6 @@ const dbTask = task({
     },
     build: "echo build",
   },
-  cwd: ".",
 })
 ```
 
@@ -336,7 +455,6 @@ const dbTask = task({
   commands: {
     dev: "docker-compose up",
   },
-  cwd: ".",
 })
 
 const apiTask = task({
@@ -345,7 +463,6 @@ const apiTask = task({
     dev: "npm run dev",
     build: "npm run build",
   },
-  cwd: ".",
   dependencies: [dbTask],
 })
 
@@ -386,7 +503,6 @@ const dbTask = task({
   commands: {
     dev: "docker-compose up",
   },
-  cwd: ".",
   allowSkip: true,
 })
 

package/dist/errors.d.ts

@@ -0,0 +1,12 @@
+export type PipelineErrorCode = typeof PipelineError.Aborted | typeof PipelineError.ProcessTerminated | typeof PipelineError.TaskFailed | typeof PipelineError.TaskCompletedTimeout | typeof PipelineError.TaskReadyTimeout | typeof PipelineError.InvalidTask;
+export declare class PipelineError extends Error {
+    readonly code: PipelineErrorCode;
+    readonly taskName?: string;
+    constructor(message: string, code: PipelineErrorCode, taskName?: string);
+    static Aborted: "aborted";
+    static ProcessTerminated: "process-terminated";
+    static TaskFailed: "task-failed";
+    static TaskReadyTimeout: "task-ready-timeout";
+    static TaskCompletedTimeout: "task-completed-timeout";
+    static InvalidTask: "invalid-task";
+}

package/dist/{pipeline-error.js → errors.js}

@@ -22,29 +22,35 @@ Object.defineProperty(PipelineError, "Aborted", {
     enumerable: true,
     configurable: true,
     writable: true,
-    value: 0
+    value: "aborted"
 });
 Object.defineProperty(PipelineError, "ProcessTerminated", {
     enumerable: true,
     configurable: true,
     writable: true,
-    value: 1
+    value: "process-terminated"
 });
 Object.defineProperty(PipelineError, "TaskFailed", {
     enumerable: true,
     configurable: true,
     writable: true,
-    value: 2
+    value: "task-failed"
 });
-Object.defineProperty(PipelineError, "InvalidSignal", {
+Object.defineProperty(PipelineError, "TaskReadyTimeout", {
     enumerable: true,
     configurable: true,
     writable: true,
-    value: 3
+    value: "task-ready-timeout"
+});
+Object.defineProperty(PipelineError, "TaskCompletedTimeout", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: "task-completed-timeout"
 });
 Object.defineProperty(PipelineError, "InvalidTask", {
     enumerable: true,
     configurable: true,
     writable: true,
-    value: 4
+    value: "invalid-task"
 });

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 export { task } from "./task.js";
 export { pipeline } from "./pipeline.js";
-export { PipelineError, type PipelineErrorCode } from "./pipeline-error.js";
+export { PipelineError, type PipelineErrorCode } from "./errors.js";
 export type { Task, Pipeline, TaskConfig, Command, CommandConfig, Commands, PipelineRunConfig, PipelineTaskConfig, RunResult, PipelineStats, TaskStats, TaskStatus, } from "./types.js";

package/dist/index.js

@@ -1,3 +1,3 @@
 export { task } from "./task.js";
 export { pipeline } from "./pipeline.js";
-export { PipelineError } from "./pipeline-error.js";
+export { PipelineError } from "./errors.js";

package/dist/internal/constants.d.ts

@@ -0,0 +1,2 @@
+export declare const $TASK_INTERNAL: unique symbol;
+export declare const $PIPELINE_INTERNAL: unique symbol;

package/dist/internal/constants.js

@@ -0,0 +1,2 @@
+export const $TASK_INTERNAL = Symbol("task-internal");
+export const $PIPELINE_INTERNAL = Symbol("pipeline-internal");

package/dist/internal/execution-context.d.ts

@@ -0,0 +1,33 @@
+import type { PipelineRunConfig, TaskStats } from "../types.js";
+import type { TeardownManager } from "./teardown-manager.js";
+import type { TimeoutManager } from "./timeout-manager.js";
+import type { QueueManager } from "./queue-manager.js";
+/**
+ * Represents a task execution in progress
+ */
+export interface TaskExecution {
+    taskId: string;
+    taskName: string;
+    process?: import("node:child_process").ChildProcess;
+    startedAt: number;
+    readyAt?: number;
+}
+/**
+ * Centralized execution context that replaces scattered configuration parameters
+ * and provides unified access to execution state and helper methods
+ */
+export interface ExecutionContext {
+    config: PipelineRunConfig;
+    signal?: AbortSignal;
+    spawn: typeof import("node:child_process").spawn;
+    teardownManager: TeardownManager;
+    timeoutManager: TimeoutManager;
+    queueManager: QueueManager;
+    taskStats: Map<string, TaskStats>;
+    updateTaskStatus: (taskId: string, updates: Partial<TaskStats>) => void;
+    isAborted: () => boolean;
+}
+/**
+ * Creates an execution context for pipeline execution
+ */
+export declare function createExecutionContext(config: PipelineRunConfig, teardownManager: TeardownManager, timeoutManager: TimeoutManager, queueManager: QueueManager, taskStats: Map<string, TaskStats>): ExecutionContext;

package/dist/internal/execution-context.js

@@ -0,0 +1,30 @@
+import { spawn } from "node:child_process";
+/**
+ * Creates an execution context for pipeline execution
+ */
+export function createExecutionContext(config, teardownManager, timeoutManager, queueManager, taskStats) {
+    // Use dynamic import only if spawn is not provided
+    const spawnFn = config.spawn ?? spawn;
+    return {
+        config,
+        signal: config.signal,
+        spawn: spawnFn,
+        teardownManager,
+        timeoutManager,
+        queueManager,
+        taskStats,
+        updateTaskStatus(taskId, updates) {
+            const currentStats = taskStats.get(taskId);
+            const updatedStats = { ...currentStats, ...updates };
+            // Calculate duration if both start and finish times are available
+            if (updatedStats.startedAt && updatedStats.finishedAt) {
+                updatedStats.durationMs =
+                    updatedStats.finishedAt - updatedStats.startedAt;
+            }
+            taskStats.set(taskId, updatedStats);
+        },
+        isAborted() {
+            return config.signal?.aborted ?? false;
+        },
+    };
+}

package/dist/{graph.d.ts → internal/graph.d.ts}

@@ -1,2 +1,2 @@
-import type { TaskGraph, Task } from "./types.js";
+import type { TaskGraph, Task } from "../types.js";
 export declare function createTaskGraph(tasks: Task[]): TaskGraph;

package/dist/internal/queue-manager.d.ts

@@ -0,0 +1,24 @@
+import type { TaskGraph } from "../types.js";
+import type { TaskExecution } from "./execution-context.js";
+/**
+ * Queue manager interface returned by createQueueManager
+ */
+export interface QueueManager {
+    getNextReadyTask(): string | null;
+    markRunningTaskReady(taskId: string): void;
+    markTaskComplete(taskId: string): void;
+    markTaskFailed(taskId: string): void;
+    markTaskSkipped(taskId: string): void;
+    markTaskRunning(taskId: string, execution: TaskExecution): void;
+    canExecuteMore(): boolean;
+    isComplete(): boolean;
+    hasFailed(): boolean;
+    getRunningTasks(): Map<string, TaskExecution>;
+    clearQueues(): void;
+    abortAllRunningTasks(): void;
+}
+/**
+ * Creates a queue manager that replaces the generator-based scheduler
+ * with explicit queue-based execution state management
+ */
+export declare function createQueueManager(graph: TaskGraph, maxConcurrency?: number): QueueManager;

package/dist/internal/queue-manager.js

@@ -0,0 +1,182 @@
+/**
+ * Creates a queue manager that replaces the generator-based scheduler
+ * with explicit queue-based execution state management
+ */
+export function createQueueManager(graph, maxConcurrency) {
+    const readyQueue = [];
+    const waitingQueue = new Map();
+    const runningTasks = new Map();
+    const completedTasks = new Set();
+    const failedTasks = new Set();
+    const skippedTasks = new Set();
+    const maxConcurrencyLimit = maxConcurrency ?? Infinity;
+    let status = "running";
+    for (const [taskId, node] of graph.nodes) {
+        const depCount = node.dependencies.size;
+        if (depCount === 0) {
+            readyQueue.push(taskId);
+        }
+        else {
+            waitingQueue.set(taskId, depCount);
+        }
+    }
+    const updateDependentTasks = (completedTaskId) => {
+        const node = graph.nodes.get(completedTaskId);
+        if (!node)
+            return;
+        for (const dependentId of node.dependents) {
+            const currentCount = waitingQueue.get(dependentId);
+            if (currentCount === undefined)
+                continue;
+            const newCount = currentCount - 1;
+            if (newCount > 0) {
+                waitingQueue.set(dependentId, newCount);
+                continue;
+            }
+            waitingQueue.delete(dependentId);
+            readyQueue.push(dependentId);
+        }
+    };
+    const allTasksFinished = () => {
+        const totalTasks = graph.nodes.size;
+        const finishedTasks = completedTasks.size + failedTasks.size + skippedTasks.size;
+        return finishedTasks === totalTasks;
+    };
+    /**
+     * Update execution status based on current queue state
+     */
+    const updateExecutionStatus = () => {
+        if (status === "failed" || status === "aborted") {
+            return; // Don't change from terminal states
+        }
+        if (allTasksFinished()) {
+            status = "completed";
+        }
+    };
+    return {
+        /**
+         * Get the next ready task for execution, respecting concurrency limits
+         * Returns null if pipeline has failed or been aborted
+         */
+        getNextReadyTask() {
+            // Don't return tasks if pipeline has failed or been aborted
+            if (status === "failed" || status === "aborted") {
+                return null;
+            }
+            if (readyQueue.length === 0)
+                return null;
+            if (runningTasks.size >= maxConcurrencyLimit)
+                return null;
+            // Additional check: if we have failed tasks, don't process new ones
+            if (failedTasks.size > 0) {
+                return null;
+            }
+            const taskId = readyQueue.shift() ?? null;
+            if (taskId) {
+                // Final check before returning - prevent race conditions
+                // Check failed tasks to prevent race conditions (status check already done above)
+                if (failedTasks.size > 0) {
+                    // Put it back if we detected failure
+                    readyQueue.unshift(taskId);
+                    return null;
+                }
+            }
+            return taskId;
+        },
+        /**
+         * Mark a running task as ready (via readyWhen) and update dependent tasks
+         */
+        markRunningTaskReady(taskId) {
+            if (runningTasks.has(taskId)) {
+                updateDependentTasks(taskId);
+            }
+        },
+        /**
+         * Mark a task as complete and update dependent tasks
+         */
+        markTaskComplete(taskId) {
+            runningTasks.delete(taskId);
+            completedTasks.add(taskId);
+            updateDependentTasks(taskId);
+            updateExecutionStatus();
+        },
+        /**
+         * Mark a task as failed
+         * When a task fails, we clear the ready queue immediately to prevent dependent tasks from starting.
+         * Note: This clears only the ready queue. The full cleanup (including waiting queue) happens
+         * in failPipeline via clearQueues(). This immediate ready queue clearing is critical to prevent
+         * race conditions where dependent tasks might be in the ready queue when a dependency fails.
+         * Failed tasks don't update dependents as they block the pipeline.
+         */
+        markTaskFailed(taskId) {
+            runningTasks.delete(taskId);
+            failedTasks.add(taskId);
+            status = "failed";
+            // Clear ready queue immediately to prevent any dependent tasks from starting
+            // This is critical to prevent race conditions where dependent tasks might
+            // be in the ready queue when a dependency fails
+            readyQueue.length = 0;
+        },
+        /**
+         * Mark a task as skipped and update dependent tasks
+         */
+        markTaskSkipped(taskId) {
+            runningTasks.delete(taskId);
+            skippedTasks.add(taskId);
+            updateDependentTasks(taskId);
+            updateExecutionStatus();
+        },
+        /**
+         * Mark a task as running
+         */
+        markTaskRunning(taskId, execution) {
+            runningTasks.set(taskId, execution);
+        },
+        /**
+         * Check if there are more tasks that can be executed
+         */
+        canExecuteMore() {
+            return readyQueue.length > 0 && runningTasks.size < maxConcurrencyLimit;
+        },
+        /**
+         * Check if all tasks are complete (either completed, failed, or skipped)
+         */
+        isComplete: allTasksFinished,
+        /**
+         * Check if any tasks have failed
+         */
+        hasFailed() {
+            return failedTasks.size > 0;
+        },
+        /**
+         * Get current running tasks
+         */
+        getRunningTasks() {
+            return new Map(runningTasks);
+        },
+        /**
+         * Clear all queues (used during cancellation)
+         * This prevents any pending tasks from starting, but preserves their status
+         * as "pending" in the task stats (they are not marked as aborted)
+         */
+        clearQueues() {
+            readyQueue.length = 0;
+            waitingQueue.clear();
+            // Keep running tasks for cleanup, but mark them for termination
+            // Note: Tasks in waitingQueue remain with "pending" status - they are not
+            // moved to failed/aborted state since they never started
+        },
+        /**
+         * Mark all running tasks as aborted and clear them from running tasks
+         * Used during cancellation to ensure proper state consistency
+         */
+        abortAllRunningTasks() {
+            // Move all running tasks to failed state
+            for (const taskId of runningTasks.keys()) {
+                failedTasks.add(taskId);
+            }
+            runningTasks.clear();
+            status = "aborted";
+        },
+    };
+}

package/dist/{modules → internal}/signal-handler.js

@@ -21,9 +21,12 @@ export function createSignalHandler({ abortSignal, onAborted, onProcessTerminate
     // Handle abort signal if provided
     let signalCleanup = null;
     if (abortSignal) {
-        abortSignal.addEventListener("abort", onAborted);
+        const handleAbort = () => {
+            onAborted();
+        };
+        abortSignal.addEventListener("abort", handleAbort);
         signalCleanup = () => {
-            abortSignal.removeEventListener("abort", onAborted);
+            abortSignal.removeEventListener("abort", handleAbort);
         };
     }
     return {

package/dist/internal/task-executor.d.ts

@@ -0,0 +1,33 @@
+import type { Task } from "../types.js";
+import type { ExecutionContext } from "./execution-context.js";
+/**
+ * Callbacks for task execution events that replace scheduler coordination.
+ * These callbacks are invoked by the task executor to notify the queue manager
+ * of task state changes, enabling queue-based execution flow.
+ */
+export interface TaskExecutionCallbacks {
+    /**
+     * Called when a task becomes ready (e.g., via readyWhen condition).
+     * This allows dependent tasks to start executing.
+     */
+    onTaskReady: (taskId: string) => void;
+    /**
+     * Called when a task completes successfully.
+     * Updates dependent task dependency counts and moves ready tasks to execution queue.
+     */
+    onTaskComplete: (taskId: string) => void;
+    /**
+     * Called when a task fails.
+     * Triggers pipeline failure and cleanup.
+     */
+    onTaskFailed: (taskId: string, error: Error) => void;
+    /**
+     * Called when a task is skipped (e.g., missing command in non-strict mode).
+     * Treated similarly to completion for dependency resolution.
+     */
+    onTaskSkipped: (taskId: string) => void;
+}
+/**
+ * Executes a task (either a regular task or a nested pipeline).
+ */
+export declare function executeTask(task: Task, context: ExecutionContext, callbacks: TaskExecutionCallbacks): void;