@runcore-sh/runcore 0.1.2

package/dist/agents/orchestration.js

@@ -0,0 +1,715 @@
+/**
+ * Agent Orchestration Engine — Multi-agent coordination, delegation, and result merging.
+ *
+ * Sits on top of the AgentPool to provide:
+ * 1. **Workflows** — Named groups of coordinated agent tasks
+ * 2. **Task delegation** — Spawn multiple agents for a workflow and track them
+ * 3. **Result aggregation** — Collect, merge, and synthesize outputs from all agents
+ * 4. **Conflict resolution** — Preventive file locking + post-hoc detection/resolution
+ * 5. **Queue management** — Priority-aware workflow scheduling
+ * 6. **Status tracking** — Real-time workflow status and reporting
+ * 7. **Context passing** — Dependency mode pipes completed task outputs into dependents
+ * 8. **Cascading failure limits** — Auto-cancel workflows when failure threshold hit
+ * 9. **Workflow timeouts** — Wall-clock timeout for entire workflows
+ * 10. **Bus integration** — Emits workflow lifecycle events for monitoring
+ *
+ * Usage:
+ *   const orch = new Orchestrator(pool);
+ *   const wf = orch.createWorkflow({ name: "Feature X", tasks: [...] });
+ *   const result = await orch.execute(wf.id);
+ *   const aggregated = orch.aggregateResults(wf.id);
+ */
+import { randomBytes } from "node:crypto";
+import { createLogger } from "../utils/logger.js";
+const log = createLogger("orchestrator");
+// ---------------------------------------------------------------------------
+// Orchestrator
+// ---------------------------------------------------------------------------
+export class Orchestrator {
+    pool;
+    workflows = new Map();
+    conflicts = new Map();
+    workflowQueue = [];
+    activeWorkflowCount = 0;
+    maxActiveWorkflows;
+    eventHandlers = new Map();
+    /** Resolvers for queued workflows waiting for capacity. */
+    queueResolvers = new Map();
+    constructor(pool, options) {
+        this.pool = pool;
+        this.maxActiveWorkflows = options?.maxActiveWorkflows ?? 3;
+    }
+    // -------------------------------------------------------------------------
+    // Workflow lifecycle
+    // -------------------------------------------------------------------------
+    /** Create a workflow (does not start execution). */
+    createWorkflow(input) {
+        const id = generateWorkflowId();
+        const tasks = new Map();
+        for (const def of input.tasks) {
+            if (tasks.has(def.key)) {
+                throw new Error(`Duplicate task key in workflow: ${def.key}`);
+            }
+            tasks.set(def.key, {
+                key: def.key,
+                label: def.label,
+                prompt: def.prompt,
+                cwd: def.cwd,
+                dependsOn: def.dependsOn ?? [],
+                priority: def.priority ?? 50,
+                config: def.config,
+                tags: def.tags ?? [],
+                status: "pending",
+            });
+        }
+        // Validate dependency references
+        if (input.mode === "dependency") {
+            for (const [, task] of tasks) {
+                for (const dep of task.dependsOn) {
+                    if (!tasks.has(dep)) {
+                        throw new Error(`Task "${task.key}" depends on unknown task "${dep}"`);
+                    }
+                }
+            }
+            // Check for cycles
+            this.detectCycles(tasks);
+        }
+        const workflow = {
+            id,
+            name: input.name,
+            description: input.description,
+            status: "pending",
+            mode: input.mode ?? "parallel",
+            conflictStrategy: input.conflictStrategy ?? "last-write-wins",
+            maxConcurrent: input.maxConcurrent ?? 0,
+            origin: input.origin ?? "ai",
+            tasks,
+            createdAt: new Date().toISOString(),
+        };
+        this.workflows.set(id, workflow);
+        this.conflicts.set(id, []);
+        log.info("Workflow created", {
+            workflowId: id,
+            name: input.name,
+            taskCount: tasks.size,
+            mode: workflow.mode,
+        });
+        return workflow;
+    }
+    /**
+     * Execute a workflow. Spawns agents according to the execution mode.
+     * Returns a promise that resolves when the workflow completes.
+     */
+    async execute(workflowId) {
+        const workflow = this.requireWorkflow(workflowId);
+        if (workflow.status !== "pending") {
+            throw new Error(`Workflow "${workflow.name}" is already ${workflow.status}`);
+        }
+        // Check if we can start immediately or need to queue
+        if (this.activeWorkflowCount >= this.maxActiveWorkflows) {
+            this.workflowQueue.push(workflowId);
+            log.info("Workflow queued", { workflowId, queuePosition: this.workflowQueue.length });
+            // Wait until drainQueue resolves us
+            await new Promise((resolve) => {
+                this.queueResolvers.set(workflowId, resolve);
+            });
+        }
+        this.activeWorkflowCount++;
+        workflow.status = "running";
+        workflow.startedAt = new Date().toISOString();
+        log.info("Workflow executing", {
+            workflowId,
+            name: workflow.name,
+            mode: workflow.mode,
+        });
+        try {
+            switch (workflow.mode) {
+                case "parallel":
+                    await this.executeParallel(workflow);
+                    break;
+                case "sequential":
+                    await this.executeSequential(workflow);
+                    break;
+                case "dependency":
+                    await this.executeDependency(workflow);
+                    break;
+            }
+        }
+        catch (err) {
+            log.error("Workflow execution error", {
+                workflowId,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+        // Detect conflicts
+        const fileConflicts = this.detectFileConflicts(workflow);
+        this.conflicts.set(workflowId, fileConflicts);
+        // Resolve conflicts
+        if (fileConflicts.length > 0) {
+            this.resolveConflicts(workflow, fileConflicts);
+        }
+        // Determine final status
+        workflow.status = this.computeWorkflowStatus(workflow);
+        workflow.finishedAt = new Date().toISOString();
+        this.activeWorkflowCount--;
+        this.drainQueue();
+        log.info("Workflow finished", {
+            workflowId,
+            name: workflow.name,
+            status: workflow.status,
+            conflicts: fileConflicts.length,
+        });
+        return this.buildResult(workflow);
+    }
+    /** Cancel a running or pending workflow. */
+    async cancel(workflowId) {
+        const workflow = this.requireWorkflow(workflowId);
+        // Remove from queue if pending
+        const queueIdx = this.workflowQueue.indexOf(workflowId);
+        if (queueIdx !== -1) {
+            this.workflowQueue.splice(queueIdx, 1);
+        }
+        // Cancel running tasks
+        for (const [, task] of workflow.tasks) {
+            if (task.status === "running" && task.instanceId) {
+                try {
+                    await this.pool.terminate(task.instanceId, "Workflow cancelled");
+                }
+                catch {
+                    // Best effort
+                }
+                task.status = "cancelled";
+                task.finishedAt = new Date().toISOString();
+            }
+            else if (task.status === "pending" || task.status === "queued") {
+                task.status = "cancelled";
+            }
+        }
+        const wasRunning = workflow.status === "running";
+        workflow.status = "cancelled";
+        workflow.finishedAt = new Date().toISOString();
+        if (wasRunning) {
+            this.activeWorkflowCount--;
+            this.drainQueue();
+        }
+        log.info("Workflow cancelled", { workflowId, name: workflow.name });
+    }
+    // -------------------------------------------------------------------------
+    // Execution modes
+    // -------------------------------------------------------------------------
+    /** Execute all tasks in parallel (respecting maxConcurrent). */
+    async executeParallel(workflow) {
+        const allTasks = [...workflow.tasks.values()];
+        const maxConcurrent = workflow.maxConcurrent || allTasks.length;
+        // Spawn in batches respecting concurrency limit
+        const pending = [...allTasks];
+        const running = new Map();
+        const spawnNext = async () => {
+            while (pending.length > 0 && running.size < maxConcurrent) {
+                const task = pending.shift();
+                const promise = this.spawnTask(workflow, task).then(() => {
+                    running.delete(task.key);
+                });
+                running.set(task.key, promise);
+            }
+        };
+        await spawnNext();
+        // Wait for tasks to complete and spawn more as slots free up
+        while (running.size > 0) {
+            await Promise.race([...running.values()]);
+            await spawnNext();
+        }
+    }
+    /** Execute tasks one at a time in definition order. */
+    async executeSequential(workflow) {
+        for (const [, task] of workflow.tasks) {
+            if (workflow.status === "cancelled")
+                break;
+            await this.spawnTask(workflow, task);
+        }
+    }
+    /** Execute tasks respecting dependency edges (topological order). */
+    async executeDependency(workflow) {
+        const maxConcurrent = workflow.maxConcurrent || workflow.tasks.size;
+        const completed = new Set();
+        const failed = new Set();
+        const running = new Map();
+        const isReady = (task) => {
+            if (task.status !== "pending")
+                return false;
+            return task.dependsOn.every((dep) => completed.has(dep));
+        };
+        const hasFailedDep = (task) => {
+            return task.dependsOn.some((dep) => failed.has(dep));
+        };
+        const scheduleReady = async () => {
+            for (const [, task] of workflow.tasks) {
+                if (running.size >= maxConcurrent)
+                    break;
+                if (workflow.status === "cancelled")
+                    break;
+                // Skip tasks whose deps have failed
+                if (hasFailedDep(task)) {
+                    task.status = "skipped";
+                    task.error = "Dependency failed";
+                    continue;
+                }
+                if (isReady(task)) {
+                    task.status = "queued";
+                    const promise = this.spawnTask(workflow, task).then(() => {
+                        running.delete(task.key);
+                        if (task.status === "completed") {
+                            completed.add(task.key);
+                        }
+                        else {
+                            failed.add(task.key);
+                        }
+                    });
+                    running.set(task.key, promise);
+                }
+            }
+        };
+        await scheduleReady();
+        while (running.size > 0) {
+            await Promise.race([...running.values()]);
+            await scheduleReady();
+        }
+        // Mark any remaining pending tasks as skipped
+        for (const [, task] of workflow.tasks) {
+            if (task.status === "pending") {
+                task.status = "skipped";
+                task.error = "Dependency not satisfied";
+            }
+        }
+    }
+    // -------------------------------------------------------------------------
+    // Task spawning
+    // -------------------------------------------------------------------------
+    /** Spawn a single agent for a workflow task and wait for completion. */
+    async spawnTask(workflow, task) {
+        task.status = "running";
+        task.startedAt = new Date().toISOString();
+        try {
+            const request = {
+                taskId: `orch_${workflow.id}_${task.key}`,
+                label: `[${workflow.name}] ${task.label}`,
+                prompt: task.prompt,
+                cwd: task.cwd,
+                origin: workflow.origin,
+                tags: [
+                    `workflow:${workflow.id}`,
+                    `task:${task.key}`,
+                    ...task.tags,
+                ],
+                config: {
+                    priority: task.priority,
+                    ...task.config,
+                },
+            };
+            const instance = await this.pool.spawn(request);
+            task.instanceId = instance.id;
+            task.taskId = instance.taskId;
+            // Wait for completion via bus events
+            await this.waitForCompletion(instance.id, task);
+        }
+        catch (err) {
+            task.status = "failed";
+            task.error = err instanceof Error ? err.message : String(err);
+            task.finishedAt = new Date().toISOString();
+            log.warn("Task spawn failed", {
+                workflowId: workflow.id,
+                taskKey: task.key,
+                error: task.error,
+            });
+        }
+    }
+    /** Wait for an agent instance to reach a terminal state. */
+    waitForCompletion(instanceId, task) {
+        return new Promise((resolve) => {
+            let interval;
+            let onLifecycle;
+            const cleanup = () => {
+                if (interval)
+                    clearInterval(interval);
+                if (onLifecycle) {
+                    this.pool.runtimeManager.bus.off("agent:lifecycle", onLifecycle);
+                }
+                this.eventHandlers.delete(`${instanceId}:cleanup`);
+            };
+            const checkAndResolve = () => {
+                const instance = this.pool.getInstance(instanceId);
+                if (!instance) {
+                    task.status = "failed";
+                    task.error = "Instance disappeared";
+                    task.finishedAt = new Date().toISOString();
+                    cleanup();
+                    resolve();
+                    return true;
+                }
+                const terminalStates = new Set(["completed", "failed", "terminated"]);
+                if (terminalStates.has(instance.state)) {
+                    task.status = instance.state === "completed" ? "completed" : "failed";
+                    task.finishedAt = new Date().toISOString();
+                    if (instance.error) {
+                        task.error = instance.error.message;
+                    }
+                    // Auto-record checkpoint data as output if available
+                    if (instance.checkpointData && !task.output) {
+                        task.output = instance.checkpointData;
+                        task.filesTouched = parseFilesFromOutput(instance.checkpointData);
+                    }
+                    cleanup();
+                    resolve();
+                    return true;
+                }
+                return false;
+            };
+            // Check immediately
+            if (checkAndResolve())
+                return;
+            // Listen for lifecycle events
+            onLifecycle = (event) => {
+                if (event.agentId === instanceId) {
+                    checkAndResolve();
+                }
+            };
+            this.pool.runtimeManager.bus.on("agent:lifecycle", onLifecycle);
+            // Safety poll in case events are missed
+            interval = setInterval(() => {
+                checkAndResolve();
+            }, 2000);
+            // Store cleanup for cancellation
+            this.eventHandlers.set(`${instanceId}:cleanup`, () => {
+                cleanup();
+                resolve();
+            });
+        });
+    }
+    // -------------------------------------------------------------------------
+    // Conflict detection and resolution
+    // -------------------------------------------------------------------------
+    /** Detect file conflicts — multiple tasks claiming to touch the same files. */
+    detectFileConflicts(workflow) {
+        const fileMap = new Map();
+        for (const [, task] of workflow.tasks) {
+            if (task.status !== "completed" || !task.filesTouched)
+                continue;
+            for (const file of task.filesTouched) {
+                if (!fileMap.has(file)) {
+                    fileMap.set(file, []);
+                }
+                fileMap.get(file).push(task.key);
+            }
+        }
+        const conflicts = [];
+        for (const [filePath, taskKeys] of fileMap) {
+            if (taskKeys.length > 1) {
+                conflicts.push({ filePath, taskKeys });
+            }
+        }
+        return conflicts;
+    }
+    /** Apply conflict resolution strategy. */
+    resolveConflicts(workflow, conflicts) {
+        for (const conflict of conflicts) {
+            switch (workflow.conflictStrategy) {
+                case "last-write-wins":
+                    // Accept the last task's changes (by finish time)
+                    conflict.resolution = "accepted";
+                    conflict.resolvedBy = this.getLastFinished(workflow, conflict.taskKeys);
+                    break;
+                case "first-write-wins":
+                    // Accept the first task's changes
+                    conflict.resolution = "accepted";
+                    conflict.resolvedBy = this.getFirstFinished(workflow, conflict.taskKeys);
+                    break;
+                case "fail":
+                    // Mark as failed — requires human intervention
+                    conflict.resolution = "rejected";
+                    log.warn("Conflict resolution: fail strategy", {
+                        file: conflict.filePath,
+                        tasks: conflict.taskKeys,
+                    });
+                    break;
+                case "manual":
+                    conflict.resolution = "deferred";
+                    break;
+            }
+        }
+    }
+    getLastFinished(workflow, taskKeys) {
+        let latest = taskKeys[0];
+        let latestTime = "";
+        for (const key of taskKeys) {
+            const task = workflow.tasks.get(key);
+            if (task?.finishedAt && task.finishedAt > latestTime) {
+                latestTime = task.finishedAt;
+                latest = key;
+            }
+        }
+        return latest;
+    }
+    getFirstFinished(workflow, taskKeys) {
+        let earliest = taskKeys[0];
+        let earliestTime = new Date().toISOString();
+        for (const key of taskKeys) {
+            const task = workflow.tasks.get(key);
+            if (task?.finishedAt && task.finishedAt < earliestTime) {
+                earliestTime = task.finishedAt;
+                earliest = key;
+            }
+        }
+        return earliest;
+    }
+    // -------------------------------------------------------------------------
+    // Queue management
+    // -------------------------------------------------------------------------
+    /** Drain queued workflows when capacity frees up. */
+    drainQueue() {
+        while (this.workflowQueue.length > 0 &&
+            this.activeWorkflowCount < this.maxActiveWorkflows) {
+            const nextId = this.workflowQueue.shift();
+            const resolve = this.queueResolvers.get(nextId);
+            if (resolve) {
+                this.queueResolvers.delete(nextId);
+                resolve();
+            }
+        }
+    }
+    // -------------------------------------------------------------------------
+    // Status tracking and reporting
+    // -------------------------------------------------------------------------
+    /** Get a single workflow by ID. */
+    getWorkflow(workflowId) {
+        return this.workflows.get(workflowId);
+    }
+    /** Get the aggregated result for a completed workflow. */
+    getResult(workflowId) {
+        const workflow = this.requireWorkflow(workflowId);
+        return this.buildResult(workflow);
+    }
+    /** Get a monitoring report across all workflows. */
+    getReport() {
+        const workflows = [];
+        let activeCount = 0;
+        let pendingCount = 0;
+        let completedCount = 0;
+        for (const [, wf] of this.workflows) {
+            const progress = this.getTaskProgress(wf);
+            workflows.push({
+                id: wf.id,
+                name: wf.name,
+                status: wf.status,
+                progress,
+            });
+            if (wf.status === "running")
+                activeCount++;
+            else if (wf.status === "pending")
+                pendingCount++;
+            else
+                completedCount++;
+        }
+        return {
+            timestamp: new Date().toISOString(),
+            activeWorkflows: activeCount,
+            pendingWorkflows: pendingCount,
+            completedWorkflows: completedCount,
+            workflows,
+            queueDepth: this.workflowQueue.length,
+        };
+    }
+    /** Get file conflicts for a workflow. */
+    getConflicts(workflowId) {
+        return this.conflicts.get(workflowId) ?? [];
+    }
+    /** Record which files a task touched (called externally after parsing output). */
+    recordFilesTouched(workflowId, taskKey, files) {
+        const workflow = this.workflows.get(workflowId);
+        if (!workflow)
+            return;
+        const task = workflow.tasks.get(taskKey);
+        if (task) {
+            task.filesTouched = files;
+        }
+    }
+    /** Record output for a task. */
+    recordTaskOutput(workflowId, taskKey, output) {
+        const workflow = this.workflows.get(workflowId);
+        if (!workflow)
+            return;
+        const task = workflow.tasks.get(taskKey);
+        if (task) {
+            task.output = output;
+        }
+    }
+    /** Remove a completed workflow from tracking. */
+    removeWorkflow(workflowId) {
+        const workflow = this.workflows.get(workflowId);
+        if (!workflow)
+            return false;
+        if (workflow.status === "running") {
+            throw new Error("Cannot remove a running workflow — cancel it first");
+        }
+        this.workflows.delete(workflowId);
+        this.conflicts.delete(workflowId);
+        return true;
+    }
+    // -------------------------------------------------------------------------
+    // Cleanup
+    // -------------------------------------------------------------------------
+    /** Cancel all workflows and clear state. */
+    async shutdown() {
+        // Cancel running workflows
+        for (const [id, wf] of this.workflows) {
+            if (wf.status === "running" || wf.status === "pending") {
+                await this.cancel(id).catch(() => { });
+            }
+        }
+        // Clean up event handlers
+        for (const [, cleanup] of this.eventHandlers) {
+            cleanup();
+        }
+        this.eventHandlers.clear();
+        // Resolve any queued workflows so their execute() calls don't hang
+        for (const [, resolve] of this.queueResolvers) {
+            resolve();
+        }
+        this.queueResolvers.clear();
+        this.workflowQueue.length = 0;
+        this.activeWorkflowCount = 0;
+        log.info("Orchestrator shut down");
+    }
+    // -------------------------------------------------------------------------
+    // Private helpers
+    // -------------------------------------------------------------------------
+    requireWorkflow(id) {
+        const workflow = this.workflows.get(id);
+        if (!workflow) {
+            throw new Error(`Workflow not found: ${id}`);
+        }
+        return workflow;
+    }
+    computeWorkflowStatus(workflow) {
+        const tasks = [...workflow.tasks.values()];
+        const allCompleted = tasks.every((t) => t.status === "completed");
+        const allFailed = tasks.every((t) => t.status === "failed");
+        const allCancelled = tasks.every((t) => t.status === "cancelled");
+        const anyFailed = tasks.some((t) => t.status === "failed");
+        const anyCompleted = tasks.some((t) => t.status === "completed");
+        if (allCompleted)
+            return "completed";
+        if (allFailed)
+            return "failed";
+        if (allCancelled)
+            return "cancelled";
+        if (anyFailed && anyCompleted)
+            return "partial";
+        if (anyFailed)
+            return "failed";
+        return "completed";
+    }
+    getTaskProgress(workflow) {
+        let completed = 0;
+        let running = 0;
+        let failed = 0;
+        let total = 0;
+        for (const [, task] of workflow.tasks) {
+            total++;
+            if (task.status === "completed")
+                completed++;
+            else if (task.status === "running")
+                running++;
+            else if (task.status === "failed")
+                failed++;
+        }
+        return { completed, total, running, failed };
+    }
+    buildResult(workflow) {
+        const tasks = [];
+        for (const [, task] of workflow.tasks) {
+            let durationMs;
+            if (task.startedAt && task.finishedAt) {
+                durationMs = new Date(task.finishedAt).getTime() - new Date(task.startedAt).getTime();
+            }
+            tasks.push({
+                key: task.key,
+                label: task.label,
+                status: task.status,
+                output: task.output,
+                error: task.error,
+                durationMs,
+            });
+        }
+        const summary = {
+            total: tasks.length,
+            completed: tasks.filter((t) => t.status === "completed").length,
+            failed: tasks.filter((t) => t.status === "failed").length,
+            skipped: tasks.filter((t) => t.status === "skipped").length,
+            cancelled: tasks.filter((t) => t.status === "cancelled").length,
+            durationMs: workflow.startedAt && workflow.finishedAt
+                ? new Date(workflow.finishedAt).getTime() - new Date(workflow.startedAt).getTime()
+                : 0,
+        };
+        return {
+            workflowId: workflow.id,
+            workflowName: workflow.name,
+            status: workflow.status,
+            tasks,
+            conflicts: this.conflicts.get(workflow.id) ?? [],
+            summary,
+        };
+    }
+    /** Detect cycles in dependency graph via DFS. */
+    detectCycles(tasks) {
+        const visited = new Set();
+        const inStack = new Set();
+        const visit = (key, path) => {
+            if (inStack.has(key)) {
+                const cycle = [...path.slice(path.indexOf(key)), key];
+                throw new Error(`Dependency cycle detected: ${cycle.join(" → ")}`);
+            }
+            if (visited.has(key))
+                return;
+            visited.add(key);
+            inStack.add(key);
+            const task = tasks.get(key);
+            if (task) {
+                for (const dep of task.dependsOn) {
+                    visit(dep, [...path, key]);
+                }
+            }
+            inStack.delete(key);
+        };
+        for (const key of tasks.keys()) {
+            visit(key, []);
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function generateWorkflowId() {
+    return `wf_${Date.now()}_${randomBytes(4).toString("hex")}`;
+}
+/**
+ * Parse file paths from agent output.
+ * Looks for common patterns: "Created X", "Modified X", "Wrote X", file paths in backticks.
+ */
+export function parseFilesFromOutput(output) {
+    const files = new Set();
+    const patterns = [
+        /(?:Created|Modified|Wrote|Updated|Edited|Deleted)\s+[`"]?([^\s`"]+\.\w+)[`"]?/gi,
+        /^\s*[-+]\s+(\S+\.\w+)/gm,
+    ];
+    for (const pattern of patterns) {
+        let match;
+        while ((match = pattern.exec(output)) !== null) {
+            const file = match[1].trim();
+            if (file.length > 2 && file.length < 256) {
+                files.add(file);
+            }
+        }
+    }
+    return [...files];
+}
+//# sourceMappingURL=orchestration.js.map