@nodetool-ai/agents 0.7.0-rc.23 → 0.8.2

package/dist/task-executor.js

@@ -3,22 +3,19 @@
  *
  * Port of src/nodetool/agents/task_executor.py
  *
- * Iteratively finds steps whose dependencies are satisfied, runs
- * StepExecutor for each, and collects results until all steps complete
- * or the safety limit is reached.
- *
- * Process-mode steps automatically fan out over list inputs produced by
- * a preceding discover step. Each item is rendered into the step's
- * `perItemInstructions` template and executed as an ephemeral step.
- * Results are aggregated into a list that downstream aggregate steps consume.
+ * The task's steps are a DAG, handed to {@link scheduleDag}: a step starts the
+ * moment its last dependency settles and runs as a `CodeActExecutor`. Nothing
+ * counts dispatch rounds, so a chain is as deep as the plan says; what bounds
+ * the work is the run budget, the per-step iteration cap, and the permit pool.
  */
-import { createHash } from "node:crypto";
+import { budgetFromContext, createSemaphore, memoryKeys } from "@nodetool-ai/runtime";
 import { createLogger } from "@nodetool-ai/config";
+import { TaskUpdateEvent } from "@nodetool-ai/protocol";
 const log = createLogger("nodetool.agents.task-executor");
-import { StepExecutor } from "./step-executor.js";
-import { DEFAULT_TOKEN_LIMIT } from "./constants.js";
-const DEFAULT_MAX_STEPS = 50;
-const DEFAULT_MAX_STEP_ITERATIONS = 10;
+import { CodeActExecutor } from "./codeact/codeact-executor.js";
+import { branchPool, markRunSlotHeld } from "./subagent.js";
+import { ABORTED, UNSATISFIABLE_DEPENDENCY, scheduleDag } from "./utils/dag-scheduler.js";
+import { DEFAULT_MAX_CONCURRENT_AGENTS, DEFAULT_MAX_STEP_ITERATIONS } from "./constants.js";
 export class TaskExecutor {
     provider;
     model;
@@ -27,12 +24,22 @@ export class TaskExecutor {
     context;
     inputs;
     systemPrompt;
-    maxSteps;
     maxStepIterations;
-    maxTokenLimit;
-    finalStepId;
-    parallelExecution;
+    maxTokens;
+    maxConcurrentAgents;
+    upstreamMemoryKeys;
+    budget;
+    signal;
+    sandboxPackages;
     _finishStepId;
+    /** Steps by id, so settling a node is not a scan of the task per dependency. */
+    stepsById;
+    /**
+     * Steps whose `step_result` reached the consumer. On abort the merge stops
+     * draining, so a step's own terminal events can land in a queue nobody
+     * reads; {@link settleStep} re-emits them for a step not in this set.
+     */
+    resultDelivered = new Set();
     constructor(opts) {
         this.provider = opts.provider;
         this.model = opts.model;
@@ -40,320 +47,290 @@ export class TaskExecutor {
         this.task = opts.task;
         this.context = opts.context;
         this.inputs = opts.inputs ?? {};
+        this.sandboxPackages = opts.sandboxPackages ?? [];
         this.systemPrompt = opts.systemPrompt;
-        this.maxSteps = opts.maxSteps ?? DEFAULT_MAX_STEPS;
         this.maxStepIterations =
             opts.maxStepIterations ?? DEFAULT_MAX_STEP_ITERATIONS;
-        this.maxTokenLimit = opts.maxTokenLimit ?? DEFAULT_TOKEN_LIMIT;
-        this.finalStepId = opts.finalStepId;
-        this.parallelExecution = opts.parallelExecution ?? false;
+        this.maxTokens = opts.maxTokens;
+        this.maxConcurrentAgents =
+            opts.maxConcurrentAgents ?? DEFAULT_MAX_CONCURRENT_AGENTS;
+        this.upstreamMemoryKeys = opts.upstreamMemoryKeys ?? [];
+        this.budget = opts.budget ?? budgetFromContext(opts.context);
+        this.signal = opts.signal;
+        this.stepsById = new Map(opts.task.steps.map((step) => [step.id, step]));
     }
     /**
      * Execute all steps in the task plan, respecting dependency order.
      * Supports both sequential and parallel execution modes.
      */
     async *executeTasks() {
-        // Seed inputs into context
+        // Seed inputs into shared memory so every step sees them. Skip keys that
+        // were already seeded by an upstream caller (e.g. ParallelTaskExecutor) to
+        // avoid redundant writes and extra subscriber notifications.
         for (const [key, value] of Object.entries(this.inputs)) {
-            this.context.set(key, value);
+            const fullKey = memoryKeys.input(key);
+            if (this.context.memory.has(fullKey))
+                continue;
+            this.context.memory.set({
+                key: fullKey,
+                kind: "input",
+                value,
+                title: key
+            });
         }
-        // Auto-detect finish step (last step) like Python does
+        // The last step in the plan is the finish step.
         this._finishStepId =
-            this.finalStepId ??
-                (this.task.steps.length > 0
-                    ? this.task.steps[this.task.steps.length - 1].id
-                    : undefined);
+            this.task.steps.length > 0
+                ? this.task.steps[this.task.steps.length - 1].id
+                : undefined;
         log.info("Task execution started", {
             title: this.task.title,
             steps: this.task.steps.length
         });
-        let stepsTaken = 0;
-        while (!this.allTasksComplete() && stepsTaken < this.maxSteps) {
-            stepsTaken++;
-            let executableSteps = this.getExecutableSteps();
-            executableSteps = this.maybeDeferFinishStep(executableSteps);
-            if (executableSteps.length === 0) {
-                if (!this.allTasksComplete()) {
-                    yield {
-                        type: "chunk",
-                        content: "\nNo executable steps but not all complete. Possible dependency issues.\n",
-                        done: false
-                    };
-                }
-                break;
-            }
-            log.debug("Dispatching steps", {
-                stepIds: executableSteps.map((s) => s.id)
-            });
-            // Separate process-mode steps from normal steps
-            const processSteps = executableSteps.filter((s) => s.mode === "process");
-            const normalSteps = executableSteps.filter((s) => s.mode !== "process");
-            // Handle process-mode steps with fan-out
-            for (const pStep of processSteps) {
-                yield* this.handleProcessStep(pStep);
-            }
-            // Create step executors for normal steps
-            const stepGenerators = normalSteps.map((step) => {
-                const executor = new StepExecutor({
-                    task: this.task,
-                    step,
-                    context: this.context,
-                    provider: this.provider,
-                    model: this.model,
-                    tools: [...this.tools],
-                    systemPrompt: this.systemPrompt,
-                    maxTokenLimit: this.maxTokenLimit,
-                    maxIterations: this.maxStepIterations,
-                    useFinishTask: this.isFinishStep(step)
-                });
-                return executor.execute();
-            });
-            if (this.parallelExecution && stepGenerators.length > 1) {
-                // Execute all steps concurrently, merging yielded messages
-                yield* mergeAsyncGenerators(stepGenerators);
-            }
-            else {
-                // Execute steps sequentially
-                for (const generator of stepGenerators) {
-                    for await (const message of generator) {
-                        yield message;
-                    }
-                }
-            }
-        }
+        // Steps this task may run at once — a bound on this task rather than on
+        // the run.
+        const perTask = this.maxConcurrentAgents;
+        yield* scheduleDag({
+            nodes: this.stepNodes(),
+            // A step is a provider conversation, so each running one holds a permit
+            // of the branch's pool: the run's, plus the permit a sub-agent above
+            // this task already holds. Unbudgeted, a pool of the task's own so it is
+            // still bounded.
+            concurrency: this.budget
+                ? branchPool(this.budget, this.context)
+                : createSemaphore(perTask),
+            maxConcurrent: perTask,
+            signal: this.signal,
+            run: (node) => this.runStep(node.step),
+            settle: (node, outcome, error) => this.settleStep(node.step, outcome, error),
+            onBlocked: (node, by) => this.failStepEvents(node.step, this.signal?.aborted
+                ? "Step aborted"
+                : `Step blocked: dependency ${by.id} failed`)
+        });
     }
     /**
-     * Produce a short deterministic hash for a value (used in ephemeral step IDs).
+     * The steps the scheduler runs, with the dependencies each waits on.
+     *
+     * A step already settled when the task starts is not a node: its dependents
+     * treat it as satisfied (completed) or as never satisfiable (failed), which
+     * is what the round loop's `completedIds` set did. Input keys count as
+     * satisfied the same way. A dependency naming nothing at all is left in
+     * place, so the step never becomes ready and the scheduler settles it with
+     * `unsatisfiable dependency`.
      */
-    shortHash(value) {
-        const data = JSON.stringify(value, (_key, val) => {
-            if (val != null && typeof val === "object" && !Array.isArray(val)) {
-                const sorted = {};
-                for (const k of Object.keys(val).sort()) {
-                    sorted[k] = val[k];
+    stepNodes() {
+        const pending = this.task.steps.filter((step) => !step.completed && !step.failed);
+        const satisfied = new Set(Object.keys(this.inputs));
+        for (const step of this.task.steps) {
+            if (step.completed)
+                satisfied.add(step.id);
+        }
+        const deferredBehind = this.stepsFinishWaitsFor(pending);
+        return pending.map((step) => {
+            const dependsOn = step.dependsOn.filter((dep) => !satisfied.has(dep));
+            if (step.id === this._finishStepId) {
+                // A Set, not `includes`: the list it is checked against grows with
+                // every id added, which would make a wide task quadratic.
+                const declared = new Set(dependsOn);
+                for (const id of deferredBehind) {
+                    if (!declared.has(id))
+                        dependsOn.push(id);
                 }
-                return sorted;
             }
-            return val;
+            return { id: step.id, dependsOn, step };
         });
-        return createHash("sha1").update(data).digest("hex").slice(0, 12);
     }
     /**
-     * Handle a process-mode step by fanning out over list inputs.
-     * Creates ephemeral steps for each item in the discover step's result
-     * and aggregates the outputs into a list stored in context.
+     * The steps the finish step must wait for: every other pending step that
+     * does not itself depend on the finish step.
+     *
+     * The aggregation step runs last, which the round loop arranged by holding
+     * it back each round. Here it is a real dependency, so a sibling that fails
+     * blocks it (I-5) instead of aggregating over a hole. Steps downstream of
+     * the finish step are excluded — a plan may declare one, and depending on
+     * them both ways is a cycle nothing could run.
      */
-    async *handleProcessStep(step) {
-        const discoverStepId = step.dependsOn[0];
-        if (!discoverStepId) {
-            log.warn("Process step has no dependencies, skipping fan-out", {
-                stepId: step.id
-            });
-            step.completed = true;
-            return;
-        }
-        let discoverResult = this.context.get(discoverStepId);
-        if (discoverResult === undefined || discoverResult === null) {
-            log.warn("Discover step result is null/undefined, skipping fan-out", {
-                stepId: step.id
-            });
-            step.completed = true;
-            this.context.set(step.id, []);
-            step.endTime = Date.now();
-            return;
-        }
-        if (!Array.isArray(discoverResult)) {
-            log.warn("Discover step result is not an array, wrapping as single-item list", {
-                stepId: step.id,
-                resultType: typeof discoverResult
-            });
-            discoverResult = [discoverResult];
-        }
-        const items = discoverResult;
-        const template = step.perItemInstructions ?? step.instructions;
-        const perItemSchema = step.perItemSchema;
-        log.info("Fan-out processing", {
-            stepId: step.id,
-            itemCount: items.length
-        });
-        // Create ephemeral steps for each item
-        const ephemeralSteps = items.map((item) => {
-            let instructions = template;
-            if (typeof item === "object" && item !== null) {
-                for (const [key, value] of Object.entries(item)) {
-                    const escapedKey = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-                    const strValue = String(value);
-                    instructions = instructions.replace(new RegExp(`\\{${escapedKey}\\}`, "g"), () => strValue);
+    stepsFinishWaitsFor(pending) {
+        const finishId = this._finishStepId;
+        if (!finishId)
+            return [];
+        const dependents = new Map();
+        for (const step of pending) {
+            for (const dep of step.dependsOn) {
+                const list = dependents.get(dep);
+                if (list) {
+                    list.push(step);
                 }
-            }
-            else {
-                const strItem = String(item);
-                instructions = instructions.replace(/\{item\}/g, () => strItem);
-            }
-            const hash = this.shortHash(item);
-            return {
-                id: `${step.id}_item_${hash}`,
-                instructions,
-                completed: false,
-                dependsOn: [],
-                logs: [],
-                outputSchema: perItemSchema ?? step.outputSchema
-            };
-        });
-        // Create step executors for each ephemeral step
-        const generators = ephemeralSteps.map((ephStep) => {
-            const executor = new StepExecutor({
-                task: this.task,
-                step: ephStep,
-                context: this.context,
-                provider: this.provider,
-                model: this.model,
-                tools: [...this.tools],
-                systemPrompt: this.systemPrompt,
-                maxTokenLimit: this.maxTokenLimit,
-                maxIterations: this.maxStepIterations,
-                useFinishTask: false
-            });
-            return executor.execute();
-        });
-        // Execute and collect results
-        const results = [];
-        if (this.parallelExecution && generators.length > 1) {
-            for await (const msg of mergeAsyncGenerators(generators)) {
-                if (msg.type === "step_result") {
-                    results.push(msg.result);
+                else {
+                    dependents.set(dep, [step]);
                 }
-                yield msg;
             }
         }
-        else {
-            for (const gen of generators) {
-                for await (const msg of gen) {
-                    if (msg.type === "step_result") {
-                        results.push(msg.result);
-                    }
-                    yield msg;
-                }
+        const downstream = new Set([finishId]);
+        const frontier = [finishId];
+        for (let i = 0; i < frontier.length; i++) {
+            for (const dependent of dependents.get(frontier[i]) ?? []) {
+                if (downstream.has(dependent.id))
+                    continue;
+                downstream.add(dependent.id);
+                frontier.push(dependent.id);
             }
         }
-        // Store aggregated results and mark complete
-        this.context.set(step.id, results);
-        step.completed = true;
-        step.endTime = Date.now();
-        log.info("Fan-out complete", {
-            stepId: step.id,
-            resultCount: results.length
-        });
+        return pending
+            .filter((step) => !downstream.has(step.id))
+            .map((step) => step.id);
     }
     /**
-     * Check if all steps in the task are completed.
+     * Run one step and report how it settled.
+     *
+     * The step runs on a context of its own, sharing the task's memory: the
+     * permit the step merge holds for it is recorded there, so a sub-agent the
+     * step spawns finds its parent's permit rather than a sibling's. Messages a
+     * tool posts on that context are forwarded to the task's, where the host
+     * reads them.
      */
-    allTasksComplete() {
-        return this.task.steps.every((step) => step.completed);
-    }
-    /**
-     * Find steps whose dependencies are all satisfied (completed).
-     */
-    getExecutableSteps() {
-        const completedIds = new Set(this.task.steps.filter((s) => s.completed).map((s) => s.id));
-        // Also count inputs as satisfied dependencies
-        for (const key of Object.keys(this.inputs)) {
-            completedIds.add(key);
+    async *runStep(step) {
+        const stepContext = this.context.copy({
+            shareMemory: true,
+            inheritMessageListeners: false
+        });
+        stepContext.addMessageListener((msg) => this.context.postMessage(msg));
+        if (this.budget)
+            markRunSlotHeld(stepContext);
+        const executor = new CodeActExecutor({
+            task: this.task,
+            step,
+            context: stepContext,
+            provider: this.provider,
+            model: this.model,
+            tools: this.toolsForStep(step),
+            systemPrompt: this.systemPrompt,
+            maxIterations: this.maxStepIterations,
+            maxTokens: this.maxTokens,
+            turnBudget: this.budget,
+            useFinishTask: this.isFinishStep(step),
+            upstreamMemoryKeys: this.upstreamMemoryKeys,
+            signal: this.signal,
+            sandboxPackages: this.sandboxPackages
+        });
+        for await (const msg of this.withStepLog(step, executor.execute())) {
+            yield msg;
+            // Delivered only when the run was still live at the hand-off: once the
+            // signal fires the merge drains nothing more, so a result yielded after
+            // that is in a queue nobody reads.
+            if (msg.type === "step_result" && this.signal?.aborted !== true) {
+                this.resultDelivered.add(step.id);
+            }
         }
-        return this.task.steps.filter((step) => !step.completed &&
-            !this.isStepRunning(step) &&
-            step.dependsOn.every((dep) => completedIds.has(dep)));
+        // A cancelled run is not a failed step, and must not cascade to its
+        // dependents as one.
+        if (this.signal?.aborted)
+            return { outcome: "failed", error: ABORTED };
+        // A step failure does not throw: the executor records it on the step and
+        // emits its own terminal events.
+        return step.failed
+            ? { outcome: "failed", error: step.error }
+            : { outcome: "ok" };
     }
     /**
-     * Check if a step is currently running (started but not finished).
-     * Mirrors Python's Step.is_running().
+     * Terminal events for a settled step. A step that ran emitted its own, so
+     * this speaks only for one that never did — blocked by a dependency that can
+     * never be satisfied — or for one the run's signal cut short, whose own
+     * terminal events the abort may have dropped on the way out.
      */
-    isStepRunning(step) {
-        return step.startTime != null && step.endTime == null;
-    }
-    /**
-     * Check if a step is the designated finish/aggregation step.
-     */
-    isFinishStep(step) {
-        if (this._finishStepId) {
-            return step.id === this._finishStepId;
+    settleStep(step, outcome, reason) {
+        if (outcome === "ok")
+            return [];
+        if (reason === ABORTED) {
+            return this.resultDelivered.has(step.id)
+                ? []
+                : this.failStepEvents(step, "Step aborted");
         }
-        return (this.task.steps.length > 0 &&
-            step === this.task.steps[this.task.steps.length - 1]);
+        if (step.completed || step.failed)
+            return [];
+        const blocking = step.dependsOn.filter((dep) => this.stepsById.get(dep)?.failed === true);
+        return this.failStepEvents(step, blocking.length > 0
+            ? `Step blocked: dependency ${blocking.join(", ")} failed`
+            : `Step blocked: ${reason ?? UNSATISFIABLE_DEPENDENCY}`);
     }
     /**
-     * Defer the finish step until all other steps are complete.
-     * This ensures the final aggregation step runs last.
+     * Bracket one step with a start/finish line. The run log recorded task
+     * boundaries and nothing between them, so a plan that spent minutes in a
+     * step left no trace of which step, how long, or how it settled. Bounded by
+     * the plan's step count, so it cannot become the log-volume problem it fixes.
      */
-    maybeDeferFinishStep(executableSteps) {
-        if (!this._finishStepId)
-            return executableSteps;
-        const finishReady = executableSteps.some((s) => s.id === this._finishStepId);
-        if (!finishReady)
-            return executableSteps;
-        const otherPending = this.task.steps.some((s) => !s.completed && s.id !== this._finishStepId);
-        if (!otherPending)
-            return executableSteps;
-        return executableSteps.filter((s) => s.id !== this._finishStepId);
-    }
-}
-// ---------------------------------------------------------------------------
-// Async generator merge utility (TS equivalent of wrap_generators_parallel)
-// ---------------------------------------------------------------------------
-async function* mergeAsyncGenerators(generators) {
-    // Channel: a queue of resolved values with a promise-based pull mechanism
-    const queue = [];
-    let activeCount = generators.length;
-    let resolve = null;
-    let firstError = undefined;
-    function notify() {
-        if (resolve) {
-            const r = resolve;
-            resolve = null;
-            r();
-        }
-    }
-    // Start a consumer task for each generator
-    const tasks = generators.map(async (gen) => {
+    async *withStepLog(step, gen) {
+        const startedAt = Date.now();
+        log.info("Step started", {
+            taskId: this.task.id,
+            stepId: step.id,
+            tools: this.toolsForStep(step).length
+        });
         try {
-            for await (const item of gen) {
-                queue.push(item);
-                notify();
-            }
-        }
-        catch (e) {
-            if (firstError === undefined)
-                firstError = e;
+            yield* gen;
         }
         finally {
-            activeCount--;
-            notify();
-        }
-    });
-    // Yield items as they arrive
-    while (activeCount > 0 || queue.length > 0) {
-        if (queue.length > 0) {
-            yield queue.shift();
+            log.info("Step finished", {
+                taskId: this.task.id,
+                stepId: step.id,
+                durationMs: Date.now() - startedAt,
+                completed: step.completed === true,
+                failed: step.failed === true,
+                error: step.error
+            });
         }
-        else if (activeCount > 0) {
-            // Set resolve BEFORE checking queue again to avoid race condition
-            // where an item is pushed between the check and the await.
-            const waitPromise = new Promise((r) => {
-                resolve = r;
+    }
+    /**
+     * Tools a step may call. A plan's per-step `tools` allow-list is a privilege
+     * boundary, not a hint: an empty or fully-unresolvable list yields no tools
+     * rather than silently falling back to the full collection.
+     */
+    toolsForStep(step) {
+        if (!Array.isArray(step.tools))
+            return [...this.tools];
+        const allowed = new Set(step.tools);
+        const selected = this.tools.filter((tool) => allowed.has(tool.name));
+        const missing = step.tools.filter((name) => !this.tools.some((tool) => tool.name === name));
+        if (missing.length > 0) {
+            log.warn("Step requested tools that are not available", {
+                stepId: step.id,
+                missing
             });
-            // Re-check after setting resolve — item may have arrived between
-            // the outer check and setting resolve.
-            if (queue.length > 0) {
-                resolve = null;
-                continue;
-            }
-            await waitPromise;
         }
+        return selected;
+    }
+    /** Record one step as a terminal failure and build its lifecycle events. */
+    failStepEvents(step, message) {
+        step.failed = true;
+        step.error = message;
+        step.endTime = Date.now();
+        this.context.memory.set({
+            key: memoryKeys.step(step.id),
+            kind: "step_result",
+            value: { error: message },
+            source: step.id,
+            title: `Failed: ${step.instructions.slice(0, 60)}`
+        });
+        return [
+            {
+                type: "task_update",
+                node_id: step.id,
+                task: { id: this.task.id, title: this.task.title },
+                step: { id: step.id, instructions: step.instructions },
+                event: TaskUpdateEvent.StepFailed
+            },
+            {
+                type: "step_result",
+                step: { id: step.id, instructions: step.instructions },
+                result: { error: message },
+                error: message,
+                is_task_result: this.isFinishStep(step)
+            }
+        ];
     }
-    // Wait for all producer promises to settle
-    await Promise.allSettled(tasks);
-    if (firstError !== undefined) {
-        throw firstError;
+    /** The task's last step, the one whose result is the task's result. */
+    isFinishStep(step) {
+        return step.id === this._finishStepId;
     }
 }
 //# sourceMappingURL=task-executor.js.map