@mikode13/harness 10.0.0 → 11.0.1

package/dist/engines/domain/model/llmAgent.js

@@ -6,12 +6,13 @@ import { classifyHostFailure, classifyLocalFailure, classifyProviderFailure, des
 import { Conversation } from '#src/llm/domain/conversation';
 import { asNative, nativeOf } from '#src/llm/domain/tool';
 import { readAnswers } from '#src/agent/domain/question';
-import { prepareCall, RunEndingToolError, ToolExecutionError, } from '#src/tools/domain/preparedCall';
+import { isDelegating, prepareCall, RunEndingToolError, ToolExecutionError, } from '#src/tools/domain/preparedCall';
 import { ReadRegistry } from '#src/tools/domain/readRegistry';
 import { InvalidToolInputError } from '#src/tools/domain/tool';
 import { isAbortError } from '#src/shared/domain/isAbortError';
 import { addTokens } from '#src/shared/domain/tokens';
 import { anotherAttempt, askAt, CommandLoops, extensionNote, loopQuestion, loopStop, loopWarning, repeatedAnswersAt, repeatNote, repeatQuestion, repeatStop, stop, warnAt, } from "./commandLoops.js";
+import { BackgroundAgents, sentNote } from "./backgroundAgents.js";
 import { measureContext } from "./stepContext.js";
 import { askUserDefinition, askUserName, describeAnswers, readQuestions, summarizeQuestions, } from "./askUser.js";
 import { followHistory, transcriptOf, undoNote, } from "./historyFollower.js";
@@ -270,9 +271,11 @@ export class LLMAgent {
      * place. Only a `destructive` call is asked about: git can undo a `mutating` one. A throwing
      * approver ends the run, as any host callback does.
      */
-    async denial(call, { risk, command }, signal, approve) {
+    async denial(call, prepared, signal, approve) {
+        const { risk } = prepared;
         if (this.autoApprove || risk !== 'destructive')
             return undefined;
+        const command = 'command' in prepared ? prepared.command : undefined;
         const retry = 'Do not retry the same call; choose another way, or ask the user.';
         if (!approve) {
             return `The call to "${call.name}" was denied: it is destructive, and no one was asked to allow it. ${retry}`;
@@ -307,9 +310,10 @@ export class LLMAgent {
     /**
      * Always answers the call: a missing tool, a denied call or a failing one becomes an error
      * result, so the model can correct itself. Only a cancellation, a failing approver, or host
-     * code that failed inside the call, such as an agent's approver, escapes.
+     * code that failed inside the call escapes. A call that hands tasks to agents answers once
+     * it sent them, and `background` watches them from then on.
      */
-    async runTool(call, { signal, onProgress: callback = ignoreProgress, approve, ask }, effects, toolCall) {
+    async runTool(call, { signal, onProgress: callback = ignoreProgress, approve, ask }, effects, toolCall, background) {
         const result = { type: 'toolResult', callId: call.id, name: call.name };
         signal.throwIfAborted();
         if (call.name === askUserName && ask && !this.tools.has(askUserName)) {
@@ -347,7 +351,10 @@ export class LLMAgent {
         }
         let prepared;
         try {
-            prepared = await prepareCall(tool, call.input, signal, { ...toolCall, callId: call.id });
+            const context = { ...toolCall, callId: call.id };
+            prepared = isDelegating(tool)
+                ? await tool.delegate(call.input, signal, context)
+                : await prepareCall(tool, call.input, signal, context);
         }
         catch (error) {
             signal.throwIfAborted();
@@ -385,6 +392,25 @@ export class LLMAgent {
         // Set before `execute`, so a tool that fails halfway still counts: it may have had effects.
         effects.toolRan = true;
         const unrecorded = prepared.unrecorded === true && prepared.risk !== 'safe';
+        if ('start' in prepared) {
+            // Before its agents start, so the person watching reads it before anything they narrate.
+            emit({
+                type: 'tool',
+                id: call.id,
+                name: call.name,
+                status: 'sent',
+                ...(summary === undefined ? {} : { summary }),
+            }, callback);
+            const agents = prepared.start(background.signal);
+            background.add(call.id, agents, unrecorded);
+            return {
+                result: { ...result, output: sentNote(agents), isError: false },
+                denied: false,
+                sent: true,
+                unrecorded,
+                ...(summary === undefined ? {} : { summary }),
+            };
+        }
         try {
             const output = await prepared.run(signal);
             const ending = prepared.ending?.();
@@ -451,35 +477,26 @@ export class LLMAgent {
         };
     }
     /**
-     * One at a time and in order, so their effects and their narration never interleave. Only
-     * calls of a `parallel` tool next to each other run at once, such as agents being delegated
-     * to, each narrated as it goes.
+     * One at a time and in order, so their effects and their narration never interleave. Agents a
+     * call hands tasks to keep working in the background once it answered.
      */
     async runTools(calls, options, effects, toolCall, 
     // Throws the failure of a progress callback a call narrated through.
-    throwIfProgressFailed, loops) {
+    throwIfProgressFailed, loops, background) {
         const { signal } = options;
         const results = [];
-        // The calls before this index already ran, with the ones they ran beside.
-        let ranTo = 0;
         for (const [index, call] of calls.entries()) {
-            if (index < ranTo)
-                continue;
             // A call before this one may have ignored the cancellation; this one must not start,
             // nor be announced as running.
             signal.throwIfAborted();
-            const together = this.parallelFrom(calls, index);
-            if (together.length > 1) {
-                results.push(...(await this.runTogether(together, options, effects, toolCall, loops)));
-                throwIfProgressFailed();
-                ranTo = index + together.length;
-                continue;
-            }
             // `runTool` announces the call only once it is allowed to run, so a consumer never
             // shows a call as running early, nor while the user is still being asked.
-            const ran = await this.runTool(call, options, effects, toolCall);
+            const ran = await this.runTool(call, options, effects, toolCall, background);
             // A call that failed to narrate ends the run before the next one can start.
             throwIfProgressFailed();
+            // An agent at work since the last call may have changed what the history cannot copy.
+            if (background.mayHaveChangedUnseen())
+                loops.unrecordedCallRan();
             const settled = this.settle(call, ran, loops, options);
             let { result } = settled;
             const { alike, key, label } = settled;
@@ -508,53 +525,12 @@ export class LLMAgent {
         }
         return { message: { role: 'tool', content: results } };
     }
-    /** The calls from `index` on whose tool is `parallel`, up to the first that is not. */
-    parallelFrom(calls, index) {
-        const together = [];
-        for (const call of calls.slice(index)) {
-            const tool = this.tools.get(call.name);
-            if (!tool || !('parallel' in tool) || tool.parallel !== true)
-                break;
-            together.push(call);
-        }
-        return together;
-    }
-    /**
-     * Runs the calls at once, their results in the order of the calls. The first failure that
-     * ends the run stops the others, and the run ends once every one has.
-     */
-    async runTogether(calls, options, effects, toolCall, loops) {
-        const stopping = new AbortController();
-        const together = { ...options, signal: AbortSignal.any([options.signal, stopping.signal]) };
-        const ended = {};
-        const settled = await Promise.allSettled(calls.map(async (call) => {
-            try {
-                const ran = await this.runTool(call, together, effects, toolCall);
-                return this.settle(call, ran, loops, options).result;
-            }
-            catch (error) {
-                ended.failure ??= { error };
-                stopping.abort();
-                throw error;
-            }
-        }));
-        // The others were stopped for it: what ended the run is the first failure, or the user.
-        options.signal.throwIfAborted();
-        if (ended.failure) {
-            const { error } = ended.failure;
-            // A call stopped on its way may have spent on the model, and a stop carries no tokens:
-            // the run's total is unknown, not short.
-            const stopped = settled.some(outcome => outcome.status === 'rejected' && outcome.reason !== error);
-            throw stopped ? withSpentTokens(error, undefined, true) : error;
-        }
-        return settled.flatMap(outcome => (outcome.status === 'fulfilled' ? [outcome.value] : []));
-    }
     /**
      * Narrates how a call ended and notes its command's ending, with the warning for the model
      * when that ending keeps coming back alike.
      */
     settle(call, ran, loops, { onProgress: callback = ignoreProgress, ask }) {
-        const { denied, diff, summary, exitCode, ending } = ran;
+        const { denied, diff, summary, exitCode, ending, sent } = ran;
         let { result } = ran;
         if (ran.unrecorded)
             loops.unrecordedCallRan();
@@ -580,6 +556,9 @@ export class LLMAgent {
                 ...(summary === undefined ? {} : { summary }),
             }, callback);
         }
+        else if (sent === true) {
+            // Narrated as it sent its agents, which go on after it until each one ends.
+        }
         // The diff of a change it made, for the person watching; only once it was made.
         else if (diff !== undefined) {
             emit({
@@ -721,6 +700,22 @@ export class LLMAgent {
     }
     async takeSteps(prompt, options, context, seen) {
         const { signal, onProgress: callback = ignoreProgress } = options;
+        // A tool narrates through this without ever failing: a failing callback would reach
+        // only the tool, which turns any failure into a result for the model. The run ends on
+        // it once that tool returns.
+        let progressFailure;
+        const onProgress = (event) => {
+            if (progressFailure)
+                return;
+            try {
+                emit(event, callback);
+            }
+            catch (error) {
+                progressFailure = { error };
+            }
+        };
+        // The agents the run's calls send to work in the background.
+        const background = new BackgroundAgents(signal, onProgress);
         const start = Date.now();
         const runMessages = [{ role: 'user', content: [{ type: 'text', text: prompt }] }];
         let tokens;
@@ -764,23 +759,11 @@ export class LLMAgent {
                 tokens = addTokens(tokens, spent);
                 unreported ||= unreportedCall;
             };
-            // A tool narrates through this without ever failing: a failing callback would reach
-            // only the tool, which turns any failure into a result for the model. The run ends on
-            // it once that tool returns.
-            let progressFailure;
-            const onProgress = (event) => {
-                if (progressFailure)
-                    return;
-                try {
-                    emit(event, callback);
-                }
-                catch (error) {
-                    progressFailure = { error };
-                }
-            };
+            // Also ends the run at its next boundary when an agent at work failed in a way that ends it.
             const throwIfProgressFailed = () => {
                 if (progressFailure)
                     throw progressFailure.error;
+                background.throwIfEnded();
             };
             // The agent's own `askUser` exists only for a run someone can answer, and never beside
             // a consumer's tool of that name, which the consumer gave first and which stays theirs.
@@ -796,7 +779,18 @@ export class LLMAgent {
                 if (followed)
                     this.seenRevision = followed.latestRevision;
             };
+            // A run kept although it failed stops its agents first, and tells the next run what
+            // became of each one, which would otherwise wait for answers that never come.
+            const keepFailed = async () => {
+                await background.stop();
+                const ended = background.takeAnswers();
+                if (ended)
+                    runMessages.push(ended);
+                remember();
+            };
             for (let step = 1; step <= this.maxSteps; step++) {
+                // An agent at work may have narrated through a callback that failed.
+                throwIfProgressFailed();
                 const { response, context: sent } = await this.llmCall(runMessages, after, {
                     signal,
                     step,
@@ -808,6 +802,7 @@ export class LLMAgent {
                 // After the tokens are counted, so a throwing callback still reports them, and for an
                 // unusable answer too: the call was sent, and billed.
                 this.narrateStep(step, response.usage, { context: sent, tools }, callback);
+                throwIfProgressFailed();
                 if (response.stopReason === 'truncated') {
                     // What it wrote before the output limit, kept with the run so the next run can
                     // take it up. Not its calls, whose input may be cut, nor what only the provider
@@ -817,7 +812,7 @@ export class LLMAgent {
                         narrate(part, callback);
                     if (written.length > 0)
                         runMessages.push({ role: 'assistant', content: written });
-                    remember();
+                    await keepFailed();
                     throw new UnrecoverableError('The LLM stopped before completing its answer', {
                         cause: 'The model stopped with "truncated". The run, and what the answer held before the limit, are kept in the conversation.',
                     });
@@ -837,6 +832,24 @@ export class LLMAgent {
                 }
                 const calls = response.message.content.filter(part => part.type === 'toolCall');
                 if (calls.length === 0) {
+                    // An answer given while agents work, or before the model read what they answered,
+                    // is a wait: the model goes on with their answers.
+                    let answers = background.takeAnswers();
+                    if (!answers && background.working && step < this.maxSteps) {
+                        await background.nextAnswer();
+                        throwIfProgressFailed();
+                        answers = background.takeAnswers();
+                    }
+                    if (answers || background.working) {
+                        if (step === this.maxSteps) {
+                            throw new UnrecoverableError('The LLM did not finish within its step limit', {
+                                cause: `The model was still waiting for the agents it sent after ${String(this.maxSteps)} steps.`,
+                            });
+                        }
+                        if (answers)
+                            runMessages.push(answers);
+                        continue;
+                    }
                     remember();
                     return {
                         // Only the final answer: text from earlier steps was narrated as it came.
@@ -844,7 +857,7 @@ export class LLMAgent {
                             .filter(part => part.type === 'text')
                             .map(part => part.text)
                             .join('\n'),
-                        tokens: unreported ? undefined : tokens,
+                        tokens: unreported || background.usageUnknown ? undefined : tokens,
                         duration: (Date.now() - start) / 1000,
                     };
                 }
@@ -875,7 +888,7 @@ export class LLMAgent {
                                 ...rest,
                             ],
                         });
-                        remember();
+                        await keepFailed();
                         throw decision;
                     }
                     repeats.reset();
@@ -891,13 +904,17 @@ export class LLMAgent {
                         depth: delegationOf(options)?.depth ?? 0,
                         ...(options.approve ? { approve: options.approve } : {}),
                         ...(options.ask ? { ask: options.ask } : {}),
-                    }, throwIfProgressFailed, loops);
+                    }, throwIfProgressFailed, loops, background);
                     runMessages.push(repeatedNote === undefined ? message : withNote(message, repeatedNote));
                     // Kept like a run cut by the output limit: the user may take it up.
                     if (stopped) {
-                        remember();
+                        await keepFailed();
                         throw stopped;
                     }
+                    // What agents answered meanwhile, after the results the model is waiting for.
+                    const answers = background.takeAnswers();
+                    if (answers)
+                        runMessages.push(answers);
                 }
             }
             throw new UnrecoverableError('The LLM did not finish within its step limit', {
@@ -905,6 +922,9 @@ export class LLMAgent {
             });
         }
         catch (error) {
+            // The run ends only after the agents still working, which stop with it: what they
+            // changed and spent is the run's.
+            await background.stop();
             let failure = error;
             if (effects.toolRan && error instanceof RecoverableError) {
                 // RetryingAgent would run the whole prompt again, and with it every tool this run
@@ -918,7 +938,7 @@ export class LLMAgent {
                 failure = unrecoverable;
             }
             // The run spent the tokens of every step before the one that failed.
-            throw withSpentTokens(failure, tokens, unreported);
+            throw withSpentTokens(failure, tokens, unreported || background.usageUnknown);
         }
     }
 }