@mikode13/harness 9.1.0 → 9.2.0

package/README.md

@@ -523,10 +523,17 @@ or per role, with no upper bound:
 createLLMOrchestrator({ workspace, maxSteps: { executor: 120 } });
 ```
 
-A run also stops sooner when the model is stuck: when five answers in a row ask for exactly
-the same tool calls, or when one command, with the same input, ends the same way with exactly
-the same output five times in a row, whatever the model did between them. `createLLMAgent`
-takes one number, 65 by default.
+`createLLMAgent` takes one number, 65 by default.
+
+A run also watches for a model that is stuck. Two endings of one command, with the same input,
+are alike when it printed the same, all of it, colors aside and the times a known test runner
+prints, and either failed both times or passed with no file changed between them: a check that
+passes after every change is never counted. On the third ending alike the model is warned. On the
+fifth, the run asks you through `ask` whether to go on, and so it does, before they run, when
+five answers in a row ask for exactly the same tool calls. **Another attempt** lets it go on
+once, **Stop** ends it, and words of your own reach the model. A run stopped this way, or with no
+`ask` to ask, fails with `UnrecoverableError`, and is kept in the conversation with its changes,
+so your next prompt continues from it.
 
 No conversation is compacted unless you ask. `compactAt` is the number of input tokens at
 which the provider compacts it on its side, replacing the turns before that call with its own

package/dist/engines/domain/model/commandLoops.d.ts

@@ -0,0 +1,59 @@
+import type { UserQuestion } from '#src/agent/domain/question';
+import type { CommandEnding } from '#src/tools/domain/preparedCall';
+export declare const warnAt = 3;
+export declare const askAt = 5;
+export declare const repeatedAnswersAt = 5;
+export declare const anotherAttempt = "Another attempt";
+export declare const stop = "Stop";
+interface Seen {
+    ending: CommandEnding;
+    unrecorded: number;
+}
+/** One command call's endings alike, this one included: how many, and the first of them. */
+export interface Alike {
+    times: number;
+    first: Seen;
+    last: Seen;
+}
+/**
+ * Every ending of each command call in one run, by tool and input, to tell when one keeps
+ * ending alike. A failure printed again is alike whatever changed in between: the changes did
+ * not reach what it checks. A pass printed again is alike only when nothing changed, since a
+ * passing check prints the same whatever it checked. No other call breaks the count, and an
+ * ending alike an earlier one counts even with others between, so A, B, A is still suspect.
+ */
+export declare class CommandLoops {
+    private readonly endings;
+    private unrecorded;
+    private extended;
+    /** A call ran that may have changed files the history does not record. */
+    unrecordedCallRan(): void;
+    /** Records how the call `key` ended, and returns its endings alike. */
+    observe(key: string, ending: CommandEnding): Alike;
+    /** Whether the user already let the run go on, so the next fifth ending alike stops it. */
+    get wentOn(): boolean;
+    /**
+     * Whether the user may still let the run go on: once per run, for a command or for repeated
+     * answers alike. Granting it forgets the endings of the calls `keys` names, the ones the user
+     * let run again, so their counts start again and the same loop is not judged twice.
+     */
+    extend(keys: readonly string[]): boolean;
+}
+/**
+ * Told to the model with the result that makes the third ending alike. `asks` is whether the
+ * fifth will ask the user: not once they let the run go on, nor when no one can be asked.
+ */
+export declare function loopWarning(label: string, alike: Alike, asks: boolean): string;
+/** The question for the user on the fifth ending alike. */
+export declare function loopQuestion(label: string, alike: Alike): UserQuestion;
+/** Told to the model when the user let the run go on, with their words if they wrote some. */
+export declare function extensionNote(alike: Alike, words?: string): string;
+/** Why the run stopped, for the failure it ends with. */
+export declare function loopStop(label: string, alike: Alike, reason: string): string;
+/** The question for the user when the model asks for the same `calls` again, before they run. */
+export declare function repeatQuestion(calls: string): UserQuestion;
+/** Told to the model, with the results of the calls, when the user let them run again. */
+export declare function repeatNote(words?: string): string;
+/** Why the run stopped on repeated answers, for the failure it ends with. */
+export declare function repeatStop(reason: string): string;
+export {};

package/dist/engines/domain/model/commandLoops.js

@@ -0,0 +1,143 @@
+// The third ending alike warns the model; the fifth asks the user whether the run goes on.
+export const warnAt = 3;
+export const askAt = 5;
+// So many answers in a row asking for exactly the same calls ask the user too, before they run.
+export const repeatedAnswersAt = 5;
+export const anotherAttempt = 'Another attempt';
+export const stop = 'Stop';
+// The end of a repeated output the user is shown: enough to recognize it.
+const shownLines = 12;
+const shownCharacters = 2_000;
+/**
+ * Every ending of each command call in one run, by tool and input, to tell when one keeps
+ * ending alike. A failure printed again is alike whatever changed in between: the changes did
+ * not reach what it checks. A pass printed again is alike only when nothing changed, since a
+ * passing check prints the same whatever it checked. No other call breaks the count, and an
+ * ending alike an earlier one counts even with others between, so A, B, A is still suspect.
+ */
+export class CommandLoops {
+    endings = new Map();
+    unrecorded = 0;
+    extended = false;
+    /** A call ran that may have changed files the history does not record. */
+    unrecordedCallRan() {
+        this.unrecorded++;
+    }
+    /** Records how the call `key` ended, and returns its endings alike. */
+    observe(key, ending) {
+        const seen = this.endings.get(key) ?? [];
+        const now = { ending, unrecorded: this.unrecorded };
+        const alike = seen.filter(earlier => sameEnding(earlier, now));
+        seen.push(now);
+        this.endings.set(key, seen);
+        return { times: alike.length + 1, first: alike[0] ?? now, last: now };
+    }
+    /** Whether the user already let the run go on, so the next fifth ending alike stops it. */
+    get wentOn() {
+        return this.extended;
+    }
+    /**
+     * Whether the user may still let the run go on: once per run, for a command or for repeated
+     * answers alike. Granting it forgets the endings of the calls `keys` names, the ones the user
+     * let run again, so their counts start again and the same loop is not judged twice.
+     */
+    extend(keys) {
+        if (this.extended)
+            return false;
+        this.extended = true;
+        for (const key of keys)
+            this.endings.delete(key);
+        return true;
+    }
+}
+function sameEnding(earlier, now) {
+    const a = earlier.ending;
+    const b = now.ending;
+    if (a.passed !== b.passed || a.how !== b.how || a.fingerprint !== b.fingerprint)
+        return false;
+    if (!b.passed)
+        return true;
+    // A state that could not be read, or a call the history cannot see, may hide a change.
+    return a.state !== undefined && a.state === b.state && earlier.unrecorded === now.unrecorded;
+}
+/** What changed between the first ending alike and the last, as the model and the user read it. */
+function describeChanges({ first, last }) {
+    const files = [...new Set(last.ending.changed.slice(first.ending.changed.length))];
+    const unseen = last.unrecorded > first.unrecorded
+        ? ' Tools whose changes the history cannot see also ran.'
+        : '';
+    if (files.length === 0)
+        return `No file the history covers changed between them.${unseen}`;
+    const back = first.ending.state !== undefined && first.ending.state === last.ending.state
+        ? ', and went back to what they held'
+        : '';
+    return `Files changed between them${back}: ${files.join(', ')}.${unseen}`;
+}
+/**
+ * Told to the model with the result that makes the third ending alike. `asks` is whether the
+ * fifth will ask the user: not once they let the run go on, nor when no one can be asked.
+ */
+export function loopWarning(label, alike, asks) {
+    const again = alike.last.ending.passed
+        ? 'Running it again with nothing changed tells you nothing new.'
+        : 'Find out why your changes do not reach what it reports before you run it again.';
+    const then = asks ? 'the user is asked whether the run goes on' : 'the run stops';
+    return `"${label}" ${alike.last.ending.how} with the same output ${String(alike.times)} times in this run. ${describeChanges(alike)} ${again} If it ends this way ${String(askAt)} times, ${then}.`;
+}
+/** The question for the user on the fifth ending alike. */
+export function loopQuestion(label, alike) {
+    const lines = alike.last.ending.output.trimEnd().split('\n').slice(-shownLines).join('\n');
+    const output = lines.length > shownCharacters ? `…${lines.slice(-shownCharacters)}` : lines || '(nothing)';
+    return {
+        question: [
+            `Possible loop: "${label}" ${alike.last.ending.how} with the same output ${String(alike.times)} times in this run. ${describeChanges(alike)}`,
+            'It may be a check that passes after every change, which is no loop, or the model repeating itself.',
+            `What it printed, the last lines:\n${output}`,
+        ].join('\n'),
+        options: loopOptions(`If it ends this way ${String(askAt)} more times`),
+        multiSelect: false,
+    };
+}
+function loopOptions(again) {
+    return [
+        {
+            label: anotherAttempt,
+            description: `Let the run go on, once. ${again}, or any other loop is found, the run stops without asking.`,
+        },
+        {
+            label: stop,
+            description: 'End the run here. What it did and changed is kept, and your next prompt continues from it.',
+        },
+    ];
+}
+/** Told to the model when the user let the run go on, with their words if they wrote some. */
+export function extensionNote(alike, words) {
+    const said = words === undefined ? '' : ` They said: ${words}`;
+    return `It has ended this way ${String(alike.times)} times. The user let the run go on, once.${said} Change your approach: if it ends this way ${String(askAt)} more times, the run stops.`;
+}
+/** Why the run stopped, for the failure it ends with. */
+export function loopStop(label, alike, reason) {
+    return `"${label}" ${alike.last.ending.how} with the same output ${String(alike.times)} times. ${describeChanges(alike)} ${reason} The run is kept in the conversation.`;
+}
+/** The question for the user when the model asks for the same `calls` again, before they run. */
+export function repeatQuestion(calls) {
+    return {
+        question: [
+            `Possible loop: the model asked for the same tool calls ${String(repeatedAnswersAt)} times in a row, with nothing else between.`,
+            'It may be waiting for something that changes by itself, which is no loop, or repeating itself.',
+            `The calls:\n${calls}`,
+        ].join('\n'),
+        options: loopOptions(`If it asks for the same calls ${String(repeatedAnswersAt)} more times in a row`),
+        multiSelect: false,
+    };
+}
+/** Told to the model, with the results of the calls, when the user let them run again. */
+export function repeatNote(words) {
+    const said = words === undefined ? '' : ` They said: ${words}`;
+    return `You asked for these same calls ${String(repeatedAnswersAt)} times in a row. The user let the run go on, once.${said} Change your approach: if you ask for the same calls ${String(repeatedAnswersAt)} more times in a row, the run stops.`;
+}
+/** Why the run stopped on repeated answers, for the failure it ends with. */
+export function repeatStop(reason) {
+    return `The model asked for the same tool calls ${String(repeatedAnswersAt)} times in a row. ${reason} The run is kept in the conversation.`;
+}
+//# sourceMappingURL=commandLoops.js.map

package/dist/engines/domain/model/commandLoops.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"commandLoops.js","sourceRoot":"","sources":["../../../../src/engines/domain/model/commandLoops.ts"],"names":[],"mappings":"AAGA,2FAA2F;AAC3F,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC;AACxB,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC;AACvB,gGAAgG;AAChG,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC,MAAM,CAAC,MAAM,cAAc,GAAG,iBAAiB,CAAC;AAChD,MAAM,CAAC,MAAM,IAAI,GAAG,MAAM,CAAC;AAE3B,0EAA0E;AAC1E,MAAM,UAAU,GAAG,EAAE,CAAC;AACtB,MAAM,eAAe,GAAG,KAAK,CAAC;AAe9B;;;;;;GAMG;AACH,MAAM,OAAO,YAAY;IACP,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,UAAU,GAAG,CAAC,CAAC;IACf,QAAQ,GAAG,KAAK,CAAC;IAEzB,0EAA0E;IAC1E,iBAAiB;QAChB,IAAI,CAAC,UAAU,EAAE,CAAC;IACnB,CAAC;IAED,uEAAuE;IACvE,OAAO,CAAC,GAAW,EAAE,MAAqB;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC5B,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;IACvE,CAAC;IAED,2FAA2F;IAC3F,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,IAAuB;QAC7B,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,KAAK,MAAM,GAAG,IAAI,IAAI;YAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjD,OAAO,IAAI,CAAC;IACb,CAAC;CACD;AAED,SAAS,UAAU,CAAC,OAAa,EAAE,GAAS;IAC3C,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC;IACrB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IAC9F,IAAI,CAAC,CAAC,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC3B,uFAAuF;IACvF,OAAO,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,UAAU,KAAK,GAAG,CAAC,UAAU,CAAC;AAC9F,CAAC;AAED,mGAAmG;AACnG,SAAS,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,EAAS;IAC9C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,MAAM,GACX,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU;QACjC,CAAC,CAAC,uDAAuD;QACzD,CAAC,CAAC,EAAE,CAAC;IACP,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,mDAAmD,MAAM,EAAE,CAAC;IAC3F,MAAM,IAAI,GACT,KAAK,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK;QAC3E,CAAC,CAAC,mCAAmC;QACrC,CAAC,CAAC,EAAE,CAAC;IACP,OAAO,6BAA6B,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;AAC3E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,KAAa,EAAE,KAAY,EAAE,IAAa;IACrE,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM;QACrC,CAAC,CAAC,8DAA8D;QAChE,CAAC,CAAC,iFAAiF,CAAC;IACrF,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,2CAA2C,CAAC,CAAC,CAAC,eAAe,CAAC;IAClF,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,yBAAyB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,uBAAuB,eAAe,CAAC,KAAK,CAAC,IAAI,KAAK,wBAAwB,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,GAAG,CAAC;AACrM,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,KAAY;IACvD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3F,MAAM,MAAM,GACX,KAAK,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,WAAW,CAAC;IAC7F,OAAO;QACN,QAAQ,EAAE;YACT,mBAAmB,KAAK,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,yBAAyB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,uBAAuB,eAAe,CAAC,KAAK,CAAC,EAAE;YAC7I,oGAAoG;YACpG,qCAAqC,MAAM,EAAE;SAC7C,CAAC,IAAI,CAAC,IAAI,CAAC;QACZ,OAAO,EAAE,WAAW,CAAC,uBAAuB,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC;QACvE,WAAW,EAAE,KAAK;KAClB,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IACjC,OAAO;QACN;YACC,KAAK,EAAE,cAAc;YACrB,WAAW,EAAE,4BAA4B,KAAK,6DAA6D;SAC3G;QACD;YACC,KAAK,EAAE,IAAI;YACX,WAAW,EACV,4FAA4F;SAC7F;KACD,CAAC;AACH,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,aAAa,CAAC,KAAY,EAAE,KAAc;IACzD,MAAM,IAAI,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,KAAK,EAAE,CAAC;IAC/D,OAAO,yBAAyB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,4CAA4C,IAAI,8CAA8C,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC;AAC7L,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,QAAQ,CAAC,KAAa,EAAE,KAAY,EAAE,MAAc;IACnE,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,yBAAyB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,eAAe,CAAC,KAAK,CAAC,IAAI,MAAM,uCAAuC,CAAC;AAC1K,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,cAAc,CAAC,KAAa;IAC3C,OAAO;QACN,QAAQ,EAAE;YACT,0DAA0D,MAAM,CAAC,iBAAiB,CAAC,6CAA6C;YAChI,gGAAgG;YAChG,eAAe,KAAK,EAAE;SACtB,CAAC,IAAI,CAAC,IAAI,CAAC;QACZ,OAAO,EAAE,WAAW,CACnB,iCAAiC,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,CAChF;QACD,WAAW,EAAE,KAAK;KAClB,CAAC;AACH,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,UAAU,CAAC,KAAc;IACxC,MAAM,IAAI,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,KAAK,EAAE,CAAC;IAC/D,OAAO,kCAAkC,MAAM,CAAC,iBAAiB,CAAC,qDAAqD,IAAI,wDAAwD,MAAM,CAAC,iBAAiB,CAAC,sCAAsC,CAAC;AACpP,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,UAAU,CAAC,MAAc;IACxC,OAAO,2CAA2C,MAAM,CAAC,iBAAiB,CAAC,oBAAoB,MAAM,uCAAuC,CAAC;AAC9I,CAAC","sourcesContent":["import type { UserQuestion } from '#src/agent/domain/question';\nimport type { CommandEnding } from '#src/tools/domain/preparedCall';\n\n// The third ending alike warns the model; the fifth asks the user whether the run goes on.\nexport const warnAt = 3;\nexport const askAt = 5;\n// So many answers in a row asking for exactly the same calls ask the user too, before they run.\nexport const repeatedAnswersAt = 5;\n\nexport const anotherAttempt = 'Another attempt';\nexport const stop = 'Stop';\n\n// The end of a repeated output the user is shown: enough to recognize it.\nconst shownLines = 12;\nconst shownCharacters = 2_000;\n\ninterface Seen {\n\tending: CommandEnding;\n\t// How many calls whose changes the history cannot see had run by then.\n\tunrecorded: number;\n}\n\n/** One command call's endings alike, this one included: how many, and the first of them. */\nexport interface Alike {\n\ttimes: number;\n\tfirst: Seen;\n\tlast: Seen;\n}\n\n/**\n * Every ending of each command call in one run, by tool and input, to tell when one keeps\n * ending alike. A failure printed again is alike whatever changed in between: the changes did\n * not reach what it checks. A pass printed again is alike only when nothing changed, since a\n * passing check prints the same whatever it checked. No other call breaks the count, and an\n * ending alike an earlier one counts even with others between, so A, B, A is still suspect.\n */\nexport class CommandLoops {\n\tprivate readonly endings = new Map<string, Seen[]>();\n\tprivate unrecorded = 0;\n\tprivate extended = false;\n\n\t/** A call ran that may have changed files the history does not record. */\n\tunrecordedCallRan(): void {\n\t\tthis.unrecorded++;\n\t}\n\n\t/** Records how the call `key` ended, and returns its endings alike. */\n\tobserve(key: string, ending: CommandEnding): Alike {\n\t\tconst seen = this.endings.get(key) ?? [];\n\t\tconst now = { ending, unrecorded: this.unrecorded };\n\t\tconst alike = seen.filter(earlier => sameEnding(earlier, now));\n\t\tseen.push(now);\n\t\tthis.endings.set(key, seen);\n\t\treturn { times: alike.length + 1, first: alike[0] ?? now, last: now };\n\t}\n\n\t/** Whether the user already let the run go on, so the next fifth ending alike stops it. */\n\tget wentOn(): boolean {\n\t\treturn this.extended;\n\t}\n\n\t/**\n\t * Whether the user may still let the run go on: once per run, for a command or for repeated\n\t * answers alike. Granting it forgets the endings of the calls `keys` names, the ones the user\n\t * let run again, so their counts start again and the same loop is not judged twice.\n\t */\n\textend(keys: readonly string[]): boolean {\n\t\tif (this.extended) return false;\n\t\tthis.extended = true;\n\t\tfor (const key of keys) this.endings.delete(key);\n\t\treturn true;\n\t}\n}\n\nfunction sameEnding(earlier: Seen, now: Seen): boolean {\n\tconst a = earlier.ending;\n\tconst b = now.ending;\n\tif (a.passed !== b.passed || a.how !== b.how || a.fingerprint !== b.fingerprint) return false;\n\tif (!b.passed) return true;\n\t// A state that could not be read, or a call the history cannot see, may hide a change.\n\treturn a.state !== undefined && a.state === b.state && earlier.unrecorded === now.unrecorded;\n}\n\n/** What changed between the first ending alike and the last, as the model and the user read it. */\nfunction describeChanges({ first, last }: Alike): string {\n\tconst files = [...new Set(last.ending.changed.slice(first.ending.changed.length))];\n\tconst unseen =\n\t\tlast.unrecorded > first.unrecorded\n\t\t\t? ' Tools whose changes the history cannot see also ran.'\n\t\t\t: '';\n\tif (files.length === 0) return `No file the history covers changed between them.${unseen}`;\n\tconst back =\n\t\tfirst.ending.state !== undefined && first.ending.state === last.ending.state\n\t\t\t? ', and went back to what they held'\n\t\t\t: '';\n\treturn `Files changed between them${back}: ${files.join(', ')}.${unseen}`;\n}\n\n/**\n * Told to the model with the result that makes the third ending alike. `asks` is whether the\n * fifth will ask the user: not once they let the run go on, nor when no one can be asked.\n */\nexport function loopWarning(label: string, alike: Alike, asks: boolean): string {\n\tconst again = alike.last.ending.passed\n\t\t? 'Running it again with nothing changed tells you nothing new.'\n\t\t: 'Find out why your changes do not reach what it reports before you run it again.';\n\tconst then = asks ? 'the user is asked whether the run goes on' : 'the run stops';\n\treturn `\"${label}\" ${alike.last.ending.how} with the same output ${String(alike.times)} times in this run. ${describeChanges(alike)} ${again} If it ends this way ${String(askAt)} times, ${then}.`;\n}\n\n/** The question for the user on the fifth ending alike. */\nexport function loopQuestion(label: string, alike: Alike): UserQuestion {\n\tconst lines = alike.last.ending.output.trimEnd().split('\\n').slice(-shownLines).join('\\n');\n\tconst output =\n\t\tlines.length > shownCharacters ? `…${lines.slice(-shownCharacters)}` : lines || '(nothing)';\n\treturn {\n\t\tquestion: [\n\t\t\t`Possible loop: \"${label}\" ${alike.last.ending.how} with the same output ${String(alike.times)} times in this run. ${describeChanges(alike)}`,\n\t\t\t'It may be a check that passes after every change, which is no loop, or the model repeating itself.',\n\t\t\t`What it printed, the last lines:\\n${output}`,\n\t\t].join('\\n'),\n\t\toptions: loopOptions(`If it ends this way ${String(askAt)} more times`),\n\t\tmultiSelect: false,\n\t};\n}\n\nfunction loopOptions(again: string): UserQuestion['options'] {\n\treturn [\n\t\t{\n\t\t\tlabel: anotherAttempt,\n\t\t\tdescription: `Let the run go on, once. ${again}, or any other loop is found, the run stops without asking.`,\n\t\t},\n\t\t{\n\t\t\tlabel: stop,\n\t\t\tdescription:\n\t\t\t\t'End the run here. What it did and changed is kept, and your next prompt continues from it.',\n\t\t},\n\t];\n}\n\n/** Told to the model when the user let the run go on, with their words if they wrote some. */\nexport function extensionNote(alike: Alike, words?: string): string {\n\tconst said = words === undefined ? '' : ` They said: ${words}`;\n\treturn `It has ended this way ${String(alike.times)} times. The user let the run go on, once.${said} Change your approach: if it ends this way ${String(askAt)} more times, the run stops.`;\n}\n\n/** Why the run stopped, for the failure it ends with. */\nexport function loopStop(label: string, alike: Alike, reason: string): string {\n\treturn `\"${label}\" ${alike.last.ending.how} with the same output ${String(alike.times)} times. ${describeChanges(alike)} ${reason} The run is kept in the conversation.`;\n}\n\n/** The question for the user when the model asks for the same `calls` again, before they run. */\nexport function repeatQuestion(calls: string): UserQuestion {\n\treturn {\n\t\tquestion: [\n\t\t\t`Possible loop: the model asked for the same tool calls ${String(repeatedAnswersAt)} times in a row, with nothing else between.`,\n\t\t\t'It may be waiting for something that changes by itself, which is no loop, or repeating itself.',\n\t\t\t`The calls:\\n${calls}`,\n\t\t].join('\\n'),\n\t\toptions: loopOptions(\n\t\t\t`If it asks for the same calls ${String(repeatedAnswersAt)} more times in a row`,\n\t\t),\n\t\tmultiSelect: false,\n\t};\n}\n\n/** Told to the model, with the results of the calls, when the user let them run again. */\nexport function repeatNote(words?: string): string {\n\tconst said = words === undefined ? '' : ` They said: ${words}`;\n\treturn `You asked for these same calls ${String(repeatedAnswersAt)} times in a row. The user let the run go on, once.${said} Change your approach: if you ask for the same calls ${String(repeatedAnswersAt)} more times in a row, the run stops.`;\n}\n\n/** Why the run stopped on repeated answers, for the failure it ends with. */\nexport function repeatStop(reason: string): string {\n\treturn `The model asked for the same tool calls ${String(repeatedAnswersAt)} times in a row. ${reason} The run is kept in the conversation.`;\n}\n"]}

package/dist/engines/domain/model/llmAgent.d.ts

@@ -78,6 +78,12 @@ export declare class LLMAgent implements Agent {
     private askUser;
     /** One at a time and in order, so their effects and their narration never interleave. */
     private runTools;
+    /**
+     * On a loop found, the fifth ending alike of a command or the fifth answer in a row asking for
+     * the same calls: the note for the model if the user lets the run go on, or the failure that
+     * stops it. The run goes on once at most, and never without someone to ask.
+     */
+    private decideOnLoop;
     /**
      * Runs inside the context of the run that reached it, or in one of its own that it ends with
      * the run: completed, failed or cancelled. Only a run that owns its context names the history

package/dist/engines/domain/model/llmAgent.js

@@ -10,6 +10,7 @@ import { ReadRegistry } from '#src/tools/domain/readRegistry';
 import { InvalidToolInputError } from '#src/tools/domain/tool';
 import { isAbortError } from '#src/shared/domain/isAbortError';
 import { addTokens } from '#src/shared/domain/tokens';
+import { anotherAttempt, askAt, CommandLoops, extensionNote, loopQuestion, loopStop, loopWarning, repeatedAnswersAt, repeatNote, repeatQuestion, repeatStop, stop, warnAt, } from "./commandLoops.js";
 import { measureContext } from "./stepContext.js";
 import { askUserDefinition, askUserName, describeAnswers, readAnswers, readQuestions, summarizeQuestions, } from "./askUser.js";
 import { followHistory, transcriptOf, undoNote, } from "./historyFollower.js";
@@ -64,13 +65,17 @@ function canonical(value) {
         .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
         .map(([key, inner]) => [key, canonical(inner)]));
 }
-// A model that asks for exactly the same calls this many times in a row is stuck: it already
-// has what they return, and running them again changes nothing.
-const maxRepeatedSteps = 5;
-/** How many steps in a row have asked for exactly the same tool calls, with the same input. */
+/**
+ * How many steps in a row have asked for exactly the same tool calls, with the same input. A
+ * model that keeps asking is likely stuck: it already has what they return.
+ */
 class RepeatedCalls {
     last;
     times = 0;
+    /** The user let the run go on: the count starts again. */
+    reset() {
+        this.times = 0;
+    }
     count(calls) {
         // Keys sorted at every level: the same input with its properties in another order is the
         // same call. Arrays, and the order of the calls, keep theirs.
@@ -80,27 +85,35 @@ class RepeatedCalls {
         return this.times;
     }
 }
-// A command that ends exactly as it did the last times, in output too, this many times in a row
-// means the model's changes in between are not reaching what the command checks.
-const maxRepeatedEndings = 5;
-/**
- * How many times in a row each command call, by tool and input, has ended exactly the same way.
- * Calls in between to other tools, or to other commands, do not break the count.
- */
-class RepeatedEndings {
-    last = new Map();
-    count(call, ending) {
-        const key = JSON.stringify([call.name, canonical(call.input)]);
-        // A call that never ran a command, such as one denied, says nothing of how it ends.
-        if (ending === undefined) {
-            this.last.delete(key);
-            return 0;
-        }
-        const previous = this.last.get(key);
-        const times = previous?.ending === ending ? previous.times + 1 : 1;
-        this.last.set(key, { ending, times });
-        return times;
-    }
+/** A call by its tool and input, its properties in any order. */
+function callKey(call) {
+    return JSON.stringify([call.name, canonical(call.input)]);
+}
+// How a call the user is asked about is shown: its tool and input, cut short.
+const shownCallCharacters = 300;
+function describeCall(call) {
+    const text = `${call.name} ${JSON.stringify(canonical(call.input))}`;
+    return text.length > shownCallCharacters ? `${text.slice(0, shownCallCharacters)}…` : text;
+}
+/** The result of a call the run stopped before. */
+function notRun(call) {
+    return {
+        type: 'toolResult',
+        callId: call.id,
+        name: call.name,
+        output: 'Not run: the run stopped before it.',
+        isError: true,
+    };
+}
+/** The tool results, with `note` for the model after the last of them. */
+function withNote(message, note) {
+    const last = message.content.at(-1);
+    if (last?.type !== 'toolResult')
+        return message;
+    return {
+        ...message,
+        content: [...message.content.slice(0, -1), { ...last, output: `${last.output}\n\n${note}` }],
+    };
 }
 /**
  * An agent whose conversation is owned by MiKode: every run sends the whole context to a
@@ -370,12 +383,14 @@ export class LLMAgent {
         signal.throwIfAborted();
         // Set before `execute`, so a tool that fails halfway still counts: it may have had effects.
         effects.toolRan = true;
+        const unrecorded = prepared.unrecorded === true && prepared.risk !== 'safe';
         try {
             const output = await prepared.run(signal);
             const ending = prepared.ending?.();
             return {
                 result: { ...result, output, isError: false },
                 denied: false,
+                unrecorded,
                 ...(summary === undefined ? {} : { summary }),
                 ...(prepared.diff === undefined ? {} : { diff: prepared.diff }),
                 ...(ending === undefined ? {} : { ending }),
@@ -387,6 +402,7 @@ export class LLMAgent {
             return {
                 result: { ...result, output: describeFailure(error), isError: true },
                 denied: false,
+                unrecorded,
                 ...(summary === undefined ? {} : { summary }),
                 ...(error instanceof ToolExecutionError && error.exitCode !== undefined
                     ? { exitCode: error.exitCode }
@@ -434,18 +450,35 @@ export class LLMAgent {
     /** One at a time and in order, so their effects and their narration never interleave. */
     async runTools(calls, options, effects, toolCall, 
     // Throws the failure of a progress callback a call narrated through.
-    throwIfProgressFailed, endings) {
+    throwIfProgressFailed, loops) {
         const { signal, onProgress: callback = ignoreProgress } = options;
         const results = [];
-        for (const call of calls) {
+        for (const [index, call] of calls.entries()) {
             // A call before this one may have ignored the cancellation; this one must not start,
             // nor be announced as running.
             signal.throwIfAborted();
             // `runTool` announces the call only once it is allowed to run, so a consumer never
             // shows a call as running early, nor while the user is still being asked.
-            const { result, denied, diff, summary, exitCode, ending } = await this.runTool(call, options, effects, toolCall);
+            const ran = await this.runTool(call, options, effects, toolCall);
+            const { denied, diff, summary, exitCode, ending } = ran;
+            let { result } = ran;
             // A call that failed to narrate ends the run before the next one can start.
             throwIfProgressFailed();
+            if (ran.unrecorded)
+                loops.unrecordedCallRan();
+            const key = callKey(call);
+            const label = summary ?? call.name;
+            const alike = ending === undefined ? undefined : loops.observe(key, ending);
+            // A command that changed what the history cannot copy hides that change from the next.
+            if (ending !== undefined && ending.state === undefined)
+                loops.unrecordedCallRan();
+            // Narrated with it, so the person watching reads what the model is told.
+            if (alike?.times === warnAt) {
+                result = {
+                    ...result,
+                    output: `${result.output}\n\n${loopWarning(label, alike, options.ask !== undefined && !loops.wentOn)}`,
+                };
+            }
             if (denied) {
                 emit({
                     type: 'tool',
@@ -468,15 +501,62 @@ export class LLMAgent {
             }
             else
                 narrate(result, callback, summary, exitCode);
-            results.push(result);
-            // After its result is narrated, so the consumer sees the ending that stopped the run.
-            if (endings.count(call, ending) >= maxRepeatedEndings) {
-                throw new UnrecoverableError('The LLM kept getting the same result from a command', {
-                    cause: `${summary === undefined ? `"${call.name}"` : `"${summary}"`} ended the same way, with the same output, ${String(maxRepeatedEndings)} times in a row.`,
-                });
+            // After its result is narrated, so the user is asked about the ending they have seen.
+            if (alike !== undefined && alike.times >= askAt) {
+                const decision = await this.decideOnLoop({
+                    question: loopQuestion(label, alike),
+                    failure: 'The LLM kept getting the same result from a command',
+                    stopped: reason => loopStop(label, alike, reason),
+                    note: words => extensionNote(alike, words),
+                }, () => loops.extend([key]), options);
+                if (typeof decision !== 'string') {
+                    // So the next run's model reads why this one ended where it did.
+                    results.push({
+                        ...result,
+                        output: `${result.output}\n\nThe run stopped here: ${decision.cause}`,
+                    });
+                    // The calls after it never run, but each still gets a result, so the run that is
+                    // kept has none without one.
+                    results.push(...calls.slice(index + 1).map(notRun));
+                    return { message: { role: 'tool', content: results }, stopped: decision };
+                }
+                result = { ...result, output: `${result.output}\n\n${decision}` };
             }
+            results.push(result);
         }
-        return { role: 'tool', content: results };
+        return { message: { role: 'tool', content: results } };
+    }
+    /**
+     * On a loop found, the fifth ending alike of a command or the fifth answer in a row asking for
+     * the same calls: the note for the model if the user lets the run go on, or the failure that
+     * stops it. The run goes on once at most, and never without someone to ask.
+     */
+    async decideOnLoop(loop, extend, { ask, signal }) {
+        const stopped = (reason) => new UnrecoverableError(loop.failure, { cause: loop.stopped(reason) });
+        if (!ask)
+            return stopped('No one could be asked whether to go on.');
+        if (!extend())
+            return stopped('The user had already let the run go on once.');
+        const questions = [loop.question];
+        let answers;
+        try {
+            answers = await ask(questions, signal);
+        }
+        catch (error) {
+            signal.throwIfAborted();
+            throw classifyHostFailure(error, 'The asker failed');
+        }
+        // An answer that arrives after a cancellation is not acted on.
+        signal.throwIfAborted();
+        const [answer] = readAnswers(answers, questions) ?? [];
+        if (answer === undefined) {
+            throw new UnrecoverableError('The asker failed', {
+                cause: 'It must answer the question with some text.',
+            });
+        }
+        if (answer === stop)
+            return stopped('The user stopped it.');
+        return loop.note(answer === anotherAttempt ? undefined : answer);
     }
     /**
      * Runs inside the context of the run that reached it, or in one of its own that it ends with
@@ -596,7 +676,7 @@ export class LLMAgent {
             // After following the history, which may have made every earlier read stale.
             const reads = this.reads.begin();
             const repeats = new RepeatedCalls();
-            const endings = new RepeatedEndings();
+            const loops = new CommandLoops();
             // A tool that does its own work on the model, such as running subagents, counts it here.
             const spend = (spent, unreportedCall) => {
                 tokens = addTokens(tokens, spent);
@@ -687,15 +767,47 @@ export class LLMAgent {
                     };
                 }
                 // Before running them again: the model has already seen what they return.
-                if (repeats.count(calls) >= maxRepeatedSteps) {
-                    throw new UnrecoverableError('The LLM repeated the same tool calls', {
-                        cause: `The model asked for the same tool calls ${String(maxRepeatedSteps)} times in a row.`,
-                    });
+                let repeatedNote;
+                // Not on the last step, whose calls never run: there is nothing to let go on.
+                if (step < this.maxSteps && repeats.count(calls) >= repeatedAnswersAt) {
+                    const decision = await this.decideOnLoop({
+                        question: repeatQuestion(calls.map(describeCall).join('\n')),
+                        failure: 'The LLM repeated the same tool calls',
+                        stopped: repeatStop,
+                        note: repeatNote,
+                    }, () => loops.extend(calls.map(callKey)), options);
+                    if (typeof decision !== 'string') {
+                        // Kept like a run stopped on a command, every call given a result.
+                        const [first, ...rest] = calls.map(notRun);
+                        runMessages.push({
+                            role: 'tool',
+                            content: [
+                                ...(first
+                                    ? [
+                                        {
+                                            ...first,
+                                            output: `Not run: the run stopped here. ${decision.cause}`,
+                                        },
+                                    ]
+                                    : []),
+                                ...rest,
+                            ],
+                        });
+                        remember();
+                        throw decision;
+                    }
+                    repeats.reset();
+                    repeatedNote = decision;
                 }
                 // On the last step no call is left to send the results to, so the tools do not run.
                 if (step < this.maxSteps) {
-                    const results = await this.runTools(calls, options, effects, { run: context, reads, onProgress, spend }, throwIfProgressFailed, endings);
-                    runMessages.push(results);
+                    const { message, stopped } = await this.runTools(calls, options, effects, { run: context, reads, onProgress, spend }, throwIfProgressFailed, loops);
+                    runMessages.push(repeatedNote === undefined ? message : withNote(message, repeatedNote));
+                    // Kept like a run cut by the output limit: the user may take it up.
+                    if (stopped) {
+                        remember();
+                        throw stopped;
+                    }
                 }
             }
             throw new UnrecoverableError('The LLM did not finish within its step limit', {