@bridge4dev/runner 0.48.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude.js +60 -0
- package/dist/adapters/codex-protocol.d.ts +13 -0
- package/dist/adapters/codex-protocol.js +17 -1
- package/dist/adapters/codex.js +122 -12
- package/dist/adapters/types.d.ts +46 -2
- package/dist/agent-versions.d.ts +33 -2
- package/dist/agent-versions.js +29 -5
- package/dist/stop-cycle.d.ts +148 -0
- package/dist/stop-cycle.js +86 -0
- package/dist/supervisor.d.ts +92 -0
- package/dist/supervisor.js +509 -14
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/supervisor.js
CHANGED
|
@@ -3,6 +3,7 @@ import path from 'node:path';
|
|
|
3
3
|
import { log } from './log.js';
|
|
4
4
|
import { CONTEXT_USAGE_MIN_DELTA_RATIO, CONTEXT_USAGE_MIN_DELTA_TOKENS, RATE_LIMITS_RESEND_INTERVAL_MS, } from './levels.js';
|
|
5
5
|
import { claimAutoResume, clearAutoResume, pruneAutoResume } from './auto-resume.js';
|
|
6
|
+
import { errorVerdict, isNewWork, isStoppedToolTail, ownsProcess, turnEndVerdict, } from './stop-cycle.js';
|
|
6
7
|
import { classifyFailure, isRepeatOfSameFailure, MAX_RETRIES_PER_SESSION, retryDelayMs, } from './adapters/error-policy.js';
|
|
7
8
|
import { evaluateRecipeCommand, maskSecrets, maskString } from './policy.js';
|
|
8
9
|
import { agentPromptSizeLabel, inspectAgentPrompt, quotePath, readAgentPrompt, } from './agent-prompt.js';
|
|
@@ -97,6 +98,8 @@ export class Supervisor {
|
|
|
97
98
|
/** The window actually used — the constant, or a test's own shorter one. */
|
|
98
99
|
emptyTurnSettleMs;
|
|
99
100
|
rateLimitsResendMs;
|
|
101
|
+
/** The stop-settle window actually used — the constant, or a test's own. */
|
|
102
|
+
stopSettleMs;
|
|
100
103
|
/** A finished session's journal is kept this long for a late reconnect. */
|
|
101
104
|
static JOURNAL_TTL_MS = 72 * 3_600_000;
|
|
102
105
|
/** Backstop: events the API will never accept must not pile up forever. */
|
|
@@ -148,6 +151,7 @@ export class Supervisor {
|
|
|
148
151
|
this.journals = opts.journals ?? new JournalStore();
|
|
149
152
|
this.emptyTurnSettleMs = opts.emptyTurnSettleMs ?? Supervisor.EMPTY_TURN_SETTLE_MS;
|
|
150
153
|
this.rateLimitsResendMs = opts.rateLimitsResendMs ?? RATE_LIMITS_RESEND_INTERVAL_MS;
|
|
154
|
+
this.stopSettleMs = opts.stopSettleMs ?? Supervisor.STOP_SETTLE_MS;
|
|
151
155
|
this.verify = new VerifyRunner({
|
|
152
156
|
enabled: opts.verifyEnabled !== false,
|
|
153
157
|
onReport: (report) => {
|
|
@@ -764,6 +768,10 @@ export class Supervisor {
|
|
|
764
768
|
extraBudgetMinutes: descriptor.extraBudgetMinutes,
|
|
765
769
|
epoch: descriptor.epoch,
|
|
766
770
|
openQuestions: new Set(),
|
|
771
|
+
openPermissions: new Set(),
|
|
772
|
+
liveToolUses: new Set(),
|
|
773
|
+
pauseEpoch: 0,
|
|
774
|
+
processSeq: 0,
|
|
767
775
|
answeredAsks: new Set(),
|
|
768
776
|
deliveredMessageIds: new Set(),
|
|
769
777
|
backgroundTasks: 0,
|
|
@@ -855,9 +863,18 @@ export class Supervisor {
|
|
|
855
863
|
// where the agent left off» there is the one thing the feed must not do.
|
|
856
864
|
// It happens for real: a process that dies before it reports its session
|
|
857
865
|
// id (the SIGABRT this ticket came from) leaves the row with none.
|
|
866
|
+
//
|
|
867
|
+
// Ticket #370: the sentence about the agent's MEMORY waits for proof.
|
|
868
|
+
// It used to be written here, on the strength of a stored id and before
|
|
869
|
+
// the CLI had been asked anything — and on Athanor it appeared sixty
|
|
870
|
+
// seconds before `thread/resume` timed out and took the session with it.
|
|
871
|
+
// What is honest at this moment is that the runner is back; whether the
|
|
872
|
+
// conversation reopens is answered by the process, in `provider_session`.
|
|
873
|
+
if (descriptor.providerSessionId)
|
|
874
|
+
running.resumeClaimPending = true;
|
|
858
875
|
this.sendEvent(running, 'system_note', {
|
|
859
876
|
text: descriptor.providerSessionId
|
|
860
|
-
? 'Session resumed —
|
|
877
|
+
? 'Session resumed — send a message and the agent picks its conversation back up.'
|
|
861
878
|
: 'Session resumed, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
|
|
862
879
|
});
|
|
863
880
|
}
|
|
@@ -967,6 +984,14 @@ export class Supervisor {
|
|
|
967
984
|
// apply it never started (ticket #225). The feed has already told them the
|
|
968
985
|
// conversation was cut.
|
|
969
986
|
const rewind = running.rewindAnchor;
|
|
987
|
+
// #373: a new process is a new subject. Everything the previous one left
|
|
988
|
+
// behind — the stop it was going through, the tools it had in flight, the
|
|
989
|
+
// cards it was parked on — belongs to a process that no longer exists, and
|
|
990
|
+
// carrying any of it over would let an old cancellation swallow a new
|
|
991
|
+
// failure.
|
|
992
|
+
delete running.stopCycle;
|
|
993
|
+
running.liveToolUses.clear();
|
|
994
|
+
running.openPermissions.clear();
|
|
970
995
|
try {
|
|
971
996
|
const agentPrompt = this.resolveAgentPrompt(running, resuming && !rewinding);
|
|
972
997
|
const workspaceContext = composeWorkspaceContext(descriptor, agentPrompt?.text);
|
|
@@ -977,6 +1002,15 @@ export class Supervisor {
|
|
|
977
1002
|
// from. Its transcript is still on disk — that is what makes the point
|
|
978
1003
|
// usable at all.
|
|
979
1004
|
const resumeTarget = rewind ? rewind.agentSession : resumeId;
|
|
1005
|
+
// #370: whether the promise about the agent's memory has anything to
|
|
1006
|
+
// prove. A launch with no conversation to reopen can never confirm one.
|
|
1007
|
+
running.launchResumed = Boolean(resumeTarget);
|
|
1008
|
+
// Nothing to prove on a launch that opens no conversation (a stale-resume
|
|
1009
|
+
// recovery nulls the id) — and a promise left standing would be kept
|
|
1010
|
+
// later, by a relaunch that had nothing to do with it.
|
|
1011
|
+
if (!running.launchResumed)
|
|
1012
|
+
delete running.resumeClaimPending;
|
|
1013
|
+
running.processSeq += 1;
|
|
980
1014
|
running.session = adapter.startSession({
|
|
981
1015
|
sessionId: descriptor.id,
|
|
982
1016
|
cwd: running.worktreePath,
|
|
@@ -1368,6 +1402,16 @@ export class Supervisor {
|
|
|
1368
1402
|
// Guarded like the event loop above — a journal that cannot be written (a
|
|
1369
1403
|
// full disk, a daemon already shutting down) must not take the exit path
|
|
1370
1404
|
// with it, because everything below it is cleanup.
|
|
1405
|
+
// #373: the stop this process was going through has reached its end. Marked
|
|
1406
|
+
// before the cleanup below so anything still queued behind it is judged
|
|
1407
|
+
// against a cycle that knows the process is gone.
|
|
1408
|
+
if (running.stopCycle && running.stopCycle.session === session) {
|
|
1409
|
+
this.clearStopSettleTimer(running.stopCycle);
|
|
1410
|
+
running.stopCycle.phase = 'parked';
|
|
1411
|
+
}
|
|
1412
|
+
// Both sets describe THIS process and nothing else (#373).
|
|
1413
|
+
running.liveToolUses.clear();
|
|
1414
|
+
running.openPermissions.clear();
|
|
1371
1415
|
try {
|
|
1372
1416
|
this.clearApiRetry(running);
|
|
1373
1417
|
this.withdrawOpenQuestions(running, 'session_stopped');
|
|
@@ -1797,6 +1841,17 @@ export class Supervisor {
|
|
|
1797
1841
|
ok: event.ok,
|
|
1798
1842
|
errorMessage: event.errorMessage,
|
|
1799
1843
|
...(event.aborted ? { aborted: true } : {}),
|
|
1844
|
+
// #258 / #373: two facts the adapters have always measured and this line
|
|
1845
|
+
// has always dropped on the floor. `limitBlocked` is what tells the API a
|
|
1846
|
+
// turn was REFUSED rather than run, and `produced` is what tells it the
|
|
1847
|
+
// turn did something — without which the phantom ending a resumed process
|
|
1848
|
+
// emits (#300) resets the «three limit pauses in a row» counter one second
|
|
1849
|
+
// after the third one, and «woke → refused → slept» never stops.
|
|
1850
|
+
...(event.limitBlocked ? { limitBlocked: true } : {}),
|
|
1851
|
+
// ALWAYS present, true or false — the absence of this field is what the
|
|
1852
|
+
// API reads as «this runner is older than the fix», and a new runner that
|
|
1853
|
+
// sometimes omitted it would be indistinguishable from one.
|
|
1854
|
+
produced: event.produced === true,
|
|
1800
1855
|
});
|
|
1801
1856
|
// …and whatever the adapter measures once the turn is over (Claude does,
|
|
1802
1857
|
// asynchronously) is news, not a step of the same turn.
|
|
@@ -1970,6 +2025,13 @@ export class Supervisor {
|
|
|
1970
2025
|
isParkable(running) {
|
|
1971
2026
|
return ((running.lastReported === 'REVIEW' || running.lastReported === 'WAITING_INPUT') &&
|
|
1972
2027
|
running.openQuestions.size === 0 &&
|
|
2028
|
+
// #373. `stop()` answers every pending permission «denied» in the RUNNER's
|
|
2029
|
+
// name, and the status is not a reliable reading of «a card is open»: a
|
|
2030
|
+
// pause reports a resting status the instant it interrupts, and a card
|
|
2031
|
+
// resolved by POLICY never moves the status back. So the seat-freeing path
|
|
2032
|
+
// needs the same guard the pause has — without it `ensureCapacity` walks
|
|
2033
|
+
// straight past invariant 8 and denies the card to make room.
|
|
2034
|
+
running.openPermissions.size === 0 &&
|
|
1973
2035
|
Boolean(running.descriptor.providerSessionId));
|
|
1974
2036
|
}
|
|
1975
2037
|
/**
|
|
@@ -2063,7 +2125,7 @@ export class Supervisor {
|
|
|
2063
2125
|
// and Codex's own «interrupted, so start the next queued turn» both come
|
|
2064
2126
|
// through no frame this runner could refuse.
|
|
2065
2127
|
if (Supervisor.isPaused(running))
|
|
2066
|
-
this.stopWorkUnderPause(running);
|
|
2128
|
+
this.stopWorkUnderPause(running, event);
|
|
2067
2129
|
else
|
|
2068
2130
|
this.noteAgentIsWorking(running);
|
|
2069
2131
|
}
|
|
@@ -2073,6 +2135,16 @@ export class Supervisor {
|
|
|
2073
2135
|
this.reportStatus(descriptor.id, statusForReport(running), {
|
|
2074
2136
|
providerSessionId: event.providerSessionId,
|
|
2075
2137
|
});
|
|
2138
|
+
// #370: the promise, now that it can be kept. Both CLIs answer a REFUSED
|
|
2139
|
+
// reopen with no session of any kind — Claude emits no `system:init` at
|
|
2140
|
+
// all, Codex throws before it gets here — so this event is the proof the
|
|
2141
|
+
// conversation really came back, and the only honest moment to say so.
|
|
2142
|
+
if (running.resumeClaimPending && running.launchResumed) {
|
|
2143
|
+
delete running.resumeClaimPending;
|
|
2144
|
+
this.sendEvent(running, 'system_note', {
|
|
2145
|
+
text: 'The agent still has this conversation — carry on where it left off.',
|
|
2146
|
+
});
|
|
2147
|
+
}
|
|
2076
2148
|
// The CLI resumed at the anchor and forked, so the cut announced when
|
|
2077
2149
|
// the rewind was asked for stands. Nothing to send — it is already in
|
|
2078
2150
|
// the feed; this only retires the promise to take it back.
|
|
@@ -2090,6 +2162,39 @@ export class Supervisor {
|
|
|
2090
2162
|
});
|
|
2091
2163
|
return;
|
|
2092
2164
|
case 'turn_end': {
|
|
2165
|
+
// #373: a stop in flight owns the endings of the process it is stopping.
|
|
2166
|
+
// The incident was two results 1.2 seconds apart — the first the stop we
|
|
2167
|
+
// asked for, the second `error_during_execution` — and the second one
|
|
2168
|
+
// filed a paused session as FAILED, which is terminal, with a live clock
|
|
2169
|
+
// still on it.
|
|
2170
|
+
const cycle = running.stopCycle;
|
|
2171
|
+
if (cycle && ownsProcess(cycle, running.session)) {
|
|
2172
|
+
const verdict = turnEndVerdict(cycle);
|
|
2173
|
+
if (verdict === 'tail') {
|
|
2174
|
+
log.info('supervisor: a late turn ending belongs to the stop', {
|
|
2175
|
+
sessionId: descriptor.id,
|
|
2176
|
+
process: running.processSeq,
|
|
2177
|
+
reason: cycle.reason,
|
|
2178
|
+
phase: cycle.phase,
|
|
2179
|
+
ok: event.ok,
|
|
2180
|
+
});
|
|
2181
|
+
return;
|
|
2182
|
+
}
|
|
2183
|
+
if (verdict === 'stopped') {
|
|
2184
|
+
cycle.turnEndSent = true;
|
|
2185
|
+
this.publishStoppedTurn(running, descriptor, event);
|
|
2186
|
+
this.settleStopCycle(running, cycle);
|
|
2187
|
+
return;
|
|
2188
|
+
}
|
|
2189
|
+
// `ordinary` — a manual Stop keeps the contract it has had since
|
|
2190
|
+
// QA-120: this ending goes out unchanged, and the next genuine failure
|
|
2191
|
+
// is not hidden behind it (`errorVerdict` answers `failure` for a
|
|
2192
|
+
// `user` cycle whatever its phase). Settled rather than deleted, so
|
|
2193
|
+
// the acknowledgement still on its way finds its cycle and writes the
|
|
2194
|
+
// one line this stop owes the feed — the result can beat the ACK.
|
|
2195
|
+
this.clearStopSettleTimer(cycle);
|
|
2196
|
+
cycle.settled = true;
|
|
2197
|
+
}
|
|
2093
2198
|
// #252: decided BEFORE the frame goes out. A `turn_end{ok:false}` files
|
|
2094
2199
|
// the session as FAILED, and FAILED is terminal — a session we intend to
|
|
2095
2200
|
// retry must never be told it has ended. So a turn that is going to be
|
|
@@ -2188,6 +2293,31 @@ export class Supervisor {
|
|
|
2188
2293
|
});
|
|
2189
2294
|
return;
|
|
2190
2295
|
}
|
|
2296
|
+
// #373, invariant 4. Everything above this line is a named, recoverable
|
|
2297
|
+
// condition and keeps its own handling; what is left is «an error we do
|
|
2298
|
+
// not recognise», and that is the branch that files the session FAILED.
|
|
2299
|
+
// A process this runner is closing on purpose produces exactly that on
|
|
2300
|
+
// its way out, and a pause must not end in a terminal status.
|
|
2301
|
+
//
|
|
2302
|
+
// The verdict, not a string match: a process that fell over while we
|
|
2303
|
+
// had merely ASKED it to stop has genuinely failed and still says so
|
|
2304
|
+
// (`processGone` on a cycle that closed nothing).
|
|
2305
|
+
{
|
|
2306
|
+
const cycle = running.stopCycle;
|
|
2307
|
+
if (cycle &&
|
|
2308
|
+
ownsProcess(cycle, running.session) &&
|
|
2309
|
+
errorVerdict(cycle, event.processGone === true) === 'tail') {
|
|
2310
|
+
log.info('supervisor: an error from the process we are closing', {
|
|
2311
|
+
sessionId: descriptor.id,
|
|
2312
|
+
process: running.processSeq,
|
|
2313
|
+
reason: cycle.reason,
|
|
2314
|
+
phase: cycle.phase,
|
|
2315
|
+
processGone: event.processGone === true,
|
|
2316
|
+
code: event.code ?? null,
|
|
2317
|
+
});
|
|
2318
|
+
return;
|
|
2319
|
+
}
|
|
2320
|
+
}
|
|
2191
2321
|
// Forward the code: the API stores the payload as-is, so the dashboard
|
|
2192
2322
|
// can offer "Sign in" instead of a dead error card.
|
|
2193
2323
|
this.sendEvent(running, 'error', {
|
|
@@ -2206,6 +2336,10 @@ export class Supervisor {
|
|
|
2206
2336
|
});
|
|
2207
2337
|
return;
|
|
2208
2338
|
case 'permission':
|
|
2339
|
+
// #373, invariant 8: a card the agent is parked on. Parking the session
|
|
2340
|
+
// would answer it «no» in the runner's name, so the pause stops this
|
|
2341
|
+
// process but leaves it open.
|
|
2342
|
+
running.openPermissions.add(event.requestId);
|
|
2209
2343
|
this.sendEvent(running, 'permission', {
|
|
2210
2344
|
requestId: event.requestId,
|
|
2211
2345
|
toolName: event.toolName,
|
|
@@ -2218,6 +2352,7 @@ export class Supervisor {
|
|
|
2218
2352
|
});
|
|
2219
2353
|
return;
|
|
2220
2354
|
case 'permission_resolved':
|
|
2355
|
+
running.openPermissions.delete(event.requestId);
|
|
2221
2356
|
this.sendEvent(running, 'permission_resolved', {
|
|
2222
2357
|
requestId: event.requestId,
|
|
2223
2358
|
allow: event.allow,
|
|
@@ -2375,6 +2510,15 @@ export class Supervisor {
|
|
|
2375
2510
|
return;
|
|
2376
2511
|
}
|
|
2377
2512
|
case 'tool':
|
|
2513
|
+
// #373: which calls are in flight, so a stop can tell its own tail from
|
|
2514
|
+
// work starting up. Kept here rather than in the adapters — both of them
|
|
2515
|
+
// already report the two phases, and one reader is one place to be wrong.
|
|
2516
|
+
if (event.toolUseId) {
|
|
2517
|
+
if (event.phase === 'use')
|
|
2518
|
+
running.liveToolUses.add(event.toolUseId);
|
|
2519
|
+
else
|
|
2520
|
+
running.liveToolUses.delete(event.toolUseId);
|
|
2521
|
+
}
|
|
2378
2522
|
this.sendEvent(running, 'tool', {
|
|
2379
2523
|
phase: event.phase,
|
|
2380
2524
|
name: event.name,
|
|
@@ -2829,6 +2973,14 @@ export class Supervisor {
|
|
|
2829
2973
|
* Now it goes back in the queue, exactly like a parked process below, and
|
|
2830
2974
|
* the feed says why.
|
|
2831
2975
|
*/
|
|
2976
|
+
// #373: a new turn on this process ends the previous stop's ownership of
|
|
2977
|
+
// it. Dropped HERE and not when the clock came off, because this is the
|
|
2978
|
+
// moment the endings stop being the stop's own — a release alone leaves
|
|
2979
|
+
// the interrupt in flight and its second result still coming.
|
|
2980
|
+
if (running.stopCycle && running.stopCycle.session === running.session) {
|
|
2981
|
+
this.clearStopSettleTimer(running.stopCycle);
|
|
2982
|
+
delete running.stopCycle;
|
|
2983
|
+
}
|
|
2832
2984
|
if (!running.session.send(text)) {
|
|
2833
2985
|
log.warn('supervisor: the agent process refused the message — requeued', {
|
|
2834
2986
|
sessionId: running.descriptor.id,
|
|
@@ -3054,23 +3206,329 @@ export class Supervisor {
|
|
|
3054
3206
|
running.interruptWhenReady = true;
|
|
3055
3207
|
return;
|
|
3056
3208
|
}
|
|
3209
|
+
const open = running.stopCycle;
|
|
3210
|
+
if (open && ownsProcess(open, running.session) && !open.settled) {
|
|
3211
|
+
// #357 part 2, and invariant 2 of the pause plan. Ten frames arrive while
|
|
3212
|
+
// the CLI is waiting out a provider retry — from a phone and a laptop at
|
|
3213
|
+
// once, or from a person pressing a button that gave them no sign it had
|
|
3214
|
+
// been heard. One stop, one line in the feed, one status report; the rest
|
|
3215
|
+
// wait for the same answer.
|
|
3216
|
+
//
|
|
3217
|
+
// A clock landing on a Stop the person pressed a second earlier is the
|
|
3218
|
+
// same stop with a stricter reason: the CLI has one outstanding interrupt
|
|
3219
|
+
// and answers it once, but the ending now belongs to a PAUSE, which closes
|
|
3220
|
+
// the process. Left as `user` this cycle would step aside at the first
|
|
3221
|
+
// ending and the second result would file the session FAILED — the
|
|
3222
|
+
// incident, through a door the fix did not cover.
|
|
3223
|
+
if (reason === 'pause' && open.reason === 'user') {
|
|
3224
|
+
open.reason = 'pause';
|
|
3225
|
+
log.info('supervisor: a clock landed on a stop already in flight', { sessionId });
|
|
3226
|
+
}
|
|
3227
|
+
log.info('supervisor: a stop is already in flight — joining it', {
|
|
3228
|
+
sessionId,
|
|
3229
|
+
reason,
|
|
3230
|
+
phase: open.phase,
|
|
3231
|
+
});
|
|
3232
|
+
await open.inFlight?.catch(() => undefined);
|
|
3233
|
+
return;
|
|
3234
|
+
}
|
|
3235
|
+
await this.beginStopCycle(running, reason, announce);
|
|
3236
|
+
}
|
|
3237
|
+
/**
|
|
3238
|
+
* How long a stop waits for the ending that should follow it (#373).
|
|
3239
|
+
*
|
|
3240
|
+
* The transition out of `interrupting` is the first result, not the
|
|
3241
|
+
* acknowledgement — an ACK only says the request was heard, and the incident
|
|
3242
|
+
* shows an ACK and a result are different events that arrive in either order.
|
|
3243
|
+
* This is the backstop for the case where no result comes at all: an idle
|
|
3244
|
+
* session had no turn to end, and a CLI can always simply not answer.
|
|
3245
|
+
*/
|
|
3246
|
+
static STOP_SETTLE_MS = 5_000;
|
|
3247
|
+
/**
|
|
3248
|
+
* How long after one stop the next output is still read as its tail (#196).
|
|
3249
|
+
*
|
|
3250
|
+
* The floor under the stop cycle rather than a substitute for it: while a
|
|
3251
|
+
* cycle is open every event joins it, and this is what keeps a burst from
|
|
3252
|
+
* opening a NEW cycle per line in the window after one closed without closing
|
|
3253
|
+
* its process. Far shorter than the gap before a genuinely new turn — a
|
|
3254
|
+
* background subagent finishing — which still gets its own stop.
|
|
3255
|
+
*/
|
|
3256
|
+
static PAUSE_BURST_MS = 2_000;
|
|
3257
|
+
/**
|
|
3258
|
+
* Stop one agent process once, and own what stopping it produces (#373).
|
|
3259
|
+
*
|
|
3260
|
+
* The shape is deliberate: the interrupt is a round trip that can take half a
|
|
3261
|
+
* minute (a CLI sitting out a provider retry answers only when it comes back
|
|
3262
|
+
* — 28.5 seconds, measured in #357), so nothing here may be on the path the
|
|
3263
|
+
* event pump takes. The cycle is stored before the request goes out, which is
|
|
3264
|
+
* what makes every later request join this one instead of starting a second.
|
|
3265
|
+
*/
|
|
3266
|
+
async beginStopCycle(running, reason, announce) {
|
|
3267
|
+
const session = running.session;
|
|
3268
|
+
if (!session)
|
|
3269
|
+
return;
|
|
3270
|
+
const sessionId = running.descriptor.id;
|
|
3271
|
+
const cycle = {
|
|
3272
|
+
session,
|
|
3273
|
+
reason,
|
|
3274
|
+
epoch: running.pauseEpoch,
|
|
3275
|
+
phase: 'interrupting',
|
|
3276
|
+
inFlight: null,
|
|
3277
|
+
outcome: null,
|
|
3278
|
+
refused: false,
|
|
3279
|
+
closed: false,
|
|
3280
|
+
turnEndSent: false,
|
|
3281
|
+
settled: false,
|
|
3282
|
+
hadTurn: running.lastReported === 'RUNNING' || running.lastReported === 'STARTING',
|
|
3283
|
+
toolsAtStop: new Set(running.liveToolUses),
|
|
3284
|
+
};
|
|
3285
|
+
running.stopCycle = cycle;
|
|
3286
|
+
// Kept for the cool-down in `stopWorkUnderPause`: a cycle that ends without
|
|
3287
|
+
// closing the process must not be replaced by another one on the next event.
|
|
3288
|
+
running.pauseInterruptAt = Date.now();
|
|
3289
|
+
log.info('supervisor: stopping the agent', {
|
|
3290
|
+
sessionId,
|
|
3291
|
+
reason,
|
|
3292
|
+
process: running.processSeq,
|
|
3293
|
+
epoch: cycle.epoch,
|
|
3294
|
+
status: running.lastReported,
|
|
3295
|
+
hadTurn: cycle.hadTurn,
|
|
3296
|
+
toolsInFlight: cycle.toolsAtStop.size,
|
|
3297
|
+
openQuestions: running.openQuestions.size,
|
|
3298
|
+
openPermissions: running.openPermissions.size,
|
|
3299
|
+
});
|
|
3057
3300
|
// The turn being interrupted is the turn the question belongs to — leaving
|
|
3058
3301
|
// the ask parked would make the user's next message be swallowed as its
|
|
3059
3302
|
// answer (QA-106 m7). The adapter reports each withdrawal itself; the local
|
|
3060
3303
|
// set is cleared so the session can be parked again.
|
|
3061
|
-
|
|
3304
|
+
session.cancelQuestions('turn_aborted');
|
|
3062
3305
|
running.openQuestions.clear();
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3306
|
+
// Armed BEFORE the request goes out, not after it answers. A control channel
|
|
3307
|
+
// that never answers is a real case — it is half of #357 — and a stop that
|
|
3308
|
+
// waited for an acknowledgement before starting its own clock would hold the
|
|
3309
|
+
// session, its seat and the pause for as long as the CLI stayed silent.
|
|
3310
|
+
this.armStopSettle(running, cycle);
|
|
3311
|
+
const inFlight = (async () => {
|
|
3312
|
+
let outcome;
|
|
3313
|
+
try {
|
|
3314
|
+
outcome = await session.interrupt();
|
|
3315
|
+
}
|
|
3316
|
+
catch (error) {
|
|
3317
|
+
// An adapter is not supposed to throw here, and one that does has told
|
|
3318
|
+
// us nothing about whether the turn stopped — which is exactly the
|
|
3319
|
+
// «refused» case.
|
|
3320
|
+
outcome = 'refused';
|
|
3321
|
+
log.warn('supervisor: the interrupt request threw', {
|
|
3322
|
+
sessionId,
|
|
3323
|
+
error: String(error),
|
|
3324
|
+
});
|
|
3325
|
+
}
|
|
3326
|
+
// A newer cycle (or a new process) took over while we waited: this answer
|
|
3327
|
+
// is about a stop nobody is waiting for any more.
|
|
3328
|
+
if (running.stopCycle !== cycle)
|
|
3329
|
+
return;
|
|
3330
|
+
cycle.outcome = outcome;
|
|
3331
|
+
// Invariant 7: a refusal that arrives after WE closed this process is
|
|
3332
|
+
// void. `stop()` rejects every outstanding control request on close, and
|
|
3333
|
+
// reading our own park as a failure is the defect with the sign flipped.
|
|
3334
|
+
cycle.refused = outcome === 'refused' && cycle.phase === 'interrupting';
|
|
3335
|
+
if (announce) {
|
|
3336
|
+
this.sendEvent(running, 'notice', {
|
|
3337
|
+
level: 'info',
|
|
3338
|
+
text: reason === 'pause'
|
|
3339
|
+
? 'Paused — the turn was stopped. Nothing else will start until the clock runs out.'
|
|
3340
|
+
: 'Turn interrupted by the user',
|
|
3341
|
+
});
|
|
3342
|
+
}
|
|
3343
|
+
const next = running.descriptor.kind === 'CHAT' ? 'WAITING_INPUT' : 'REVIEW';
|
|
3344
|
+
this.reportStatus(sessionId, next, {
|
|
3345
|
+
costUsd: running.costUsd,
|
|
3346
|
+
activeMs: Supervisor.spentMs(running),
|
|
3347
|
+
});
|
|
3348
|
+
this.armStopSettle(running, cycle);
|
|
3349
|
+
})();
|
|
3350
|
+
cycle.inFlight = inFlight;
|
|
3351
|
+
await inFlight;
|
|
3352
|
+
}
|
|
3353
|
+
/**
|
|
3354
|
+
* Wait a bounded time for the ending an accepted interrupt should produce.
|
|
3355
|
+
*
|
|
3356
|
+
* «Bounded» rather than «for ever» because two real cases produce no ending
|
|
3357
|
+
* at all: a pause landing between turns (there was nothing to stop), and a
|
|
3358
|
+
* CLI that takes the request and then says nothing. Leaving the cycle open
|
|
3359
|
+
* there would hold the seat and the clock indefinitely.
|
|
3360
|
+
*/
|
|
3361
|
+
armStopSettle(running, cycle) {
|
|
3362
|
+
// The result can beat the acknowledgement — they are different events and
|
|
3363
|
+
// the incident shows both orders. If it already has, this cycle is settled
|
|
3364
|
+
// and there is nothing left to wait for.
|
|
3365
|
+
if (cycle.phase !== 'interrupting' || running.stopCycle !== cycle)
|
|
3366
|
+
return;
|
|
3367
|
+
if (this.isStale(running))
|
|
3368
|
+
return;
|
|
3369
|
+
this.clearStopSettleTimer(cycle);
|
|
3370
|
+
const timer = setTimeout(() => {
|
|
3371
|
+
delete cycle.settleTimer;
|
|
3372
|
+
if (running.stopCycle !== cycle || this.isStale(running))
|
|
3373
|
+
return;
|
|
3374
|
+
const midTurn = running.lastReported === 'RUNNING' || running.lastReported === 'STARTING';
|
|
3375
|
+
if (cycle.reason === 'user' && (cycle.outcome === null || midTurn)) {
|
|
3376
|
+
// A manual Stop never forces: #357's own rule is that the turn stops
|
|
3377
|
+
// when the CLI comes back, and the measured wait there was 28.5 seconds
|
|
3378
|
+
// — a provider retry the person cannot see. So this cycle stands for as
|
|
3379
|
+
// long as the turn does, not for five seconds: the guarantee it carries
|
|
3380
|
+
// is «one stop per turn», and a stop that expired underneath a running
|
|
3381
|
+
// turn would let the eleventh press become a second interrupt.
|
|
3382
|
+
this.armStopSettle(running, cycle);
|
|
3383
|
+
return;
|
|
3384
|
+
}
|
|
3385
|
+
if (!cycle.turnEndSent && cycle.hadTurn && cycle.reason === 'pause') {
|
|
3386
|
+
// A turn was running and its ending never came. One goes out anyway:
|
|
3387
|
+
// the feed's readers retire a turn on `turn_end`, and a turn with no
|
|
3388
|
+
// ending leaves the tray counting work that stopped minutes ago.
|
|
3389
|
+
cycle.turnEndSent = true;
|
|
3390
|
+
this.publishStoppedTurn(running, running.descriptor, { ok: true, aborted: true });
|
|
3391
|
+
}
|
|
3392
|
+
log.info('supervisor: the stop produced no ending in time', {
|
|
3393
|
+
sessionId: running.descriptor.id,
|
|
3394
|
+
process: running.processSeq,
|
|
3395
|
+
reason: cycle.reason,
|
|
3396
|
+
outcome: cycle.outcome,
|
|
3397
|
+
hadTurn: cycle.hadTurn,
|
|
3070
3398
|
});
|
|
3399
|
+
this.settleStopCycle(running, cycle);
|
|
3400
|
+
}, this.stopSettleMs);
|
|
3401
|
+
timer.unref?.();
|
|
3402
|
+
cycle.settleTimer = timer;
|
|
3403
|
+
}
|
|
3404
|
+
clearStopSettleTimer(cycle) {
|
|
3405
|
+
if (cycle.settleTimer) {
|
|
3406
|
+
clearTimeout(cycle.settleTimer);
|
|
3407
|
+
delete cycle.settleTimer;
|
|
3071
3408
|
}
|
|
3072
|
-
|
|
3073
|
-
|
|
3409
|
+
}
|
|
3410
|
+
/**
|
|
3411
|
+
* The turn is over — close the process, keeping the session resumable.
|
|
3412
|
+
*
|
|
3413
|
+
* This is the change the whole plan is about. A pause used to leave a live
|
|
3414
|
+
* CLI standing for up to five hours, which is where the second result came
|
|
3415
|
+
* from; now the runner takes the ending it asked for and parks, and the next
|
|
3416
|
+
* message (or the clock's own «continue») starts a process that resumes the
|
|
3417
|
+
* same conversation.
|
|
3418
|
+
*/
|
|
3419
|
+
settleStopCycle(running, cycle) {
|
|
3420
|
+
if (running.stopCycle !== cycle)
|
|
3421
|
+
return;
|
|
3422
|
+
this.clearStopSettleTimer(cycle);
|
|
3423
|
+
cycle.settled = true;
|
|
3424
|
+
// Settled, never DELETED here. The cycle goes on owning this process's
|
|
3425
|
+
// endings — a late second result after a stop that closed nothing is the
|
|
3426
|
+
// incident itself — and is dropped only when a new turn starts on this
|
|
3427
|
+
// process (`deliverMessage`) or a new process starts (`launchAgent`).
|
|
3428
|
+
const refusal = cycle.reason === 'user' ? 'manual-stop' : this.parkRefusalUnderPause(running);
|
|
3429
|
+
if (refusal) {
|
|
3430
|
+
// A manual Stop closes nothing by design: the person stopped a turn, not
|
|
3431
|
+
// the session. A pause may be refused the close for its own reasons — an
|
|
3432
|
+
// open permission card, a conversation the CLI has not named yet.
|
|
3433
|
+
log.info('supervisor: the stopped process stays open', {
|
|
3434
|
+
sessionId: running.descriptor.id,
|
|
3435
|
+
reason: cycle.reason,
|
|
3436
|
+
why: refusal,
|
|
3437
|
+
});
|
|
3438
|
+
this.sayIfTheStopWasRefused(running, cycle, false);
|
|
3439
|
+
return;
|
|
3440
|
+
}
|
|
3441
|
+
cycle.phase = 'parking';
|
|
3442
|
+
cycle.closed = true;
|
|
3443
|
+
this.sayIfTheStopWasRefused(running, cycle, true);
|
|
3444
|
+
this.park(running, { quiet: true });
|
|
3445
|
+
}
|
|
3446
|
+
/**
|
|
3447
|
+
* The agent never said yes — say so, and say only what actually happened.
|
|
3448
|
+
*
|
|
3449
|
+
* Invariant 5: a stop the CLI refused must not read as a clean cancellation.
|
|
3450
|
+
* Which sentence is true depends on what came next, so it is written here and
|
|
3451
|
+
* not where the refusal was noticed — a line promising «its process was
|
|
3452
|
+
* closed» over a manual Stop, which closes nothing, is the same class of lie
|
|
3453
|
+
* as the one this ticket is about (gotcha 324).
|
|
3454
|
+
*/
|
|
3455
|
+
sayIfTheStopWasRefused(running, cycle, closing) {
|
|
3456
|
+
if (!cycle.refused)
|
|
3457
|
+
return;
|
|
3458
|
+
cycle.refused = false; // one sentence per stop
|
|
3459
|
+
this.sendEvent(running, 'notice', {
|
|
3460
|
+
level: 'warn',
|
|
3461
|
+
text: closing
|
|
3462
|
+
? 'The agent did not confirm the stop, so its process was closed. Nothing was lost — the conversation continues when the session does.'
|
|
3463
|
+
: 'The agent did not confirm the stop, so the turn may still be running. Nothing has been lost.',
|
|
3464
|
+
});
|
|
3465
|
+
}
|
|
3466
|
+
/**
|
|
3467
|
+
* Why this paused process must NOT be closed, or null when it may be.
|
|
3468
|
+
*
|
|
3469
|
+
* Invariant 8 lives here. `park()` calls the adapter's `stop()`, and `stop()`
|
|
3470
|
+
* answers every pending permission «denied» in the RUNNER's name — a decision
|
|
3471
|
+
* nobody made, in a place that keeps decisions for ninety days. Worse, the API
|
|
3472
|
+
* sends nothing when the clock is lifted over an open card
|
|
3473
|
+
* (`pauseInterruptedTurn` is false while `openAsks > 0`), so the session would
|
|
3474
|
+
* simply stand there, silent, with the card gone.
|
|
3475
|
+
*
|
|
3476
|
+
* A conversation the CLI has not named yet is the other refusal: parking
|
|
3477
|
+
* promises the context is saved, and there is nothing to resume from.
|
|
3478
|
+
*/
|
|
3479
|
+
parkRefusalUnderPause(running) {
|
|
3480
|
+
if (!running.session)
|
|
3481
|
+
return 'no-process';
|
|
3482
|
+
if (running.stopRequested || running.budgetSpent)
|
|
3483
|
+
return 'session-ending';
|
|
3484
|
+
// The clock came off while the stop was in flight. There is nothing left to
|
|
3485
|
+
// park FOR, and closing the process now would make the old operation stop
|
|
3486
|
+
// the new work — the acceptance matrix says it must not. The cycle stays,
|
|
3487
|
+
// settled, so the ending still on its way is still its own.
|
|
3488
|
+
if (!Supervisor.isPaused(running))
|
|
3489
|
+
return 'clock-lifted';
|
|
3490
|
+
if (running.openPermissions.size > 0)
|
|
3491
|
+
return 'open-permission';
|
|
3492
|
+
if (running.openQuestions.size > 0)
|
|
3493
|
+
return 'open-question';
|
|
3494
|
+
if (!running.descriptor.providerSessionId)
|
|
3495
|
+
return 'no-conversation';
|
|
3496
|
+
return null;
|
|
3497
|
+
}
|
|
3498
|
+
/**
|
|
3499
|
+
* The one ending a stop cycle publishes (#373).
|
|
3500
|
+
*
|
|
3501
|
+
* `ok` unconditionally: a turn this runner stopped is not a turn that failed,
|
|
3502
|
+
* and `turn_end{ok:false}` is what put a paused session into the terminal
|
|
3503
|
+
* status it could never be woken out of. The facts the turn actually carried
|
|
3504
|
+
* ride along — `limitBlocked` because the API counts refusals with it, and
|
|
3505
|
+
* `produced` because it is what keeps the refusal counter honest (#300).
|
|
3506
|
+
*/
|
|
3507
|
+
publishStoppedTurn(running, descriptor, event) {
|
|
3508
|
+
// The session is already on its way out with a status that MEANS something —
|
|
3509
|
+
// a spent budget, a Stop, a teardown — and a cheerful `turn_end{ok:true}`
|
|
3510
|
+
// after it replaces a true ending with a wrong one. The same guard
|
|
3511
|
+
// `completeTurn` has had since the budget pause was written.
|
|
3512
|
+
if (running.stopRequested || running.budgetSpent)
|
|
3513
|
+
return;
|
|
3514
|
+
// #366: the ring at rest must be exact — the value the gate held back during
|
|
3515
|
+
// the turn goes out before the ending, as in `completeTurn`.
|
|
3516
|
+
this.flushHeldContextUsage(running);
|
|
3517
|
+
// A phantom held from an earlier turn must not fire after this one closed.
|
|
3518
|
+
this.clearEmptyTurn(running);
|
|
3519
|
+
this.sendEvent(running, 'turn_end', {
|
|
3520
|
+
ok: true,
|
|
3521
|
+
// A turn that finished on its own in the same instant is not «stopped».
|
|
3522
|
+
...(event.aborted === true || !event.ok ? { aborted: true } : {}),
|
|
3523
|
+
...(event.limitBlocked ? { limitBlocked: true } : {}),
|
|
3524
|
+
produced: event.produced === true,
|
|
3525
|
+
});
|
|
3526
|
+
running.levels.contextPassNext = true;
|
|
3527
|
+
// The last instant this conversation is both complete and readable — the
|
|
3528
|
+
// process is about to go, and `conversationAnchor()` needs a live one.
|
|
3529
|
+
this.currentAnchor(running);
|
|
3530
|
+
const next = descriptor.kind === 'CHAT' ? 'WAITING_INPUT' : 'REVIEW';
|
|
3531
|
+
this.reportStatus(descriptor.id, next, {
|
|
3074
3532
|
costUsd: running.costUsd,
|
|
3075
3533
|
activeMs: Supervisor.spentMs(running),
|
|
3076
3534
|
});
|
|
@@ -3093,6 +3551,10 @@ export class Supervisor {
|
|
|
3093
3551
|
const until = pausedUntil ? Date.parse(pausedUntil) : Number.NaN;
|
|
3094
3552
|
if (Number.isFinite(until) && until > Date.now()) {
|
|
3095
3553
|
const first = !Supervisor.isPaused(running);
|
|
3554
|
+
// #373: a NEW clock is a new stop. A pause that merely MOVED is the same
|
|
3555
|
+
// one — it must not start a second cycle on a process already parking.
|
|
3556
|
+
if (first)
|
|
3557
|
+
running.pauseEpoch += 1;
|
|
3096
3558
|
running.pausedUntil = until;
|
|
3097
3559
|
// Interrupting on every pause frame, not only the first: the frame is
|
|
3098
3560
|
// also how a MOVED pause arrives, and a turn that slipped through in
|
|
@@ -3113,6 +3575,13 @@ export class Supervisor {
|
|
|
3113
3575
|
// user», over a turn the user never touched. Exactly the lie #196 removed.
|
|
3114
3576
|
delete running.interruptWhenReady;
|
|
3115
3577
|
delete running.pauseInterruptAt;
|
|
3578
|
+
// #373: the clock coming off does NOT drop the stop it started. The
|
|
3579
|
+
// interrupt may still be in flight, and its second result is the incident —
|
|
3580
|
+
// «снять паузу во время остановки» is a row of the acceptance matrix, and a
|
|
3581
|
+
// cycle deleted here would let that result file the session FAILED. What
|
|
3582
|
+
// ends the cycle's ownership is a NEW turn on this process
|
|
3583
|
+
// (`deliverMessage`) or a new process (`launchAgent`); both are the moment
|
|
3584
|
+
// its endings stop being its own.
|
|
3116
3585
|
// Whatever was held while the clock ran goes now, in the order it arrived.
|
|
3117
3586
|
this.flushPendingMessages(running);
|
|
3118
3587
|
}
|
|
@@ -3128,9 +3597,29 @@ export class Supervisor {
|
|
|
3128
3597
|
* Deliberately quiet: the notice was already written when the pause landed,
|
|
3129
3598
|
* and one per interrupted background turn would bury the feed.
|
|
3130
3599
|
*/
|
|
3131
|
-
stopWorkUnderPause(running) {
|
|
3600
|
+
stopWorkUnderPause(running, event) {
|
|
3132
3601
|
if (!Supervisor.isPaused(running) || !running.session)
|
|
3133
3602
|
return;
|
|
3603
|
+
// #373. The stop already in flight owns everything the process is still
|
|
3604
|
+
// saying — including the result of the very question this runner withdrew,
|
|
3605
|
+
// which is what fired the incident's SECOND interrupt: `AGENT_OUTPUT_EVENTS`
|
|
3606
|
+
// could not tell a tool RESULT from a tool starting, so the tail of the stop
|
|
3607
|
+
// read as a turn beginning.
|
|
3608
|
+
const cycle = running.stopCycle;
|
|
3609
|
+
if (cycle && ownsProcess(cycle, running.session) && !cycle.settled) {
|
|
3610
|
+
log.info('supervisor: output under a pause belongs to the stop in flight', {
|
|
3611
|
+
sessionId: running.descriptor.id,
|
|
3612
|
+
phase: cycle.phase,
|
|
3613
|
+
type: event.type,
|
|
3614
|
+
stoppedToolTail: isStoppedToolTail(cycle, event),
|
|
3615
|
+
});
|
|
3616
|
+
return;
|
|
3617
|
+
}
|
|
3618
|
+
// A tool RESULT is the tail of something. Whose tail decides whether it is
|
|
3619
|
+
// worth another stop: a call that was already running when we stopped is the
|
|
3620
|
+
// first stop finishing, one we never saw start is work that began after it.
|
|
3621
|
+
if (!isNewWork(event, cycle))
|
|
3622
|
+
return;
|
|
3134
3623
|
// One interrupt per burst, not per event. An abort is not instant: a turn
|
|
3135
3624
|
// being stopped still emits whatever was already in flight, and without
|
|
3136
3625
|
// this every one of those lines would fire another `interrupt()` — a
|
|
@@ -3138,8 +3627,10 @@ export class Supervisor {
|
|
|
3138
3627
|
// stopping. Two seconds is far shorter than the gap before a genuinely NEW
|
|
3139
3628
|
// turn (a background subagent finishing), which still gets its own.
|
|
3140
3629
|
const now = Date.now();
|
|
3141
|
-
if (running.pauseInterruptAt !== undefined &&
|
|
3630
|
+
if (running.pauseInterruptAt !== undefined &&
|
|
3631
|
+
now - running.pauseInterruptAt < Supervisor.PAUSE_BURST_MS) {
|
|
3142
3632
|
return;
|
|
3633
|
+
}
|
|
3143
3634
|
running.pauseInterruptAt = now;
|
|
3144
3635
|
// Through the ordinary path rather than straight at the adapter (QA-149
|
|
3145
3636
|
// MAJOR-2): it also withdraws the cards this turn had opened. A turn killed
|
|
@@ -3557,6 +4048,10 @@ export class Supervisor {
|
|
|
3557
4048
|
extraBudgetMinutes: descriptor.extraBudgetMinutes,
|
|
3558
4049
|
epoch: descriptor.epoch,
|
|
3559
4050
|
openQuestions: new Set(),
|
|
4051
|
+
openPermissions: new Set(),
|
|
4052
|
+
liveToolUses: new Set(),
|
|
4053
|
+
pauseEpoch: 0,
|
|
4054
|
+
processSeq: 0,
|
|
3560
4055
|
answeredAsks: new Set(),
|
|
3561
4056
|
deliveredMessageIds: new Set(),
|
|
3562
4057
|
backgroundTasks: 0,
|