@bridge4dev/runner 0.47.0 → 0.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,9 @@
1
1
  import fs from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { log } from './log.js';
4
+ import { CONTEXT_USAGE_MIN_DELTA_RATIO, CONTEXT_USAGE_MIN_DELTA_TOKENS, RATE_LIMITS_RESEND_INTERVAL_MS, } from './levels.js';
4
5
  import { claimAutoResume, clearAutoResume, pruneAutoResume } from './auto-resume.js';
6
+ import { errorVerdict, isNewWork, isStoppedToolTail, ownsProcess, turnEndVerdict, } from './stop-cycle.js';
5
7
  import { classifyFailure, isRepeatOfSameFailure, MAX_RETRIES_PER_SESSION, retryDelayMs, } from './adapters/error-policy.js';
6
8
  import { evaluateRecipeCommand, maskSecrets, maskString } from './policy.js';
7
9
  import { agentPromptSizeLabel, inspectAgentPrompt, quotePath, readAgentPrompt, } from './agent-prompt.js';
@@ -58,6 +60,16 @@ function gitPolicyOf(descriptor) {
58
60
  : {}),
59
61
  };
60
62
  }
63
+ function freshLevels() {
64
+ return {
65
+ contextSent: null,
66
+ contextHeld: null,
67
+ contextPassNext: false,
68
+ limitsSent: null,
69
+ sent: 0,
70
+ dropped: 0,
71
+ };
72
+ }
61
73
  const LAUNCH_REFUSED = { ok: false, reason: 'refused' };
62
74
  export class Supervisor {
63
75
  ws;
@@ -85,6 +97,9 @@ export class Supervisor {
85
97
  static EMPTY_TURN_SETTLE_MS = 25_000;
86
98
  /** The window actually used — the constant, or a test's own shorter one. */
87
99
  emptyTurnSettleMs;
100
+ rateLimitsResendMs;
101
+ /** The stop-settle window actually used — the constant, or a test's own. */
102
+ stopSettleMs;
88
103
  /** A finished session's journal is kept this long for a late reconnect. */
89
104
  static JOURNAL_TTL_MS = 72 * 3_600_000;
90
105
  /** Backstop: events the API will never accept must not pile up forever. */
@@ -135,6 +150,8 @@ export class Supervisor {
135
150
  this.opts = opts;
136
151
  this.journals = opts.journals ?? new JournalStore();
137
152
  this.emptyTurnSettleMs = opts.emptyTurnSettleMs ?? Supervisor.EMPTY_TURN_SETTLE_MS;
153
+ this.rateLimitsResendMs = opts.rateLimitsResendMs ?? RATE_LIMITS_RESEND_INTERVAL_MS;
154
+ this.stopSettleMs = opts.stopSettleMs ?? Supervisor.STOP_SETTLE_MS;
138
155
  this.verify = new VerifyRunner({
139
156
  enabled: opts.verifyEnabled !== false,
140
157
  onReport: (report) => {
@@ -743,6 +760,7 @@ export class Supervisor {
743
760
  lastReported: descriptor.status,
744
761
  costUsd: descriptor.costUsd,
745
762
  costBaseUsd: descriptor.costUsd,
763
+ levels: freshLevels(),
746
764
  stopRequested: false,
747
765
  parkRequested: false,
748
766
  pendingMessages: [],
@@ -750,6 +768,10 @@ export class Supervisor {
750
768
  extraBudgetMinutes: descriptor.extraBudgetMinutes,
751
769
  epoch: descriptor.epoch,
752
770
  openQuestions: new Set(),
771
+ openPermissions: new Set(),
772
+ liveToolUses: new Set(),
773
+ pauseEpoch: 0,
774
+ processSeq: 0,
753
775
  answeredAsks: new Set(),
754
776
  deliveredMessageIds: new Set(),
755
777
  backgroundTasks: 0,
@@ -841,9 +863,18 @@ export class Supervisor {
841
863
  // where the agent left off» there is the one thing the feed must not do.
842
864
  // It happens for real: a process that dies before it reports its session
843
865
  // id (the SIGABRT this ticket came from) leaves the row with none.
866
+ //
867
+ // Ticket #370: the sentence about the agent's MEMORY waits for proof.
868
+ // It used to be written here, on the strength of a stored id and before
869
+ // the CLI had been asked anything — and on Athanor it appeared sixty
870
+ // seconds before `thread/resume` timed out and took the session with it.
871
+ // What is honest at this moment is that the runner is back; whether the
872
+ // conversation reopens is answered by the process, in `provider_session`.
873
+ if (descriptor.providerSessionId)
874
+ running.resumeClaimPending = true;
844
875
  this.sendEvent(running, 'system_note', {
845
876
  text: descriptor.providerSessionId
846
- ? 'Session resumed — the agent still has this conversation. Send a message to continue where it left off.'
877
+ ? 'Session resumed — send a message and the agent picks its conversation back up.'
847
878
  : 'Session resumed, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
848
879
  });
849
880
  }
@@ -953,6 +984,14 @@ export class Supervisor {
953
984
  // apply it never started (ticket #225). The feed has already told them the
954
985
  // conversation was cut.
955
986
  const rewind = running.rewindAnchor;
987
+ // #373: a new process is a new subject. Everything the previous one left
988
+ // behind — the stop it was going through, the tools it had in flight, the
989
+ // cards it was parked on — belongs to a process that no longer exists, and
990
+ // carrying any of it over would let an old cancellation swallow a new
991
+ // failure.
992
+ delete running.stopCycle;
993
+ running.liveToolUses.clear();
994
+ running.openPermissions.clear();
956
995
  try {
957
996
  const agentPrompt = this.resolveAgentPrompt(running, resuming && !rewinding);
958
997
  const workspaceContext = composeWorkspaceContext(descriptor, agentPrompt?.text);
@@ -963,6 +1002,15 @@ export class Supervisor {
963
1002
  // from. Its transcript is still on disk — that is what makes the point
964
1003
  // usable at all.
965
1004
  const resumeTarget = rewind ? rewind.agentSession : resumeId;
1005
+ // #370: whether the promise about the agent's memory has anything to
1006
+ // prove. A launch with no conversation to reopen can never confirm one.
1007
+ running.launchResumed = Boolean(resumeTarget);
1008
+ // Nothing to prove on a launch that opens no conversation (a stale-resume
1009
+ // recovery nulls the id) — and a promise left standing would be kept
1010
+ // later, by a relaunch that had nothing to do with it.
1011
+ if (!running.launchResumed)
1012
+ delete running.resumeClaimPending;
1013
+ running.processSeq += 1;
966
1014
  running.session = adapter.startSession({
967
1015
  sessionId: descriptor.id,
968
1016
  cwd: running.worktreePath,
@@ -1354,6 +1402,16 @@ export class Supervisor {
1354
1402
  // Guarded like the event loop above — a journal that cannot be written (a
1355
1403
  // full disk, a daemon already shutting down) must not take the exit path
1356
1404
  // with it, because everything below it is cleanup.
1405
+ // #373: the stop this process was going through has reached its end. Marked
1406
+ // before the cleanup below so anything still queued behind it is judged
1407
+ // against a cycle that knows the process is gone.
1408
+ if (running.stopCycle && running.stopCycle.session === session) {
1409
+ this.clearStopSettleTimer(running.stopCycle);
1410
+ running.stopCycle.phase = 'parked';
1411
+ }
1412
+ // Both sets describe THIS process and nothing else (#373).
1413
+ running.liveToolUses.clear();
1414
+ running.openPermissions.clear();
1357
1415
  try {
1358
1416
  this.clearApiRetry(running);
1359
1417
  this.withdrawOpenQuestions(running, 'session_stopped');
@@ -1697,11 +1755,111 @@ export class Supervisor {
1697
1755
  * its timer runs out, and it must end in exactly the way it would have
1698
1756
  * ended immediately. A copy would be two behaviours one edit apart.
1699
1757
  */
1758
+ /**
1759
+ * The level gate for the context meter (#366).
1760
+ *
1761
+ * While a turn is open, a frame goes out only when the meter moved by at
1762
+ * least one of the shared thresholds or the window itself changed; the
1763
+ * newest held-back value is flushed right before `turn_end`, so the ring at
1764
+ * rest is exact. Outside a turn every frame passes – there are only a handful
1765
+ * (process start, a model change, and the Claude adapter's own measurement,
1766
+ * which resolves AFTER `turn_end`), and each one is news.
1767
+ */
1768
+ forwardContextUsage(running, usedTokens, maxTokens) {
1769
+ const levels = running.levels;
1770
+ const frame = { usedTokens, maxTokens };
1771
+ const last = levels.contextSent;
1772
+ const turnOpen = running.lastReported === 'RUNNING';
1773
+ const moved = last === null ? Number.POSITIVE_INFINITY : Math.abs(usedTokens - last.usedTokens);
1774
+ const pass = last === null ||
1775
+ last.maxTokens !== maxTokens ||
1776
+ levels.contextPassNext ||
1777
+ !turnOpen ||
1778
+ moved >= CONTEXT_USAGE_MIN_DELTA_TOKENS ||
1779
+ moved / maxTokens >= CONTEXT_USAGE_MIN_DELTA_RATIO;
1780
+ // Only the newest value is worth keeping; whatever it replaces is gone.
1781
+ if (levels.contextHeld !== null)
1782
+ levels.dropped += 1;
1783
+ if (!pass) {
1784
+ levels.contextHeld = frame;
1785
+ return;
1786
+ }
1787
+ levels.contextHeld = null;
1788
+ levels.contextPassNext = false;
1789
+ levels.contextSent = frame;
1790
+ levels.sent += 1;
1791
+ this.sendEvent(running, 'context_usage', frame);
1792
+ }
1793
+ /** Send the value the gate was holding, if any – strictly before `turn_end`. */
1794
+ flushHeldContextUsage(running) {
1795
+ const levels = running.levels;
1796
+ const held = levels.contextHeld;
1797
+ if (held === null)
1798
+ return;
1799
+ levels.contextHeld = null;
1800
+ // A held value identical to the last one sent is not news – flushing it
1801
+ // would add a frame to every turn and give back half the saving.
1802
+ const sent = levels.contextSent;
1803
+ if (sent !== null && sent.usedTokens === held.usedTokens && sent.maxTokens === held.maxTokens) {
1804
+ return;
1805
+ }
1806
+ levels.contextSent = held;
1807
+ levels.sent += 1;
1808
+ this.sendEvent(running, 'context_usage', held);
1809
+ }
1810
+ /**
1811
+ * The level gate for plan usage (#366).
1812
+ *
1813
+ * A snapshot goes out when it is the first, when anything but `measuredAt`
1814
+ * changed, when it carries a refusal (#258 – always, even twice in a row), or
1815
+ * when the one last sent is older than the resend floor. The fingerprint is
1816
+ * the whole payload minus the clock, so a field an adapter adds tomorrow is
1817
+ * part of it without anybody remembering to list it here.
1818
+ */
1819
+ forwardRateLimits(running, limits) {
1820
+ const levels = running.levels;
1821
+ const print = JSON.stringify({ ...limits, measuredAt: undefined });
1822
+ const now = Date.now();
1823
+ const last = levels.limitsSent;
1824
+ const pass = last === null ||
1825
+ Boolean(limits.blocked) ||
1826
+ last.print !== print ||
1827
+ now - last.at >= this.rateLimitsResendMs;
1828
+ if (!pass) {
1829
+ levels.dropped += 1;
1830
+ return;
1831
+ }
1832
+ levels.limitsSent = { print, at: now };
1833
+ levels.sent += 1;
1834
+ this.sendEvent(running, 'rate_limits', { ...limits });
1835
+ }
1700
1836
  completeTurn(running, descriptor, event) {
1837
+ // #366: the ring at rest must be exact – the value the gate held back
1838
+ // during the turn goes out first, so it is older than `turn_end` by seq.
1839
+ this.flushHeldContextUsage(running);
1701
1840
  this.sendEvent(running, 'turn_end', {
1702
1841
  ok: event.ok,
1703
1842
  errorMessage: event.errorMessage,
1704
1843
  ...(event.aborted ? { aborted: true } : {}),
1844
+ // #258 / #373: two facts the adapters have always measured and this line
1845
+ // has always dropped on the floor. `limitBlocked` is what tells the API a
1846
+ // turn was REFUSED rather than run, and `produced` is what tells it the
1847
+ // turn did something — without which the phantom ending a resumed process
1848
+ // emits (#300) resets the «three limit pauses in a row» counter one second
1849
+ // after the third one, and «woke → refused → slept» never stops.
1850
+ ...(event.limitBlocked ? { limitBlocked: true } : {}),
1851
+ // ALWAYS present, true or false — the absence of this field is what the
1852
+ // API reads as «this runner is older than the fix», and a new runner that
1853
+ // sometimes omitted it would be indistinguishable from one.
1854
+ produced: event.produced === true,
1855
+ });
1856
+ // …and whatever the adapter measures once the turn is over (Claude does,
1857
+ // asynchronously) is news, not a step of the same turn.
1858
+ running.levels.contextPassNext = true;
1859
+ log.info('supervisor: levels', {
1860
+ sessionId: descriptor.id,
1861
+ sent: running.levels.sent,
1862
+ dropped: running.levels.dropped,
1705
1863
  });
1706
1864
  // The session is already on its way out with a status that MEANS
1707
1865
  // something — a spent budget, a Stop, a teardown. A turn ending inside
@@ -1867,6 +2025,13 @@ export class Supervisor {
1867
2025
  isParkable(running) {
1868
2026
  return ((running.lastReported === 'REVIEW' || running.lastReported === 'WAITING_INPUT') &&
1869
2027
  running.openQuestions.size === 0 &&
2028
+ // #373. `stop()` answers every pending permission «denied» in the RUNNER's
2029
+ // name, and the status is not a reliable reading of «a card is open»: a
2030
+ // pause reports a resting status the instant it interrupts, and a card
2031
+ // resolved by POLICY never moves the status back. So the seat-freeing path
2032
+ // needs the same guard the pause has — without it `ensureCapacity` walks
2033
+ // straight past invariant 8 and denies the card to make room.
2034
+ running.openPermissions.size === 0 &&
1870
2035
  Boolean(running.descriptor.providerSessionId));
1871
2036
  }
1872
2037
  /**
@@ -1960,7 +2125,7 @@ export class Supervisor {
1960
2125
  // and Codex's own «interrupted, so start the next queued turn» both come
1961
2126
  // through no frame this runner could refuse.
1962
2127
  if (Supervisor.isPaused(running))
1963
- this.stopWorkUnderPause(running);
2128
+ this.stopWorkUnderPause(running, event);
1964
2129
  else
1965
2130
  this.noteAgentIsWorking(running);
1966
2131
  }
@@ -1970,6 +2135,16 @@ export class Supervisor {
1970
2135
  this.reportStatus(descriptor.id, statusForReport(running), {
1971
2136
  providerSessionId: event.providerSessionId,
1972
2137
  });
2138
+ // #370: the promise, now that it can be kept. Both CLIs answer a REFUSED
2139
+ // reopen with no session of any kind — Claude emits no `system:init` at
2140
+ // all, Codex throws before it gets here — so this event is the proof the
2141
+ // conversation really came back, and the only honest moment to say so.
2142
+ if (running.resumeClaimPending && running.launchResumed) {
2143
+ delete running.resumeClaimPending;
2144
+ this.sendEvent(running, 'system_note', {
2145
+ text: 'The agent still has this conversation — carry on where it left off.',
2146
+ });
2147
+ }
1973
2148
  // The CLI resumed at the anchor and forked, so the cut announced when
1974
2149
  // the rewind was asked for stands. Nothing to send — it is already in
1975
2150
  // the feed; this only retires the promise to take it back.
@@ -1987,6 +2162,39 @@ export class Supervisor {
1987
2162
  });
1988
2163
  return;
1989
2164
  case 'turn_end': {
2165
+ // #373: a stop in flight owns the endings of the process it is stopping.
2166
+ // The incident was two results 1.2 seconds apart — the first the stop we
2167
+ // asked for, the second `error_during_execution` — and the second one
2168
+ // filed a paused session as FAILED, which is terminal, with a live clock
2169
+ // still on it.
2170
+ const cycle = running.stopCycle;
2171
+ if (cycle && ownsProcess(cycle, running.session)) {
2172
+ const verdict = turnEndVerdict(cycle);
2173
+ if (verdict === 'tail') {
2174
+ log.info('supervisor: a late turn ending belongs to the stop', {
2175
+ sessionId: descriptor.id,
2176
+ process: running.processSeq,
2177
+ reason: cycle.reason,
2178
+ phase: cycle.phase,
2179
+ ok: event.ok,
2180
+ });
2181
+ return;
2182
+ }
2183
+ if (verdict === 'stopped') {
2184
+ cycle.turnEndSent = true;
2185
+ this.publishStoppedTurn(running, descriptor, event);
2186
+ this.settleStopCycle(running, cycle);
2187
+ return;
2188
+ }
2189
+ // `ordinary` — a manual Stop keeps the contract it has had since
2190
+ // QA-120: this ending goes out unchanged, and the next genuine failure
2191
+ // is not hidden behind it (`errorVerdict` answers `failure` for a
2192
+ // `user` cycle whatever its phase). Settled rather than deleted, so
2193
+ // the acknowledgement still on its way finds its cycle and writes the
2194
+ // one line this stop owes the feed — the result can beat the ACK.
2195
+ this.clearStopSettleTimer(cycle);
2196
+ cycle.settled = true;
2197
+ }
1990
2198
  // #252: decided BEFORE the frame goes out. A `turn_end{ok:false}` files
1991
2199
  // the session as FAILED, and FAILED is terminal — a session we intend to
1992
2200
  // retry must never be told it has ended. So a turn that is going to be
@@ -2085,6 +2293,31 @@ export class Supervisor {
2085
2293
  });
2086
2294
  return;
2087
2295
  }
2296
+ // #373, invariant 4. Everything above this line is a named, recoverable
2297
+ // condition and keeps its own handling; what is left is «an error we do
2298
+ // not recognise», and that is the branch that files the session FAILED.
2299
+ // A process this runner is closing on purpose produces exactly that on
2300
+ // its way out, and a pause must not end in a terminal status.
2301
+ //
2302
+ // The verdict, not a string match: a process that fell over while we
2303
+ // had merely ASKED it to stop has genuinely failed and still says so
2304
+ // (`processGone` on a cycle that closed nothing).
2305
+ {
2306
+ const cycle = running.stopCycle;
2307
+ if (cycle &&
2308
+ ownsProcess(cycle, running.session) &&
2309
+ errorVerdict(cycle, event.processGone === true) === 'tail') {
2310
+ log.info('supervisor: an error from the process we are closing', {
2311
+ sessionId: descriptor.id,
2312
+ process: running.processSeq,
2313
+ reason: cycle.reason,
2314
+ phase: cycle.phase,
2315
+ processGone: event.processGone === true,
2316
+ code: event.code ?? null,
2317
+ });
2318
+ return;
2319
+ }
2320
+ }
2088
2321
  // Forward the code: the API stores the payload as-is, so the dashboard
2089
2322
  // can offer "Sign in" instead of a dead error card.
2090
2323
  this.sendEvent(running, 'error', {
@@ -2103,6 +2336,10 @@ export class Supervisor {
2103
2336
  });
2104
2337
  return;
2105
2338
  case 'permission':
2339
+ // #373, invariant 8: a card the agent is parked on. Parking the session
2340
+ // would answer it «no» in the runner's name, so the pause stops this
2341
+ // process but leaves it open.
2342
+ running.openPermissions.add(event.requestId);
2106
2343
  this.sendEvent(running, 'permission', {
2107
2344
  requestId: event.requestId,
2108
2345
  toolName: event.toolName,
@@ -2115,6 +2352,7 @@ export class Supervisor {
2115
2352
  });
2116
2353
  return;
2117
2354
  case 'permission_resolved':
2355
+ running.openPermissions.delete(event.requestId);
2118
2356
  this.sendEvent(running, 'permission_resolved', {
2119
2357
  requestId: event.requestId,
2120
2358
  allow: event.allow,
@@ -2221,17 +2459,17 @@ export class Supervisor {
2221
2459
  });
2222
2460
  return;
2223
2461
  case 'context_usage':
2224
- this.sendEvent(running, 'context_usage', {
2225
- usedTokens: event.usedTokens,
2226
- maxTokens: event.maxTokens,
2227
- });
2462
+ // #366. A LEVEL: only a value that moved is worth a frame. The gate
2463
+ // decides before `sendEvent`, so a dropped frame never spends a seq.
2464
+ this.forwardContextUsage(running, event.usedTokens, event.maxTokens);
2228
2465
  return;
2229
2466
  // #279. A LEVEL signal like `agent_tasks`: every frame carries the whole
2230
- // snapshot, so the API stores what arrived rather than merging, and a
2467
+ // snapshot, so the API keeps what arrived rather than merging, and a
2231
2468
  // dropped frame costs freshness, never correctness. Account-wide, not
2232
2469
  // session-wide — the API files it under the SERVER, not this session.
2470
+ // #366: gated the same way as the context meter, see `forwardRateLimits`.
2233
2471
  case 'rate_limits':
2234
- this.sendEvent(running, 'rate_limits', { ...event.limits });
2472
+ this.forwardRateLimits(running, event.limits);
2235
2473
  return;
2236
2474
  case 'agent_tasks': {
2237
2475
  // Ticket #113. A LEVEL signal: every frame carries the whole live set,
@@ -2272,6 +2510,15 @@ export class Supervisor {
2272
2510
  return;
2273
2511
  }
2274
2512
  case 'tool':
2513
+ // #373: which calls are in flight, so a stop can tell its own tail from
2514
+ // work starting up. Kept here rather than in the adapters — both of them
2515
+ // already report the two phases, and one reader is one place to be wrong.
2516
+ if (event.toolUseId) {
2517
+ if (event.phase === 'use')
2518
+ running.liveToolUses.add(event.toolUseId);
2519
+ else
2520
+ running.liveToolUses.delete(event.toolUseId);
2521
+ }
2275
2522
  this.sendEvent(running, 'tool', {
2276
2523
  phase: event.phase,
2277
2524
  name: event.name,
@@ -2726,6 +2973,14 @@ export class Supervisor {
2726
2973
  * Now it goes back in the queue, exactly like a parked process below, and
2727
2974
  * the feed says why.
2728
2975
  */
2976
+ // #373: a new turn on this process ends the previous stop's ownership of
2977
+ // it. Dropped HERE and not when the clock came off, because this is the
2978
+ // moment the endings stop being the stop's own — a release alone leaves
2979
+ // the interrupt in flight and its second result still coming.
2980
+ if (running.stopCycle && running.stopCycle.session === running.session) {
2981
+ this.clearStopSettleTimer(running.stopCycle);
2982
+ delete running.stopCycle;
2983
+ }
2729
2984
  if (!running.session.send(text)) {
2730
2985
  log.warn('supervisor: the agent process refused the message — requeued', {
2731
2986
  sessionId: running.descriptor.id,
@@ -2951,23 +3206,329 @@ export class Supervisor {
2951
3206
  running.interruptWhenReady = true;
2952
3207
  return;
2953
3208
  }
3209
+ const open = running.stopCycle;
3210
+ if (open && ownsProcess(open, running.session) && !open.settled) {
3211
+ // #357 part 2, and invariant 2 of the pause plan. Ten frames arrive while
3212
+ // the CLI is waiting out a provider retry — from a phone and a laptop at
3213
+ // once, or from a person pressing a button that gave them no sign it had
3214
+ // been heard. One stop, one line in the feed, one status report; the rest
3215
+ // wait for the same answer.
3216
+ //
3217
+ // A clock landing on a Stop the person pressed a second earlier is the
3218
+ // same stop with a stricter reason: the CLI has one outstanding interrupt
3219
+ // and answers it once, but the ending now belongs to a PAUSE, which closes
3220
+ // the process. Left as `user` this cycle would step aside at the first
3221
+ // ending and the second result would file the session FAILED — the
3222
+ // incident, through a door the fix did not cover.
3223
+ if (reason === 'pause' && open.reason === 'user') {
3224
+ open.reason = 'pause';
3225
+ log.info('supervisor: a clock landed on a stop already in flight', { sessionId });
3226
+ }
3227
+ log.info('supervisor: a stop is already in flight — joining it', {
3228
+ sessionId,
3229
+ reason,
3230
+ phase: open.phase,
3231
+ });
3232
+ await open.inFlight?.catch(() => undefined);
3233
+ return;
3234
+ }
3235
+ await this.beginStopCycle(running, reason, announce);
3236
+ }
3237
+ /**
3238
+ * How long a stop waits for the ending that should follow it (#373).
3239
+ *
3240
+ * The transition out of `interrupting` is the first result, not the
3241
+ * acknowledgement — an ACK only says the request was heard, and the incident
3242
+ * shows an ACK and a result are different events that arrive in either order.
3243
+ * This is the backstop for the case where no result comes at all: an idle
3244
+ * session had no turn to end, and a CLI can always simply not answer.
3245
+ */
3246
+ static STOP_SETTLE_MS = 5_000;
3247
+ /**
3248
+ * How long after one stop the next output is still read as its tail (#196).
3249
+ *
3250
+ * The floor under the stop cycle rather than a substitute for it: while a
3251
+ * cycle is open every event joins it, and this is what keeps a burst from
3252
+ * opening a NEW cycle per line in the window after one closed without closing
3253
+ * its process. Far shorter than the gap before a genuinely new turn — a
3254
+ * background subagent finishing — which still gets its own stop.
3255
+ */
3256
+ static PAUSE_BURST_MS = 2_000;
3257
+ /**
3258
+ * Stop one agent process once, and own what stopping it produces (#373).
3259
+ *
3260
+ * The shape is deliberate: the interrupt is a round trip that can take half a
3261
+ * minute (a CLI sitting out a provider retry answers only when it comes back
3262
+ * — 28.5 seconds, measured in #357), so nothing here may be on the path the
3263
+ * event pump takes. The cycle is stored before the request goes out, which is
3264
+ * what makes every later request join this one instead of starting a second.
3265
+ */
3266
+ async beginStopCycle(running, reason, announce) {
3267
+ const session = running.session;
3268
+ if (!session)
3269
+ return;
3270
+ const sessionId = running.descriptor.id;
3271
+ const cycle = {
3272
+ session,
3273
+ reason,
3274
+ epoch: running.pauseEpoch,
3275
+ phase: 'interrupting',
3276
+ inFlight: null,
3277
+ outcome: null,
3278
+ refused: false,
3279
+ closed: false,
3280
+ turnEndSent: false,
3281
+ settled: false,
3282
+ hadTurn: running.lastReported === 'RUNNING' || running.lastReported === 'STARTING',
3283
+ toolsAtStop: new Set(running.liveToolUses),
3284
+ };
3285
+ running.stopCycle = cycle;
3286
+ // Kept for the cool-down in `stopWorkUnderPause`: a cycle that ends without
3287
+ // closing the process must not be replaced by another one on the next event.
3288
+ running.pauseInterruptAt = Date.now();
3289
+ log.info('supervisor: stopping the agent', {
3290
+ sessionId,
3291
+ reason,
3292
+ process: running.processSeq,
3293
+ epoch: cycle.epoch,
3294
+ status: running.lastReported,
3295
+ hadTurn: cycle.hadTurn,
3296
+ toolsInFlight: cycle.toolsAtStop.size,
3297
+ openQuestions: running.openQuestions.size,
3298
+ openPermissions: running.openPermissions.size,
3299
+ });
2954
3300
  // The turn being interrupted is the turn the question belongs to — leaving
2955
3301
  // the ask parked would make the user's next message be swallowed as its
2956
3302
  // answer (QA-106 m7). The adapter reports each withdrawal itself; the local
2957
3303
  // set is cleared so the session can be parked again.
2958
- running.session.cancelQuestions('turn_aborted');
3304
+ session.cancelQuestions('turn_aborted');
2959
3305
  running.openQuestions.clear();
2960
- await running.session.interrupt();
2961
- if (announce) {
2962
- this.sendEvent(running, 'notice', {
2963
- level: 'info',
2964
- text: reason === 'pause'
2965
- ? 'Paused — the turn was stopped. Nothing else will start until the clock runs out.'
2966
- : 'Turn interrupted by the user',
3306
+ // Armed BEFORE the request goes out, not after it answers. A control channel
3307
+ // that never answers is a real case — it is half of #357 — and a stop that
3308
+ // waited for an acknowledgement before starting its own clock would hold the
3309
+ // session, its seat and the pause for as long as the CLI stayed silent.
3310
+ this.armStopSettle(running, cycle);
3311
+ const inFlight = (async () => {
3312
+ let outcome;
3313
+ try {
3314
+ outcome = await session.interrupt();
3315
+ }
3316
+ catch (error) {
3317
+ // An adapter is not supposed to throw here, and one that does has told
3318
+ // us nothing about whether the turn stopped — which is exactly the
3319
+ // «refused» case.
3320
+ outcome = 'refused';
3321
+ log.warn('supervisor: the interrupt request threw', {
3322
+ sessionId,
3323
+ error: String(error),
3324
+ });
3325
+ }
3326
+ // A newer cycle (or a new process) took over while we waited: this answer
3327
+ // is about a stop nobody is waiting for any more.
3328
+ if (running.stopCycle !== cycle)
3329
+ return;
3330
+ cycle.outcome = outcome;
3331
+ // Invariant 7: a refusal that arrives after WE closed this process is
3332
+ // void. `stop()` rejects every outstanding control request on close, and
3333
+ // reading our own park as a failure is the defect with the sign flipped.
3334
+ cycle.refused = outcome === 'refused' && cycle.phase === 'interrupting';
3335
+ if (announce) {
3336
+ this.sendEvent(running, 'notice', {
3337
+ level: 'info',
3338
+ text: reason === 'pause'
3339
+ ? 'Paused — the turn was stopped. Nothing else will start until the clock runs out.'
3340
+ : 'Turn interrupted by the user',
3341
+ });
3342
+ }
3343
+ const next = running.descriptor.kind === 'CHAT' ? 'WAITING_INPUT' : 'REVIEW';
3344
+ this.reportStatus(sessionId, next, {
3345
+ costUsd: running.costUsd,
3346
+ activeMs: Supervisor.spentMs(running),
3347
+ });
3348
+ this.armStopSettle(running, cycle);
3349
+ })();
3350
+ cycle.inFlight = inFlight;
3351
+ await inFlight;
3352
+ }
3353
+ /**
3354
+ * Wait a bounded time for the ending an accepted interrupt should produce.
3355
+ *
3356
+ * «Bounded» rather than «for ever» because two real cases produce no ending
3357
+ * at all: a pause landing between turns (there was nothing to stop), and a
3358
+ * CLI that takes the request and then says nothing. Leaving the cycle open
3359
+ * there would hold the seat and the clock indefinitely.
3360
+ */
3361
+ armStopSettle(running, cycle) {
3362
+ // The result can beat the acknowledgement — they are different events and
3363
+ // the incident shows both orders. If it already has, this cycle is settled
3364
+ // and there is nothing left to wait for.
3365
+ if (cycle.phase !== 'interrupting' || running.stopCycle !== cycle)
3366
+ return;
3367
+ if (this.isStale(running))
3368
+ return;
3369
+ this.clearStopSettleTimer(cycle);
3370
+ const timer = setTimeout(() => {
3371
+ delete cycle.settleTimer;
3372
+ if (running.stopCycle !== cycle || this.isStale(running))
3373
+ return;
3374
+ const midTurn = running.lastReported === 'RUNNING' || running.lastReported === 'STARTING';
3375
+ if (cycle.reason === 'user' && (cycle.outcome === null || midTurn)) {
3376
+ // A manual Stop never forces: #357's own rule is that the turn stops
3377
+ // when the CLI comes back, and the measured wait there was 28.5 seconds
3378
+ // — a provider retry the person cannot see. So this cycle stands for as
3379
+ // long as the turn does, not for five seconds: the guarantee it carries
3380
+ // is «one stop per turn», and a stop that expired underneath a running
3381
+ // turn would let the eleventh press become a second interrupt.
3382
+ this.armStopSettle(running, cycle);
3383
+ return;
3384
+ }
3385
+ if (!cycle.turnEndSent && cycle.hadTurn && cycle.reason === 'pause') {
3386
+ // A turn was running and its ending never came. One goes out anyway:
3387
+ // the feed's readers retire a turn on `turn_end`, and a turn with no
3388
+ // ending leaves the tray counting work that stopped minutes ago.
3389
+ cycle.turnEndSent = true;
3390
+ this.publishStoppedTurn(running, running.descriptor, { ok: true, aborted: true });
3391
+ }
3392
+ log.info('supervisor: the stop produced no ending in time', {
3393
+ sessionId: running.descriptor.id,
3394
+ process: running.processSeq,
3395
+ reason: cycle.reason,
3396
+ outcome: cycle.outcome,
3397
+ hadTurn: cycle.hadTurn,
3398
+ });
3399
+ this.settleStopCycle(running, cycle);
3400
+ }, this.stopSettleMs);
3401
+ timer.unref?.();
3402
+ cycle.settleTimer = timer;
3403
+ }
3404
+ clearStopSettleTimer(cycle) {
3405
+ if (cycle.settleTimer) {
3406
+ clearTimeout(cycle.settleTimer);
3407
+ delete cycle.settleTimer;
3408
+ }
3409
+ }
3410
+ /**
3411
+ * The turn is over — close the process, keeping the session resumable.
3412
+ *
3413
+ * This is the change the whole plan is about. A pause used to leave a live
3414
+ * CLI standing for up to five hours, which is where the second result came
3415
+ * from; now the runner takes the ending it asked for and parks, and the next
3416
+ * message (or the clock's own «continue») starts a process that resumes the
3417
+ * same conversation.
3418
+ */
3419
+ settleStopCycle(running, cycle) {
3420
+ if (running.stopCycle !== cycle)
3421
+ return;
3422
+ this.clearStopSettleTimer(cycle);
3423
+ cycle.settled = true;
3424
+ // Settled, never DELETED here. The cycle goes on owning this process's
3425
+ // endings — a late second result after a stop that closed nothing is the
3426
+ // incident itself — and is dropped only when a new turn starts on this
3427
+ // process (`deliverMessage`) or a new process starts (`launchAgent`).
3428
+ const refusal = cycle.reason === 'user' ? 'manual-stop' : this.parkRefusalUnderPause(running);
3429
+ if (refusal) {
3430
+ // A manual Stop closes nothing by design: the person stopped a turn, not
3431
+ // the session. A pause may be refused the close for its own reasons — an
3432
+ // open permission card, a conversation the CLI has not named yet.
3433
+ log.info('supervisor: the stopped process stays open', {
3434
+ sessionId: running.descriptor.id,
3435
+ reason: cycle.reason,
3436
+ why: refusal,
2967
3437
  });
3438
+ this.sayIfTheStopWasRefused(running, cycle, false);
3439
+ return;
2968
3440
  }
2969
- const next = running.descriptor.kind === 'CHAT' ? 'WAITING_INPUT' : 'REVIEW';
2970
- this.reportStatus(sessionId, next, {
3441
+ cycle.phase = 'parking';
3442
+ cycle.closed = true;
3443
+ this.sayIfTheStopWasRefused(running, cycle, true);
3444
+ this.park(running, { quiet: true });
3445
+ }
3446
+ /**
3447
+ * The agent never said yes — say so, and say only what actually happened.
3448
+ *
3449
+ * Invariant 5: a stop the CLI refused must not read as a clean cancellation.
3450
+ * Which sentence is true depends on what came next, so it is written here and
3451
+ * not where the refusal was noticed — a line promising «its process was
3452
+ * closed» over a manual Stop, which closes nothing, is the same class of lie
3453
+ * as the one this ticket is about (gotcha 324).
3454
+ */
3455
+ sayIfTheStopWasRefused(running, cycle, closing) {
3456
+ if (!cycle.refused)
3457
+ return;
3458
+ cycle.refused = false; // one sentence per stop
3459
+ this.sendEvent(running, 'notice', {
3460
+ level: 'warn',
3461
+ text: closing
3462
+ ? 'The agent did not confirm the stop, so its process was closed. Nothing was lost — the conversation continues when the session does.'
3463
+ : 'The agent did not confirm the stop, so the turn may still be running. Nothing has been lost.',
3464
+ });
3465
+ }
3466
+ /**
3467
+ * Why this paused process must NOT be closed, or null when it may be.
3468
+ *
3469
+ * Invariant 8 lives here. `park()` calls the adapter's `stop()`, and `stop()`
3470
+ * answers every pending permission «denied» in the RUNNER's name — a decision
3471
+ * nobody made, in a place that keeps decisions for ninety days. Worse, the API
3472
+ * sends nothing when the clock is lifted over an open card
3473
+ * (`pauseInterruptedTurn` is false while `openAsks > 0`), so the session would
3474
+ * simply stand there, silent, with the card gone.
3475
+ *
3476
+ * A conversation the CLI has not named yet is the other refusal: parking
3477
+ * promises the context is saved, and there is nothing to resume from.
3478
+ */
3479
+ parkRefusalUnderPause(running) {
3480
+ if (!running.session)
3481
+ return 'no-process';
3482
+ if (running.stopRequested || running.budgetSpent)
3483
+ return 'session-ending';
3484
+ // The clock came off while the stop was in flight. There is nothing left to
3485
+ // park FOR, and closing the process now would make the old operation stop
3486
+ // the new work — the acceptance matrix says it must not. The cycle stays,
3487
+ // settled, so the ending still on its way is still its own.
3488
+ if (!Supervisor.isPaused(running))
3489
+ return 'clock-lifted';
3490
+ if (running.openPermissions.size > 0)
3491
+ return 'open-permission';
3492
+ if (running.openQuestions.size > 0)
3493
+ return 'open-question';
3494
+ if (!running.descriptor.providerSessionId)
3495
+ return 'no-conversation';
3496
+ return null;
3497
+ }
3498
+ /**
3499
+ * The one ending a stop cycle publishes (#373).
3500
+ *
3501
+ * `ok` unconditionally: a turn this runner stopped is not a turn that failed,
3502
+ * and `turn_end{ok:false}` is what put a paused session into the terminal
3503
+ * status it could never be woken out of. The facts the turn actually carried
3504
+ * ride along — `limitBlocked` because the API counts refusals with it, and
3505
+ * `produced` because it is what keeps the refusal counter honest (#300).
3506
+ */
3507
+ publishStoppedTurn(running, descriptor, event) {
3508
+ // The session is already on its way out with a status that MEANS something —
3509
+ // a spent budget, a Stop, a teardown — and a cheerful `turn_end{ok:true}`
3510
+ // after it replaces a true ending with a wrong one. The same guard
3511
+ // `completeTurn` has had since the budget pause was written.
3512
+ if (running.stopRequested || running.budgetSpent)
3513
+ return;
3514
+ // #366: the ring at rest must be exact — the value the gate held back during
3515
+ // the turn goes out before the ending, as in `completeTurn`.
3516
+ this.flushHeldContextUsage(running);
3517
+ // A phantom held from an earlier turn must not fire after this one closed.
3518
+ this.clearEmptyTurn(running);
3519
+ this.sendEvent(running, 'turn_end', {
3520
+ ok: true,
3521
+ // A turn that finished on its own in the same instant is not «stopped».
3522
+ ...(event.aborted === true || !event.ok ? { aborted: true } : {}),
3523
+ ...(event.limitBlocked ? { limitBlocked: true } : {}),
3524
+ produced: event.produced === true,
3525
+ });
3526
+ running.levels.contextPassNext = true;
3527
+ // The last instant this conversation is both complete and readable — the
3528
+ // process is about to go, and `conversationAnchor()` needs a live one.
3529
+ this.currentAnchor(running);
3530
+ const next = descriptor.kind === 'CHAT' ? 'WAITING_INPUT' : 'REVIEW';
3531
+ this.reportStatus(descriptor.id, next, {
2971
3532
  costUsd: running.costUsd,
2972
3533
  activeMs: Supervisor.spentMs(running),
2973
3534
  });
@@ -2990,6 +3551,10 @@ export class Supervisor {
2990
3551
  const until = pausedUntil ? Date.parse(pausedUntil) : Number.NaN;
2991
3552
  if (Number.isFinite(until) && until > Date.now()) {
2992
3553
  const first = !Supervisor.isPaused(running);
3554
+ // #373: a NEW clock is a new stop. A pause that merely MOVED is the same
3555
+ // one — it must not start a second cycle on a process already parking.
3556
+ if (first)
3557
+ running.pauseEpoch += 1;
2993
3558
  running.pausedUntil = until;
2994
3559
  // Interrupting on every pause frame, not only the first: the frame is
2995
3560
  // also how a MOVED pause arrives, and a turn that slipped through in
@@ -3010,6 +3575,13 @@ export class Supervisor {
3010
3575
  // user», over a turn the user never touched. Exactly the lie #196 removed.
3011
3576
  delete running.interruptWhenReady;
3012
3577
  delete running.pauseInterruptAt;
3578
+ // #373: the clock coming off does NOT drop the stop it started. The
3579
+ // interrupt may still be in flight, and its second result is the incident —
3580
+ // «снять паузу во время остановки» is a row of the acceptance matrix, and a
3581
+ // cycle deleted here would let that result file the session FAILED. What
3582
+ // ends the cycle's ownership is a NEW turn on this process
3583
+ // (`deliverMessage`) or a new process (`launchAgent`); both are the moment
3584
+ // its endings stop being its own.
3013
3585
  // Whatever was held while the clock ran goes now, in the order it arrived.
3014
3586
  this.flushPendingMessages(running);
3015
3587
  }
@@ -3025,9 +3597,29 @@ export class Supervisor {
3025
3597
  * Deliberately quiet: the notice was already written when the pause landed,
3026
3598
  * and one per interrupted background turn would bury the feed.
3027
3599
  */
3028
- stopWorkUnderPause(running) {
3600
+ stopWorkUnderPause(running, event) {
3029
3601
  if (!Supervisor.isPaused(running) || !running.session)
3030
3602
  return;
3603
+ // #373. The stop already in flight owns everything the process is still
3604
+ // saying — including the result of the very question this runner withdrew,
3605
+ // which is what fired the incident's SECOND interrupt: `AGENT_OUTPUT_EVENTS`
3606
+ // could not tell a tool RESULT from a tool starting, so the tail of the stop
3607
+ // read as a turn beginning.
3608
+ const cycle = running.stopCycle;
3609
+ if (cycle && ownsProcess(cycle, running.session) && !cycle.settled) {
3610
+ log.info('supervisor: output under a pause belongs to the stop in flight', {
3611
+ sessionId: running.descriptor.id,
3612
+ phase: cycle.phase,
3613
+ type: event.type,
3614
+ stoppedToolTail: isStoppedToolTail(cycle, event),
3615
+ });
3616
+ return;
3617
+ }
3618
+ // A tool RESULT is the tail of something. Whose tail decides whether it is
3619
+ // worth another stop: a call that was already running when we stopped is the
3620
+ // first stop finishing, one we never saw start is work that began after it.
3621
+ if (!isNewWork(event, cycle))
3622
+ return;
3031
3623
  // One interrupt per burst, not per event. An abort is not instant: a turn
3032
3624
  // being stopped still emits whatever was already in flight, and without
3033
3625
  // this every one of those lines would fire another `interrupt()` — a
@@ -3035,8 +3627,10 @@ export class Supervisor {
3035
3627
  // stopping. Two seconds is far shorter than the gap before a genuinely NEW
3036
3628
  // turn (a background subagent finishing), which still gets its own.
3037
3629
  const now = Date.now();
3038
- if (running.pauseInterruptAt !== undefined && now - running.pauseInterruptAt < 2_000)
3630
+ if (running.pauseInterruptAt !== undefined &&
3631
+ now - running.pauseInterruptAt < Supervisor.PAUSE_BURST_MS) {
3039
3632
  return;
3633
+ }
3040
3634
  running.pauseInterruptAt = now;
3041
3635
  // Through the ordinary path rather than straight at the adapter (QA-149
3042
3636
  // MAJOR-2): it also withdraws the cards this turn had opened. A turn killed
@@ -3444,6 +4038,7 @@ export class Supervisor {
3444
4038
  lastReported: descriptor.status === 'REVIEW' ? 'REVIEW' : 'WAITING_INPUT',
3445
4039
  costUsd: descriptor.costUsd,
3446
4040
  costBaseUsd: descriptor.costUsd,
4041
+ levels: freshLevels(),
3447
4042
  stopRequested: false,
3448
4043
  parkRequested: false,
3449
4044
  pendingMessages: [],
@@ -3453,6 +4048,10 @@ export class Supervisor {
3453
4048
  extraBudgetMinutes: descriptor.extraBudgetMinutes,
3454
4049
  epoch: descriptor.epoch,
3455
4050
  openQuestions: new Set(),
4051
+ openPermissions: new Set(),
4052
+ liveToolUses: new Set(),
4053
+ pauseEpoch: 0,
4054
+ processSeq: 0,
3456
4055
  answeredAsks: new Set(),
3457
4056
  deliveredMessageIds: new Set(),
3458
4057
  backgroundTasks: 0,