@oar-sh/oar 0.9.4 → 0.9.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/.github/extensions/web-relay/polling/polling-loop.mjs +12 -32
  2. package/.github/extensions/web-relay/skills/question-bridge.mjs +14 -39
  3. package/CHANGELOG.md +199 -0
  4. package/README.md +35 -6
  5. package/package.json +2 -1
  6. package/server/claude-worker/claude-remote-relay-tool.mjs +1 -0
  7. package/server/claude-worker/claude-session-process.mjs +411 -19
  8. package/server/claude-worker/claude-session-worker.mjs +5 -0
  9. package/server/claude-worker/claude-turn-failure.mjs +92 -0
  10. package/server/claude-worker/claude-turn-publisher.mjs +52 -3
  11. package/server/claude-worker/sdk-message-normalizer.mjs +13 -1
  12. package/server/copilot-worker/copilot-model-switch.mjs +29 -3
  13. package/server/copilot-worker/copilot-question-bridge.mjs +2 -4
  14. package/server/copilot-worker/copilot-runtime-tree.mjs +91 -0
  15. package/server/copilot-worker/copilot-sdk-adapter.mjs +48 -8
  16. package/server/copilot-worker/copilot-sdk-session-process.mjs +897 -166
  17. package/server/copilot-worker/copilot-sdk-session-worker.mjs +5 -0
  18. package/server/cursor-worker/cursor-sdk-adapter.mjs +80 -13
  19. package/server/cursor-worker/cursor-session-worker.mjs +5 -0
  20. package/server/cursor-worker/cursor-turn-runner.mjs +5 -2
  21. package/server/db-schema.mjs +7 -0
  22. package/server/grok-worker/acp-client.mjs +68 -12
  23. package/server/grok-worker/grok-sdk-adapter.mjs +4 -0
  24. package/server/grok-worker/grok-session-worker.mjs +5 -0
  25. package/server/grok-worker/grok-turn-runner.mjs +6 -2
  26. package/server/public/app/api-client.js +17 -2
  27. package/server/public/app/bootstrap.js +3 -0
  28. package/server/public/app/composer-control-state.mjs +4 -3
  29. package/server/public/app/conversation-view.js +97 -4
  30. package/server/public/app/journal-view.js +6 -0
  31. package/server/public/app/markdown-strikethrough.mjs +31 -0
  32. package/server/public/app/relay-board-inline.mjs +70 -0
  33. package/server/public/app/relay-board-view.js +157 -61
  34. package/server/public/app/relay-error-ctas.mjs +12 -3
  35. package/server/public/app/socket-handlers.js +9 -0
  36. package/server/public/app/steered-turn-merge.mjs +10 -1
  37. package/server/public/app/store.js +4 -1
  38. package/server/public/app/subagent-fold.mjs +61 -0
  39. package/server/public/app/usage-limit-ui.js +225 -0
  40. package/server/public/index.html +97 -4
  41. package/server/repositories/message-repository.mjs +10 -1
  42. package/server/repositories/question-repository.mjs +1 -0
  43. package/server/repositories/remote-relay-repository.mjs +30 -0
  44. package/server/routes/messages-routes.mjs +93 -17
  45. package/server/routes/relay-board-routes.mjs +56 -2
  46. package/server/routes/sessions-routes.mjs +40 -2
  47. package/server/server-runtime.mjs +33 -2
  48. package/server/services/preview-proxy-server.mjs +64 -36
  49. package/server/services/process-list-service.mjs +122 -0
  50. package/server/services/relay-console-log.mjs +199 -0
  51. package/server/services/remote-relay-caller-context.mjs +45 -20
  52. package/server/services/remote-relay-dispatcher.mjs +328 -21
  53. package/server/services/remote-relay-inbound.mjs +35 -0
  54. package/server/services/session-worker-launch-service.mjs +17 -2
  55. package/server/services/session-worker-process-service.mjs +85 -32
  56. package/server/services/session-worker-registry-service.mjs +1 -1
  57. package/server/services/session-worker-supervisor-service.mjs +33 -1
  58. package/server/services/usage-limit-pause-service.mjs +441 -0
  59. package/shared/ask-user-bridge.mjs +14 -39
  60. package/shared/claude-usage-limit.mjs +257 -0
  61. package/shared/control-poller.mjs +69 -20
  62. package/shared/failure-note-text.mjs +38 -0
  63. package/shared/question-wait.mjs +108 -0
  64. package/shared/remote-relay-contract.mjs +33 -2
  65. package/shared/steer-settle-failure.mjs +22 -0
  66. package/shared/steer-settle-markers.mjs +24 -0
  67. package/shared/worker-runtime/process-tree.mjs +483 -0
  68. package/shared/worker-runtime/relay-errors.mjs +31 -0
  69. package/shared/worker-runtime/send-and-wait-errors.mjs +4 -4
  70. package/shared/worker-runtime/turn-liveness.mjs +249 -0
  71. package/shared/worker-runtime/worker-log-file.mjs +97 -0
@@ -48,6 +48,7 @@ import { randomUUID } from 'crypto';
48
48
 
49
49
  import {
50
50
  USER_INPUT_UNSUPPORTED_ANSWER,
51
+ COPILOT_TURN_CONTINUE_GUIDANCE,
51
52
  classifyCopilotSessionError,
52
53
  classifyCopilotTurnException,
53
54
  copilotAgentModeForRelayMode,
@@ -76,6 +77,11 @@ import {
76
77
  normalizeRelayEffort,
77
78
  } from './copilot-model-switch.mjs';
78
79
  import { createCopilotQuestionBridge } from './copilot-question-bridge.mjs';
80
+ import {
81
+ describeCommandsBeingStopped,
82
+ runtimeStopLeavesCommands,
83
+ watchCopilotRuntimeTree,
84
+ } from './copilot-runtime-tree.mjs';
79
85
  import {
80
86
  RELAY_ASK_USER_TOOL_DESCRIPTION,
81
87
  RELAY_ASK_USER_TOOL_NAME,
@@ -85,7 +91,7 @@ import {
85
91
  parseAskUserToolArguments,
86
92
  } from './copilot-ask-user-tool.mjs';
87
93
  import { buildCopilotRemoteRelayTool } from './copilot-remote-relay-tool.mjs';
88
- import { createRemoteRelayToolGate } from '../../shared/remote-relay-tool-core.mjs';
94
+ import { createRemoteRelayToolGate, isRemoteRelayToolName } from '../../shared/remote-relay-tool-core.mjs';
89
95
  import {
90
96
  EXIT_PLAN_BOARD_POSTED_FEEDBACK,
91
97
  EXIT_PLAN_NO_BOARD_FEEDBACK,
@@ -102,10 +108,17 @@ import {
102
108
  import { EMPTY_TURN_COMPLETION_NOTE } from '../../shared/empty-turn-completion.mjs';
103
109
  import { buildModelSnapshotFields, extractModelDescriptors } from '../../shared/model-descriptors.mjs';
104
110
  import { buildRelayStopFailure } from '../../shared/relay-stop-failure.mjs';
105
- import { STEER_FOLDED_TEXT, STEER_STOPPED_TEXT } from '../../shared/steer-settle-markers.mjs';
111
+ import { STEER_FOLDED_TEXT, STEER_STOPPED_TEXT, STEER_TURN_FAILED_TEXT } from '../../shared/steer-settle-markers.mjs';
106
112
  import { buildSteerSettleFailure } from '../../shared/steer-settle-failure.mjs';
107
113
  import { shouldEmitStreamUpdate } from '../../shared/stream-emit-gating.mjs';
108
114
  import { withSteerNote } from '../../shared/steer-note.mjs';
115
+ import { isTransientRelayError } from '../../shared/worker-runtime/relay-errors.mjs';
116
+ import {
117
+ createTurnLiveness,
118
+ createTurnStalledError,
119
+ probeRuntime,
120
+ resolveTurnStallWindows,
121
+ } from '../../shared/worker-runtime/turn-liveness.mjs';
109
122
 
110
123
  // How long the runtime may sit with no session activity before the worker
111
124
  // closes it. The worker process itself stays up and reconnects lazily on the
@@ -114,11 +127,18 @@ import { withSteerNote } from '../../shared/steer-note.mjs';
114
127
  // holding a runtime subprocess per idle conversation is not.
115
128
  const DEFAULT_IDLE_SHUTDOWN_MS = 10 * 60_000;
116
129
  const DEFAULT_LIFECYCLE_POLL_MS = 5_000;
117
- // Mirrors the Cursor adapter's `stallTimeoutMs`. Emphatically NOT 0: the
118
- // worker's 10s heartbeat keeps renewing the relay's processing lease
119
- // (messages-routes.mjs), so without a stall ceiling a turn whose runtime went
120
- // quiet holds its queue row open indefinitely and no watchdog can free it.
121
- const DEFAULT_TURN_STALL_TIMEOUT_MS = 120_000;
130
+ /** How long the runtime may take to answer the stall watchdog's ping. */
131
+ const DEFAULT_STALL_PROBE_TIMEOUT_MS = 5_000;
132
+ /**
133
+ * How long the runtime may take to answer a request of the worker's own: a
134
+ * Stop, a read of its task registry or its message queue. None of them has a
135
+ * timeout in the SDK, and a healthy runtime answers within milliseconds even
136
+ * while it is busy (it answered a ping in 2 ms through a 150 s silent tool
137
+ * call, runtime 1.0.88).
138
+ */
139
+ const DEFAULT_RUNTIME_RPC_TIMEOUT_MS = 10_000;
140
+ /** Failed reads of the task registry in a row before its last answer stops counting. */
141
+ const MAX_TASK_REFRESH_FAILURES = 5;
122
142
  /**
123
143
  * How long live background shells ALONE may keep the runtime up (0 = no limit).
124
144
  *
@@ -143,10 +163,47 @@ const DEFAULT_BACKGROUND_TASK_TIMEOUT_MS = 30 * 60_000;
143
163
  * worker's `notificationGraceMs`, same value.
144
164
  */
145
165
  const DEFAULT_CONTINUATION_GRACE_MS = 60_000;
146
- /** Retry spacing for the synthetic-row registration (3 attempts). */
166
+ /**
167
+ * Retry spacing for the synthetic-row registration: 3 attempts while the relay
168
+ * answers without minting a row, and the first step of the backoff while it
169
+ * cannot be reached at all.
170
+ */
147
171
  const DEFAULT_CONTINUATION_RETRY_DELAY_MS = 500;
172
+ /**
173
+ * How long the registration keeps trying while the relay cannot be reached. A
174
+ * relay restart takes seconds; three attempts half a second apart gave up
175
+ * inside it, and the whole continuation (the report of a background agent
176
+ * that had worked for an hour) was dropped.
177
+ */
178
+ const DEFAULT_CONTINUATION_OUTAGE_WINDOW_MS = 120_000;
179
+ /** One registration request; the transport itself would wait 300 s. */
180
+ const DEFAULT_CONTINUATION_REQUEST_TIMEOUT_MS = 15_000;
148
181
  /** How long a continuation's actions may buffer before its row is abandoned. */
149
182
  const CONTINUATION_REGISTRATION_TIMEOUT_MS = 10_000;
183
+ /**
184
+ * How long a finished reply is offered to a relay that cannot be reached
185
+ * before the row is given up. The reply exists once; a requeue runs the
186
+ * prompt again, in a session that already holds it and its answer.
187
+ */
188
+ const DEFAULT_RESPONSE_RETRY_WINDOW_MS = 5 * 60_000;
189
+ /**
190
+ * One attempt to save a reply, or to requeue its row. Without a limit of its
191
+ * own an attempt lasts as long as the transport lets it (300 s), which is the
192
+ * whole window: a relay that took the connection and did not answer got one
193
+ * attempt and then a requeue. Generous, because the route may hold a reply
194
+ * for a while before it answers (a question card that was just answered).
195
+ */
196
+ const DEFAULT_RESPONSE_REQUEST_TIMEOUT_MS = 90_000;
197
+ /**
198
+ * Live output (stream, thoughts, activity lines) is posted from the dispatch
199
+ * chain, one event after the other. A relay that accepts the connection and
200
+ * then does not answer held every later event, the turn's terminator
201
+ * included, for the transport's 300 s. One post waits this long; after one
202
+ * that ran out, live output is skipped for the pause below. The reply itself
203
+ * does not go this way.
204
+ */
205
+ const DEFAULT_LIVE_POST_TIMEOUT_MS = 10_000;
206
+ const DEFAULT_LIVE_POST_PAUSE_MS = 30_000;
150
207
  /** Cap on activity lines carried between turns, so a chatty runtime cannot grow them. */
151
208
  const MAX_PENDING_ACTIVITIES = 20;
152
209
  /**
@@ -188,7 +245,16 @@ const DEFAULT_TASK_PUBLISH_THROTTLE_MS = 2_000;
188
245
  * Claude worker's fold grace, for the same reason.
189
246
  */
190
247
  const DEFAULT_ORPHAN_GRACE_MS = 2_000;
191
-
248
+ /**
249
+ * How long a runtime teardown may wait on the runtime itself. A healthy one
250
+ * answers `session.detach` and stops in well under a second; one whose model
251
+ * stream had stalled took 180 s to answer the detach alone (runtime 1.0.88,
252
+ * 2026-09-27), and the SDK puts no timeout on that request. Past the first
253
+ * window the session is left to the stop, past the second the process is
254
+ * killed (`forceStop`).
255
+ */
256
+ const DEFAULT_RUNTIME_DETACH_TIMEOUT_MS = 10_000;
257
+ const DEFAULT_RUNTIME_STOP_TIMEOUT_MS = 25_000;
192
258
  /**
193
259
  * Appended to the partial answer when the RUNTIME interrupted the turn on its
194
260
  * own (as opposed to the user aborting through the relay). The row has to be
@@ -325,11 +391,19 @@ export function createCopilotSdkSessionRunner({
325
391
  infiniteSessionConfig = DEFAULT_INFINITE_SESSION_CONFIG,
326
392
  idleShutdownMs = DEFAULT_IDLE_SHUTDOWN_MS,
327
393
  lifecyclePollMs = DEFAULT_LIFECYCLE_POLL_MS,
328
- // 0 disables (matching the background-task timeout's 0 = no-limit
329
- // convention). When set, a turn that goes this long without a single event
330
- // fails terminally instead of holding the queue row until the relay's own
331
- // delivery watchdog gives up.
332
- turnStallTimeoutMs = DEFAULT_TURN_STALL_TIMEOUT_MS,
394
+ // How long a turn may go without a single event before it fails terminally
395
+ // (shared/worker-runtime/turn-liveness.mjs): `turnStallTimeoutMs` with
396
+ // nothing in flight, the two others while a model request or a tool is.
397
+ // 0 disables (the background-task timeout's 0 = no-limit convention).
398
+ // Emphatically NOT off by default: the worker's 10s heartbeat keeps renewing
399
+ // the relay's processing lease (messages-routes.mjs), so without a ceiling a
400
+ // turn whose runtime went quiet holds its queue row open until the relay's
401
+ // turn ceiling.
402
+ turnStallTimeoutMs = undefined,
403
+ turnStallModelTimeoutMs = undefined,
404
+ turnStallToolTimeoutMs = undefined,
405
+ stallProbeTimeoutMs = DEFAULT_STALL_PROBE_TIMEOUT_MS,
406
+ runtimeRpcTimeoutMs = DEFAULT_RUNTIME_RPC_TIMEOUT_MS,
333
407
  // How long a `deferred: true` model switch may wait for its
334
408
  // `session.model_change` drain before the explicit selection counts as
335
409
  // unconfirmed and fails the row (`COPILOT_SDK_RELAY_MODEL_SWITCH_TIMEOUT_MS`).
@@ -344,11 +418,24 @@ export function createCopilotSdkSessionRunner({
344
418
  // How long a settled continuation waits for its row before giving up on it.
345
419
  // Must outlast the registration's own retries; a test shortens it.
346
420
  continuationRegistrationTimeoutMs = CONTINUATION_REGISTRATION_TIMEOUT_MS,
421
+ continuationOutageWindowMs = DEFAULT_CONTINUATION_OUTAGE_WINDOW_MS,
422
+ continuationRequestTimeoutMs = DEFAULT_CONTINUATION_REQUEST_TIMEOUT_MS,
423
+ responseRetryWindowMs = DEFAULT_RESPONSE_RETRY_WINDOW_MS,
424
+ responseRequestTimeoutMs = DEFAULT_RESPONSE_REQUEST_TIMEOUT_MS,
425
+ livePostTimeoutMs = DEFAULT_LIVE_POST_TIMEOUT_MS,
426
+ livePostPauseMs = DEFAULT_LIVE_POST_PAUSE_MS,
347
427
  compactionStaleMs = DEFAULT_COMPACTION_STALE_MS,
348
428
  settleRetryDelaysMs = DEFAULT_SETTLE_RETRY_DELAYS_MS,
349
429
  maxTerminalSettleAttempts = DEFAULT_MAX_TERMINAL_SETTLE_ATTEMPTS,
350
430
  queueLaneRefreshMs = DEFAULT_QUEUE_LANE_REFRESH_MS,
351
431
  orphanGraceMs = DEFAULT_ORPHAN_GRACE_MS,
432
+ runtimeDetachTimeoutMs = DEFAULT_RUNTIME_DETACH_TIMEOUT_MS,
433
+ runtimeStopTimeoutMs = DEFAULT_RUNTIME_STOP_TIMEOUT_MS,
434
+ // The watch over the runtime's process tree (copilot-runtime-tree.mjs): what
435
+ // a runtime that was killed leaves running is stopped. Tests pass one with a
436
+ // fake process list; the fake client has no process, so nothing is watched
437
+ // without one.
438
+ watchRuntimeTreeImpl = watchCopilotRuntimeTree,
352
439
  // Register the relay's own `ask_user` (with `multi_select`) over the
353
440
  // runtime's built-in one. A runtime that refuses the override falls back
354
441
  // to the built-in for the rest of this worker's life.
@@ -378,6 +465,12 @@ export function createCopilotSdkSessionRunner({
378
465
  let client = null;
379
466
  let session = null;
380
467
  let sdkPaths = null;
468
+ const stallWindows = resolveTurnStallWindows({
469
+ env,
470
+ idleMs: turnStallTimeoutMs,
471
+ modelMs: turnStallModelTimeoutMs,
472
+ toolMs: turnStallToolTimeoutMs,
473
+ });
381
474
  // The single owner of "what model/effort is the live session CONFIRMED to be
382
475
  // on" (audit #9/#13): tracks the last confirmed pair, caches the per-session
383
476
  // model catalog for effort validation, and observes `session.model_change`
@@ -415,8 +508,20 @@ export function createCopilotSdkSessionRunner({
415
508
  // test seam only: the turn path never awaits a snapshot.
416
509
  let modelSnapshotChain = Promise.resolve();
417
510
  let starting = null;
511
+ // The teardown of the previous runtime while it is still in flight. The next
512
+ // runtime waits for it: the relay session id IS the SDK session id, so a
513
+ // second runtime resuming the session while the first still holds it would
514
+ // have two processes writing one event log.
515
+ let runtimeStopping = null;
516
+ // Counts runtime stops, so a session that finishes opening after its
517
+ // runtime was stopped is recognised and let go instead of adopted.
518
+ let runtimeGeneration = 0;
519
+ // The session open in flight, if any (see ensureSession).
520
+ let opening = null;
418
521
  let disposed = false;
419
522
  let detachRuntimeExit = () => {};
523
+ // The process tree of the live runtime, or null where its pid is not known.
524
+ let runtimeTree = null;
420
525
  // A compaction in progress (`session.compaction_start` seen, no `_complete`
421
526
  // yet). Holds steering: a message pushed now would land inside the
422
527
  // compaction's replay. `compactingSince` backs the stale cap.
@@ -424,6 +529,10 @@ export function createCopilotSdkSessionRunner({
424
529
  // The last readiness value reported to `onDeliveryReadinessChange`, so the
425
530
  // link only hears about flips.
426
531
  let lastReportedDeliveryReady = null;
532
+ // Deliveries handed back because a tool of the main agent was running (see
533
+ // mainToolsInFlight), counted until that hold ends. Only for the log: the
534
+ // messages themselves wait in the relay's queue.
535
+ let heldForRunningTool = 0;
427
536
  // Rows whose prompt the runtime consumed and whose settle marker is being
428
537
  // saved (id → { message, variant }). Reported by the heartbeat WITH a
429
538
  // terminal error until the publish lands, so recovery can never re-run them
@@ -507,6 +616,15 @@ export function createCopilotSdkSessionRunner({
507
616
  const agentDetails = new Map();
508
617
  let taskRefreshTimer = null;
509
618
  let taskRefreshChain = Promise.resolve();
619
+ // Reads of the registry that failed in a row. While there are some, "no
620
+ // live task" is not known but assumed, and the runtime is not closed over
621
+ // it: an idle shutdown on the strength of a failed read killed the shells
622
+ // and agents the read would have found.
623
+ let taskRefreshFailures = 0;
624
+ // A read that is on its way counts the same: its answer is not in yet.
625
+ let taskRefreshInFlight = false;
626
+ // Reads that are due but have not asked the runtime yet.
627
+ let taskRefreshQueued = 0;
510
628
  let lastTaskRefreshAt = 0;
511
629
  let taskPublishTimer = null;
512
630
  let lastPublishedTasksSignature = '';
@@ -703,16 +821,38 @@ export function createCopilotSdkSessionRunner({
703
821
 
704
822
  // ---------------------------------------------------------------- publish --
705
823
 
824
+ // Until when live output is skipped (see DEFAULT_LIVE_POST_TIMEOUT_MS).
825
+ let livePostsPausedUntil = 0;
826
+
827
+ /**
828
+ * One post of live output. Never rejects and never waits longer than
829
+ * `livePostTimeoutMs`; `droppable` output is skipped while the relay is
830
+ * known not to answer. A subagent's status is not droppable: a run left
831
+ * "running" in the transcript is worse than a late post.
832
+ */
833
+ async function postLive(routePath, body, { droppable = true } = {}) {
834
+ if (droppable && Date.now() < livePostsPausedUntil) return;
835
+ try {
836
+ await api('POST', routePath, body, livePostTimeoutMs > 0 ? { signal: AbortSignal.timeout(livePostTimeoutMs) } : {});
837
+ } catch (error) {
838
+ if (error?.name !== 'TimeoutError') return;
839
+ if (Date.now() >= livePostsPausedUntil) {
840
+ dbg(`the relay did not answer ${routePath} within ${livePostTimeoutMs}ms; live output is skipped for ${livePostPauseMs}ms`);
841
+ }
842
+ livePostsPausedUntil = Date.now() + livePostPauseMs;
843
+ }
844
+ }
845
+
706
846
  async function postActivity(message, text, subagentRunId = null) {
707
847
  if (!text) return;
708
- await api('POST', '/api/activity', {
848
+ await postLive('/api/activity', {
709
849
  messageId: message.id,
710
850
  conversationId: message.conversationId,
711
851
  mode: message.relayMode || 'agent',
712
852
  text,
713
853
  ...(subagentRunId ? { subagentRunId } : {}),
714
854
  ...attemptFields(message),
715
- }).catch(() => {});
855
+ });
716
856
  }
717
857
 
718
858
  async function dispatchAction(message, action, state) {
@@ -726,7 +866,7 @@ export function createCopilotSdkSessionRunner({
726
866
  // Only main-thread text can stand in for the answer on the abort/error
727
867
  // fallback paths; subagent text would publish as the reply.
728
868
  if (!payload.subagentRunId) state.lastStreamedText = payload.text;
729
- await api('POST', '/api/stream', {
869
+ await postLive('/api/stream', {
730
870
  messageId: message.id,
731
871
  conversationId: message.conversationId,
732
872
  mode: message.relayMode || 'agent',
@@ -734,11 +874,11 @@ export function createCopilotSdkSessionRunner({
734
874
  done: payload.done === true,
735
875
  ...(payload.subagentRunId ? { subagentRunId: payload.subagentRunId } : {}),
736
876
  ...attemptFields(message),
737
- }).catch(() => {});
877
+ });
738
878
  return;
739
879
  }
740
880
  if (channel === 'thought') {
741
- await api('POST', '/api/thought', {
881
+ await postLive('/api/thought', {
742
882
  messageId: message.id,
743
883
  conversationId: message.conversationId,
744
884
  mode: message.relayMode || 'agent',
@@ -747,7 +887,7 @@ export function createCopilotSdkSessionRunner({
747
887
  done: payload.done === true,
748
888
  ...(payload.subagentRunId ? { subagentRunId: payload.subagentRunId } : {}),
749
889
  ...attemptFields(message),
750
- }).catch(() => {});
890
+ });
751
891
  return;
752
892
  }
753
893
  if (channel === 'activity') {
@@ -758,7 +898,7 @@ export function createCopilotSdkSessionRunner({
758
898
  // Same body as every sibling worker's, so the lane bubbles, the
759
899
  // `subagent_status` broadcast and the UI's grouping behave identically
760
900
  // whichever provider produced the run.
761
- await api('POST', '/api/subagent-run', {
901
+ await postLive('/api/subagent-run', {
762
902
  messageId: message.id,
763
903
  conversationId: message.conversationId,
764
904
  subagentRunId: payload.subagentRunId,
@@ -766,7 +906,7 @@ export function createCopilotSdkSessionRunner({
766
906
  ...(payload.displayName ? { displayName: payload.displayName } : {}),
767
907
  status: payload.status,
768
908
  ...attemptFields(message),
769
- }).catch(() => {});
909
+ }, { droppable: false });
770
910
  }
771
911
  }
772
912
 
@@ -882,14 +1022,14 @@ export function createCopilotSdkSessionRunner({
882
1022
  }
883
1023
 
884
1024
  async function publishFinalStream(message, text) {
885
- await api('POST', '/api/stream', {
1025
+ await postLive('/api/stream', {
886
1026
  messageId: message.id,
887
1027
  conversationId: message.conversationId,
888
1028
  mode: message.relayMode || 'agent',
889
1029
  text: String(text || ''),
890
1030
  done: true,
891
1031
  ...attemptFields(message),
892
- }).catch(() => {});
1032
+ }, { droppable: false });
893
1033
  }
894
1034
 
895
1035
  /**
@@ -906,48 +1046,95 @@ export function createCopilotSdkSessionRunner({
906
1046
  */
907
1047
  async function publishResponse(message, {
908
1048
  text, model, terminalError = null, modelOrigin, kind = null, consumedSteerIds = null,
909
- requeueOnFailure = true,
1049
+ requeueOnFailure = true, partialText = '',
910
1050
  }) {
911
1051
  const consumed = Array.isArray(consumedSteerIds)
912
1052
  ? consumedSteerIds
913
1053
  .map((entry) => ({ id: String(entry?.id || '').trim(), attemptId: entry?.attemptId || null }))
914
1054
  .filter((entry) => entry.id)
915
1055
  : [];
916
- try {
917
- await api('POST', '/api/response', {
918
- messageId: message.id,
919
- conversationId: message.conversationId,
920
- text: String(text || ''),
921
- model: model || null,
922
- modelOrigin: modelOrigin
923
- || (String(message?.model || '').trim().toLowerCase() === 'auto' ? 'auto' : 'manual'),
924
- ...(terminalError ? { terminalError } : {}),
925
- ...(kind ? { kind } : {}),
926
- ...(consumed.length ? { consumedSteerIds: consumed } : {}),
927
- ...attemptFields(message),
928
- });
929
- return 'published';
930
- } catch (error) {
931
- // A stale_attempt 409 means the row already moved on to another attempt
932
- // (requeued and re-delivered); the requeue would 409 the same way, and
933
- // the newer attempt owes the row its answer — drop this one.
934
- if (isStaleAttemptError(error)) {
935
- dbg('response refused as stale_attempt; dropping', message.id);
936
- return 'stale';
1056
+ const body = {
1057
+ messageId: message.id,
1058
+ conversationId: message.conversationId,
1059
+ text: String(text || ''),
1060
+ model: model || null,
1061
+ modelOrigin: modelOrigin
1062
+ || (String(message?.model || '').trim().toLowerCase() === 'auto' ? 'auto' : 'manual'),
1063
+ ...(terminalError ? { terminalError } : {}),
1064
+ // What the turn had written before it failed; the relay keeps it
1065
+ // above the failure note.
1066
+ ...(terminalError && String(partialText || '').trim() ? { partialText: String(partialText).trim() } : {}),
1067
+ ...(kind ? { kind } : {}),
1068
+ ...(consumed.length ? { consumedSteerIds: consumed } : {}),
1069
+ ...attemptFields(message),
1070
+ };
1071
+ // A row's own reply is offered again while the relay cannot be reached:
1072
+ // it exists once, and the route answers a repeat of one it already saved
1073
+ // with `already_done`. Settle markers carry their own ladder
1074
+ // (publishSettleMarker), and a Stop's note is not worth a wait.
1075
+ const patient = !kind && requeueOnFailure;
1076
+ const requestLimit = () => (responseRequestTimeoutMs > 0 ? { signal: AbortSignal.timeout(responseRequestTimeoutMs) } : {});
1077
+ const startedAt = Date.now();
1078
+ for (let attempt = 0; ; attempt += 1) {
1079
+ try {
1080
+ await api('POST', '/api/response', body, requestLimit());
1081
+ if (attempt) dbg('response saved after the relay came back', message.id, `attempt ${attempt + 1}`);
1082
+ return 'published';
1083
+ } catch (error) {
1084
+ // A stale_attempt 409 means the row already moved on to another attempt
1085
+ // (requeued and re-delivered); the requeue would 409 the same way, and
1086
+ // the newer attempt owes the row its answer — drop this one.
1087
+ if (isStaleAttemptError(error)) {
1088
+ dbg('response refused as stale_attempt; dropping', message.id);
1089
+ return 'stale';
1090
+ }
1091
+ if (patient && !disposed && isTransientRelayError(error) && Date.now() - startedAt < responseRetryWindowMs) {
1092
+ if (!attempt) dbg('the relay did not take the response; trying again', message.id, error?.message || String(error));
1093
+ await sleep(settleRetryDelaysMs[Math.min(attempt, settleRetryDelaysMs.length - 1)] ?? 8_000);
1094
+ continue;
1095
+ }
1096
+ dbg('response could not be saved', message.id, error?.message || String(error));
1097
+ if (kind || !requeueOnFailure) return 'failed';
1098
+ const requeued = await api('POST', '/api/requeue', { messageId: message.id, ...attemptFields(message) }, requestLimit())
1099
+ .then(() => true, () => false);
1100
+ return requeued ? 'requeued' : 'failed';
937
1101
  }
938
- if (kind || !requeueOnFailure) return 'failed';
939
- const requeued = await api('POST', '/api/requeue', { messageId: message.id, ...attemptFields(message) })
940
- .then(() => true, () => false);
941
- return requeued ? 'requeued' : 'failed';
942
1102
  }
943
1103
  }
944
1104
 
945
- function terminalErrorRecord(message, classified) {
1105
+ /**
1106
+ * How many commands the runtime of a failed turn had running when its
1107
+ * teardown began, as far as it was known at that moment: the note does not
1108
+ * wait for a process list. On Linux the list is read at once; on Windows
1109
+ * it takes a PowerShell start, so the note there knows what the last read
1110
+ * knew. 0 where nothing is known or the runtime was kept.
1111
+ */
1112
+ function commandsLeftRunning(turn) {
1113
+ return Number(turn?.commandsLeft) || 0;
1114
+ }
1115
+
1116
+ /**
1117
+ * `toolCalls` is how many tool calls the failed turn had started. They ran,
1118
+ * and what they changed stays changed, so the note says so: a failure that
1119
+ * reads as if nothing happened sends the user back to the start of work
1120
+ * that is mostly done.
1121
+ */
1122
+ function terminalErrorRecord(message, classified, { toolCalls = 0, commandsLeft = 0 } = {}) {
1123
+ const worked = toolCalls > 0;
1124
+ const work = worked
1125
+ ? ` Before that the agent made ${toolCalls} tool call${toolCalls === 1 ? '' : 's'}; what they changed is still in place.`
1126
+ : '';
1127
+ // `commandsLeft` is how many commands were running when the turn failed.
1128
+ // They are stopped with the runtime, and a command that was cut short is
1129
+ // something the user has to know whatever the failure was.
1130
+ const stopping = describeCommandsBeingStopped(commandsLeft);
1131
+ const guidance = worked && classified.guidance ? COPILOT_TURN_CONTINUE_GUIDANCE : classified.guidance;
946
1132
  return {
947
1133
  kind: 'copilot-turn-failed',
948
1134
  code: classified.code,
949
1135
  stableCode: classified.stableCode,
950
- message: classified.text,
1136
+ message: `${classified.text}${classified.guidance ? work : ''}${stopping ? ` ${stopping}` : ''}`,
1137
+ ...(guidance ? { guidance } : {}),
951
1138
  failedAt: new Date().toISOString(),
952
1139
  queueMessageId: String(message.id || '') || null,
953
1140
  };
@@ -1196,6 +1383,9 @@ export function createCopilotSdkSessionRunner({
1196
1383
  // days ago.
1197
1384
  if (replayGate.isReplay(event)) {
1198
1385
  dbg('dropping a replayed event', String(event?.type || ''), String(event?.timestamp || ''));
1386
+ // History, but a sign of life all the same: a long resume replays for
1387
+ // as long as it takes, and the turn waiting on it is not stalled.
1388
+ activeTurn?.liveness?.traffic('replay');
1199
1389
  return;
1200
1390
  }
1201
1391
  touch();
@@ -1235,6 +1425,10 @@ export function createCopilotSdkSessionRunner({
1235
1425
  if (!isContinuationOpeningEvent(event)) return;
1236
1426
  turn = openContinuationTurn();
1237
1427
  }
1428
+ // Liveness is noted on RECEIPT. The dispatch chain below awaits relay
1429
+ // POSTs, and a watchdog fed from there measures the relay, not the
1430
+ // runtime: one slow POST would fail a turn whose runtime never paused.
1431
+ observeLiveness(turn, event);
1238
1432
  // The owner is captured HERE, synchronously, but settlement happens later
1239
1433
  // on the dispatch chain — so an event can be captured to a turn that is
1240
1434
  // settled by the time its link runs. `handleTurnEvent`'s settled guard
@@ -1247,6 +1441,57 @@ export function createCopilotSdkSessionRunner({
1247
1441
  .catch((error) => { dbg('event dispatch failed', error?.message || String(error)); });
1248
1442
  }
1249
1443
 
1444
+ /** A stage of the bring-up finished: the turn waiting on it is not stalled. */
1445
+ function liveTurnTraffic(what) {
1446
+ const turn = activeTurn;
1447
+ if (turn && !turn.settled) turn.liveness.traffic(what);
1448
+ }
1449
+
1450
+ /**
1451
+ * What the runtime is busy with, for the stall watchdog: a tool or a model
1452
+ * request that has started and not finished, per agent (a subagent's events
1453
+ * carry its `agentId` on the envelope). `model.call_*` is what runtime
1454
+ * 1.0.88 emits around a request; `assistant.turn_start`/`turn_end` bracket
1455
+ * the same span on runtimes that do not.
1456
+ */
1457
+ function observeLiveness(turn, event) {
1458
+ const liveness = turn?.liveness;
1459
+ if (!liveness || turn.settled) return;
1460
+ const type = String(event?.type || '');
1461
+ const data = event?.data && typeof event.data === 'object' ? event.data : {};
1462
+ const agent = String(event?.agentId || '').trim() || 'root';
1463
+ liveness.traffic(type);
1464
+ switch (type) {
1465
+ case 'tool.execution_start':
1466
+ liveness.begin('tool', `${agent}:${data.toolCallId || ''}`, data.toolName);
1467
+ break;
1468
+ case 'tool.execution_complete':
1469
+ liveness.end('tool', `${agent}:${data.toolCallId || ''}`);
1470
+ break;
1471
+ case 'assistant.turn_start':
1472
+ case 'model.call_start':
1473
+ liveness.begin('model', agent);
1474
+ break;
1475
+ case 'assistant.message':
1476
+ case 'assistant.turn_end':
1477
+ case 'model.call_finished':
1478
+ case 'model.call_failure':
1479
+ liveness.end('model', agent);
1480
+ break;
1481
+ case 'subagent.completed':
1482
+ case 'subagent.failed':
1483
+ // Whatever the agent had open ended with it.
1484
+ liveness.endAgent(agent);
1485
+ break;
1486
+ default:
1487
+ break;
1488
+ }
1489
+ // A tool of the main agent that starts or ends moves the steering gate
1490
+ // (see mainToolsInFlight). Told on receipt, like the liveness itself: the
1491
+ // relay must stop delivering before the next message is on its way.
1492
+ if (agent === 'root' && type.startsWith('tool.execution_')) syncDeliveryReadiness();
1493
+ }
1494
+
1250
1495
  /**
1251
1496
  * Keep the detached-shell set current and note when one settles.
1252
1497
  *
@@ -1417,7 +1662,11 @@ export function createCopilotSdkSessionRunner({
1417
1662
  if (!sessionRpc.tasks || taskRefreshTimer) return;
1418
1663
  taskRefreshTimer = setTimeout(() => {
1419
1664
  taskRefreshTimer = null;
1420
- taskRefreshChain = taskRefreshChain.then(() => refreshBackgroundTasks()).catch(() => {});
1665
+ taskRefreshQueued += 1;
1666
+ taskRefreshChain = taskRefreshChain
1667
+ .then(() => refreshBackgroundTasks())
1668
+ .catch(() => {})
1669
+ .finally(() => { taskRefreshQueued -= 1; });
1421
1670
  }, taskRefreshMs);
1422
1671
  taskRefreshTimer.unref?.();
1423
1672
  }
@@ -1434,13 +1683,18 @@ export function createCopilotSdkSessionRunner({
1434
1683
  if (!target || !sessionRpc.tasks) return;
1435
1684
  lastTaskRefreshAt = Date.now();
1436
1685
  let listed;
1686
+ taskRefreshInFlight = true;
1437
1687
  try {
1438
- listed = await target.rpc.tasks.list();
1688
+ listed = await askRuntime('tasks.list', () => target.rpc.tasks.list());
1439
1689
  } catch (error) {
1440
1690
  dbg('tasks.list failed', error?.message || String(error));
1691
+ if (session === target) taskRefreshFailures += 1;
1441
1692
  return;
1693
+ } finally {
1694
+ taskRefreshInFlight = false;
1442
1695
  }
1443
1696
  if (session !== target) return;
1697
+ taskRefreshFailures = 0;
1444
1698
  const next = new Map();
1445
1699
  for (const task of Array.isArray(listed?.tasks) ? listed.tasks : []) {
1446
1700
  const id = String(task?.id || '').trim();
@@ -1462,7 +1716,7 @@ export function createCopilotSdkSessionRunner({
1462
1716
  if (typeof target.rpc.tasks.getProgress === 'function') {
1463
1717
  for (const task of next.values()) {
1464
1718
  try {
1465
- const progress = (await target.rpc.tasks.getProgress({ id: task.id }))?.progress || null;
1719
+ const progress = (await askRuntime('tasks.getProgress', () => target.rpc.tasks.getProgress({ id: task.id })))?.progress || null;
1466
1720
  task.progressSummary = summarizeTaskProgress(progress);
1467
1721
  } catch { /* a card without a progress line */ }
1468
1722
  }
@@ -1586,7 +1840,7 @@ export function createCopilotSdkSessionRunner({
1586
1840
  const target = session;
1587
1841
  if (!target || !sessionRpc.queue) return;
1588
1842
  try {
1589
- const snapshot = await target.rpc.queue.pendingItems();
1843
+ const snapshot = await askRuntime('queue.pendingItems', () => target.rpc.queue.pendingItems());
1590
1844
  const next = new Map();
1591
1845
  for (const item of Array.isArray(snapshot?.items) ? snapshot.items : []) {
1592
1846
  const runtimeId = String(item?.messageId || '').trim();
@@ -1710,6 +1964,9 @@ export function createCopilotSdkSessionRunner({
1710
1964
  turn = openContinuationTurn();
1711
1965
  }
1712
1966
  }
1967
+ // Noted at receipt on the turn that owned the stream then; the turn that
1968
+ // takes the event over has to learn from it what is in flight.
1969
+ observeLiveness(turn, event);
1713
1970
  await handleTurnEvent(turn, event);
1714
1971
  }
1715
1972
 
@@ -1718,7 +1975,11 @@ export function createCopilotSdkSessionRunner({
1718
1975
  await redispatchSettledEvent(event);
1719
1976
  return;
1720
1977
  }
1721
- turn.armStall?.();
1978
+ if (String(event?.type || '') === 'tool.execution_start') {
1979
+ turn.toolCalls += 1;
1980
+ // A command may be starting: its process is read once it is there.
1981
+ runtimeTree?.commandStarting();
1982
+ }
1722
1983
  // Root work ends on the runtime's own terminator, so a question-only
1723
1984
  // continuation it joins no longer closes itself (whileAwaitingHuman).
1724
1985
  if (isContinuationOpeningEvent(event)) turn.rootWork = true;
@@ -1876,6 +2137,7 @@ export function createCopilotSdkSessionRunner({
1876
2137
  }
1877
2138
 
1878
2139
  function buildSessionConfig(model, reasoningEffort = null) {
2140
+ const generation = runtimeGeneration;
1879
2141
  // Built once per session, not per request. Reads the mode off the LIVE turn
1880
2142
  // rather than closing over the one the session was built with, because one
1881
2143
  // session serves every turn and the user can switch modes between them.
@@ -1936,7 +2198,10 @@ export function createCopilotSdkSessionRunner({
1936
2198
  clientName,
1937
2199
  // The runtime's own defaults, pinned. See DEFAULT_INFINITE_SESSION_CONFIG.
1938
2200
  ...(infiniteSessionConfig ? { infiniteSessions: { ...infiniteSessionConfig } } : {}),
1939
- onEvent: routeEvent,
2201
+ // Events of a runtime that is being stopped are not this session's any
2202
+ // more: the SDK keeps delivering them until the detach is answered, and
2203
+ // by then the next delivery may already be waiting as the active turn.
2204
+ onEvent: (event) => { if (generation === runtimeGeneration) routeEvent(event); },
1940
2205
  // The relay's own tools (`relaySessionTools`); omitted when there are none.
1941
2206
  ...(tools.length ? { tools } : {}),
1942
2207
  // Permission policy follows the conversation's relay mode: agent and
@@ -2077,9 +2342,13 @@ export function createCopilotSdkSessionRunner({
2077
2342
  dbg('copilot runtime exited', detail);
2078
2343
  const turn = activeTurn;
2079
2344
  if (turn && !turn.settled) {
2080
- // driveTurn tears the runtime down and publishes the failure record
2081
- // onto every row the turn owns.
2082
- turn.fail(new Error(`the Copilot runtime exited before the turn completed (${detail})`));
2345
+ // The turn's `fail` drops the dead handles, whatever kind of turn it
2346
+ // is, and the drive path publishes the failure record onto every row
2347
+ // the turn owns.
2348
+ turn.fail(Object.assign(
2349
+ new Error(`the Copilot runtime exited before the turn completed (${detail})`),
2350
+ { runtimeGone: true },
2351
+ ));
2083
2352
  return;
2084
2353
  }
2085
2354
  // No turn to fail: drop the dead handles so the next delivery rebuilds
@@ -2087,7 +2356,43 @@ export function createCopilotSdkSessionRunner({
2087
2356
  void stopRuntime('runtime-exit').catch(() => {});
2088
2357
  }
2089
2358
 
2359
+ /**
2360
+ * Start watching the commands of a runtime that has just started. Advisory:
2361
+ * a watch that cannot be set up costs the clean-up after a killed runtime,
2362
+ * never the turn.
2363
+ */
2364
+ function watchRuntimeTree(startedClient) {
2365
+ try {
2366
+ return watchRuntimeTreeImpl({
2367
+ client: startedClient,
2368
+ // Commands run inside a turn, or as background work between turns. A
2369
+ // runtime with neither is not read.
2370
+ isBusy: () => !disposed && (
2371
+ !!activeTurn || publishingTurns.size > 0 || backgroundShells.size() > 0 || backgroundTasks.size > 0
2372
+ ),
2373
+ dbg,
2374
+ }) || null;
2375
+ } catch (error) {
2376
+ dbg('the copilot runtime process tree is not watched', error?.message || String(error));
2377
+ return null;
2378
+ }
2379
+ }
2380
+
2381
+ /**
2382
+ * After the runtime is gone: stop what it left running, where it did not
2383
+ * end in good order (see `runtimeStopLeavesCommands`). Never rejects.
2384
+ */
2385
+ async function releaseRuntimeTree(tree, reason) {
2386
+ if (!tree) return;
2387
+ if (!runtimeStopLeavesCommands(reason)) {
2388
+ tree.close();
2389
+ return;
2390
+ }
2391
+ await tree.stop(reason);
2392
+ }
2393
+
2090
2394
  async function ensureClient() {
2395
+ while (runtimeStopping) await runtimeStopping;
2091
2396
  if (client) return client;
2092
2397
  if (starting) return starting;
2093
2398
  starting = (async () => {
@@ -2095,6 +2400,7 @@ export function createCopilotSdkSessionRunner({
2095
2400
  const started = await startClientImpl({ paths: sdkPaths, cwd, clientName, logLevel, dbg });
2096
2401
  client = started.client;
2097
2402
  detachRuntimeExit = observeRuntimeExit(client, handleRuntimeExit);
2403
+ runtimeTree = watchRuntimeTree(client);
2098
2404
  // Diagnostics only, and deliberately not awaited — see startCopilotClient.
2099
2405
  Promise.resolve(started.versionReady)
2100
2406
  .then((info) => {
@@ -2166,14 +2472,43 @@ export function createCopilotSdkSessionRunner({
2166
2472
  modelSwitch.reset();
2167
2473
  // Disconnect first so the runtime is not holding two handles on one
2168
2474
  // session id while the resume runs.
2169
- try { await closing?.disconnect?.(); } catch (error) {
2170
- dbg('session disconnect before model switch failed', error?.message || String(error));
2171
- }
2172
- return ensureSession(model, effort);
2475
+ const detached = await withTimeout(
2476
+ Promise.resolve().then(() => closing?.disconnect?.()).then(() => true, (error) => {
2477
+ dbg('session disconnect before model switch failed', error?.message || String(error));
2478
+ return true;
2479
+ }),
2480
+ runtimeDetachTimeoutMs,
2481
+ false,
2482
+ );
2483
+ if (!detached) dbg(`session disconnect before model switch did not answer within ${runtimeDetachTimeoutMs}ms`);
2484
+ return openSessionOnce(model, effort);
2173
2485
  }
2174
2486
 
2487
+ /**
2488
+ * One session open at a time. A turn can end before its bring-up does (the
2489
+ * stall watchdog, a Stop), and the open it started goes on without it; the
2490
+ * next turn must wait for that one rather than issue a second resume of the
2491
+ * same session id on the same client.
2492
+ */
2175
2493
  async function ensureSession(model, effort) {
2494
+ while (opening) await opening.catch(() => {});
2495
+ if (session) return applySelection(model, effort);
2496
+ return openSessionOnce(model, effort);
2497
+ }
2498
+
2499
+ async function openSessionOnce(model, effort) {
2500
+ const open = openFreshSession(model, effort);
2501
+ opening = open;
2502
+ try {
2503
+ return await open;
2504
+ } finally {
2505
+ if (opening === open) opening = null;
2506
+ }
2507
+ }
2508
+
2509
+ async function openFreshSession(model, effort) {
2176
2510
  await ensureClient();
2511
+ liveTurnTraffic('client-ready');
2177
2512
  if (!session) {
2178
2513
  // The tool list is part of the session: wait for the worker's first
2179
2514
  // registration answer (bounded, and it fails closed).
@@ -2186,8 +2521,16 @@ export function createCopilotSdkSessionRunner({
2186
2521
  // deliberately not cached: once `createSession` succeeds, the state
2187
2522
  // exists and the NEXT reconnect must resume it.
2188
2523
  let resumed = false;
2524
+ const generation = runtimeGeneration;
2525
+ // The turn this session was being opened for can end first (the stall
2526
+ // watchdog, a Stop), and its failure stops the runtime.
2527
+ const adopt = (opened) => {
2528
+ if (generation === runtimeGeneration) return opened;
2529
+ Promise.resolve().then(() => opened?.disconnect?.()).catch(() => {});
2530
+ throw new Error('the Copilot runtime was stopped while its session was opening');
2531
+ };
2189
2532
  try {
2190
- session = await openSession((cfg) => client.resumeSession(sdkSessionId, cfg), config);
2533
+ session = adopt(await openSession((cfg) => client.resumeSession(sdkSessionId, cfg), config));
2191
2534
  resumed = true;
2192
2535
  dbg(`resumed copilot session ${sdkSessionId.slice(0, 8)}`);
2193
2536
  } catch (error) {
@@ -2196,14 +2539,15 @@ export function createCopilotSdkSessionRunner({
2196
2539
  // transient, and starting fresh over live state would silently throw
2197
2540
  // the conversation's whole history away — so it fails the turn
2198
2541
  // instead, which is retryable and loses nothing.
2199
- if (!isSessionNotFoundError(error)) {
2542
+ if (!isSessionNotFoundError(error) || generation !== runtimeGeneration) {
2200
2543
  dbg('copilot session resume failed transiently', error?.message || String(error));
2201
2544
  throw error;
2202
2545
  }
2203
2546
  dbg('copilot session state not found, creating', error?.message || String(error));
2204
- session = await openSession((cfg) => client.createSession(cfg), config);
2547
+ session = adopt(await openSession((cfg) => client.createSession(cfg), config));
2205
2548
  dbg(`created copilot session ${sdkSessionId.slice(0, 8)}`);
2206
2549
  }
2550
+ liveTurnTraffic('session-open');
2207
2551
  // Which experimental surfaces THIS runtime has (it auto-updates under
2208
2552
  // the relay): decides un-steer and the targeted Stop for the session.
2209
2553
  probeSessionRpc(session);
@@ -2243,6 +2587,7 @@ export function createCopilotSdkSessionRunner({
2243
2587
  }
2244
2588
 
2245
2589
  async function stopRuntime(reason) {
2590
+ runtimeGeneration += 1;
2246
2591
  stopLifecycleTimer();
2247
2592
  // The next session is built from a fresh registration answer.
2248
2593
  void remoteRelayGate.refresh();
@@ -2250,8 +2595,10 @@ export function createCopilotSdkSessionRunner({
2250
2595
  detachRuntimeExit = () => {};
2251
2596
  const closingSession = session;
2252
2597
  const closingClient = client;
2598
+ const closingTree = runtimeTree;
2253
2599
  session = null;
2254
2600
  client = null;
2601
+ runtimeTree = null;
2255
2602
  modelSwitch.reset();
2256
2603
  // Detached shells are children of the runtime process, so stopping it ends
2257
2604
  // them: the tracked set is state about a process that no longer exists and
@@ -2281,17 +2628,86 @@ export function createCopilotSdkSessionRunner({
2281
2628
  waitingSteers = 0;
2282
2629
  queuedItemCount = 0;
2283
2630
  steerSendsInFlight = 0;
2631
+ taskRefreshFailures = 0;
2632
+ taskRefreshInFlight = false;
2633
+ // A removal this runtime never answered must not hold the next runtime's.
2634
+ unsteerChain = Promise.resolve();
2284
2635
  sessionRpc = { queue: false, interruptMainTurn: false, tasks: false };
2285
2636
  askUserOverrideRejected = false;
2286
2637
  syncDeliveryReadiness();
2287
- if (!closingSession && !closingClient) return;
2638
+ if (!closingSession && !closingClient) {
2639
+ // Nothing of its own to close, but a caller that asked for a stopped
2640
+ // runtime (the worker's shutdown) gets one.
2641
+ closingTree?.close();
2642
+ if (runtimeStopping) await runtimeStopping;
2643
+ return;
2644
+ }
2288
2645
  dbg(`stopping the copilot runtime (${reason})`);
2289
- try { await closingSession?.disconnect?.(); } catch (error) {
2290
- dbg('session disconnect failed', error?.message || String(error));
2646
+ // Read before the runtime is told to stop, in this very tick: what is
2647
+ // running now is what the failure note names (`turn.fail`), and a runtime
2648
+ // that has to be killed leaves it behind.
2649
+ const leavesCommands = !!closingTree && runtimeStopLeavesCommands(reason);
2650
+ if (leavesCommands) void closingTree.snapshot();
2651
+ const previous = runtimeStopping;
2652
+ const closing = (async () => {
2653
+ if (previous) await previous;
2654
+ // A runtime that has already ended stops nothing any more, and the
2655
+ // teardown below still waits out its windows on a connection that has
2656
+ // nobody behind it (10 s, measured): its commands are stopped at once.
2657
+ const early = leavesCommands
2658
+ ? closingTree.runtimeEnded().then((ended) => (ended ? closingTree.stop(reason) : null))
2659
+ : null;
2660
+ await closeRuntime(closingSession, closingClient, closingTree);
2661
+ await early;
2662
+ // Part of the teardown the next runtime waits for: a command of the old
2663
+ // one must not still be writing into the workspace the new one works in.
2664
+ await releaseRuntimeTree(closingTree, reason);
2665
+ })();
2666
+ runtimeStopping = closing;
2667
+ try {
2668
+ await closing;
2669
+ } finally {
2670
+ if (runtimeStopping === closing) runtimeStopping = null;
2671
+ }
2672
+ }
2673
+
2674
+ /**
2675
+ * Close a runtime whose handles were already dropped. Every step is bounded:
2676
+ * the callers are the failure paths, where the runtime is the thing that
2677
+ * stopped answering. Never rejects.
2678
+ */
2679
+ async function closeRuntime(closingSession, closingClient, closingTree = null) {
2680
+ const startedAt = Date.now();
2681
+ const bounded = (work, ms) => withTimeout(
2682
+ Promise.resolve().then(work).then((value) => {
2683
+ // `client.stop()` reports what it could not close instead of throwing.
2684
+ for (const error of Array.isArray(value) ? value : []) dbg('runtime stop reported', error?.message || String(error));
2685
+ return 'done';
2686
+ }, (error) => error),
2687
+ ms,
2688
+ 'timeout',
2689
+ );
2690
+ if (closingSession?.disconnect) {
2691
+ const detached = await bounded(() => closingSession.disconnect(), runtimeDetachTimeoutMs);
2692
+ if (detached === 'timeout') dbg(`session disconnect did not answer within ${runtimeDetachTimeoutMs}ms; stopping the runtime without it`);
2693
+ else if (detached !== 'done') dbg('session disconnect failed', detached?.message || String(detached));
2291
2694
  }
2292
- try { await closingClient?.stop?.(); } catch (error) {
2293
- dbg('client stop failed', error?.message || String(error));
2695
+ if (closingClient?.stop) {
2696
+ const stopped = await bounded(() => closingClient.stop(), runtimeStopTimeoutMs);
2697
+ if (stopped === 'timeout') {
2698
+ dbg(`client stop did not finish within ${runtimeStopTimeoutMs}ms; killing the runtime`);
2699
+ // The last moment the runtime's commands can be told from anybody
2700
+ // else's: a killed runtime leaves them to pid 1.
2701
+ if (closingTree) await closingTree.snapshot();
2702
+ const killed = closingClient.forceStop
2703
+ ? await bounded(() => closingClient.forceStop(), runtimeDetachTimeoutMs)
2704
+ : 'done';
2705
+ if (killed !== 'done') dbg('runtime kill did not finish', killed === 'timeout' ? 'timeout' : killed?.message || String(killed));
2706
+ } else if (stopped !== 'done') {
2707
+ dbg('client stop failed', stopped?.message || String(stopped));
2708
+ }
2294
2709
  }
2710
+ dbg(`copilot runtime stopped after ${Date.now() - startedAt}ms`);
2295
2711
  }
2296
2712
 
2297
2713
  // -------------------------------------------------------------- lifecycle --
@@ -2335,8 +2751,23 @@ export function createCopilotSdkSessionRunner({
2335
2751
  // re-read once per lifecycle poll catches state the events miss (a
2336
2752
  // shell that exited after a cancel emits no notification).
2337
2753
  const capMs = Number(getBackgroundTaskTimeoutMs()) || 0;
2754
+ if (taskRefreshFailures >= MAX_TASK_REFRESH_FAILURES && backgroundTasks.size) {
2755
+ // The runtime has not answered for its tasks for that many reads in a
2756
+ // row: the set is what it was when it last did, and holds nothing.
2757
+ dbg('the task registry cannot be read; its last answer no longer holds the runtime');
2758
+ backgroundTasks = new Map();
2759
+ publishBackgroundTasks({ immediate: true });
2760
+ }
2338
2761
  const live = runningBackgroundTasks();
2339
- if (!live.length) return false;
2762
+ if (!live.length) {
2763
+ // Nothing live as far as the last read that worked. While a read is
2764
+ // due or under way, or reads fail, that is not known: read again, and
2765
+ // hold the runtime meanwhile.
2766
+ if (taskRefreshTimer || taskRefreshQueued > 0 || taskRefreshInFlight) return true;
2767
+ if (!taskRefreshFailures || taskRefreshFailures >= MAX_TASK_REFRESH_FAILURES) return false;
2768
+ if (now - lastTaskRefreshAt >= lifecyclePollMs) scheduleTaskRefresh();
2769
+ return true;
2770
+ }
2340
2771
  if (capMs > 0) {
2341
2772
  for (const task of live) {
2342
2773
  const startedAt = Date.parse(String(task.startedAt || '')) || agentDetails.get(task.id)?.startedAt || 0;
@@ -2488,6 +2919,19 @@ export function createCopilotSdkSessionRunner({
2488
2919
  // The row the user pressed Stop on (from the abort control), when known.
2489
2920
  abortedRowId: null,
2490
2921
  stallTimer: null,
2922
+ stallChecking: false,
2923
+ liveness: createTurnLiveness({ windows: stallWindows }),
2924
+ // The runtime teardown a failure started (see `fail`).
2925
+ teardown: null,
2926
+ // Until when a continuation's registration may still get its row from a
2927
+ // relay it cannot reach (0: it is not in that state).
2928
+ registrationDeadline: 0,
2929
+ // Tool calls this turn started, whichever agent made them. Told in the
2930
+ // failure note when the turn fails.
2931
+ toolCalls: 0,
2932
+ // What the runtime had running when the turn failed (a promise of the
2933
+ // tree watch's read), for the failure note.
2934
+ commandsLeft: null,
2491
2935
  planBoardPosted: false,
2492
2936
  // Set when a mutating tool was actually approved and run this turn.
2493
2937
  acted: false,
@@ -2596,6 +3040,41 @@ export function createCopilotSdkSessionRunner({
2596
3040
  if (turn.settled) return;
2597
3041
  turn.settled = true;
2598
3042
  turn.disarmStall();
3043
+ // A delivered turn that failed (a stall, a runtime death, a setup
3044
+ // failure) leaves a runtime nothing else can use. The teardown drops the
3045
+ // handles in this very tick, before the turn gives up the event stream
3046
+ // and the worker reads as ready again, so the next delivery rebuilds
3047
+ // instead of sending into it. It is awaited only after the failure is
3048
+ // published (driveTurn): a runtime that stopped answering also takes
3049
+ // its time to stop, and the user waited three minutes for a failure
3050
+ // that was already decided (2026-09-27).
3051
+ //
3052
+ // An UNCONFIRMED MODEL SWITCH is the exception: the session is healthy
3053
+ // and merely still on its previous model, and tearing the runtime down
3054
+ // over it would kill any live background work for a selection problem
3055
+ // the user fixes by picking another model.
3056
+ //
3057
+ // A turn the runtime opened by itself keeps its runtime through a
3058
+ // failure, because the background work that opened it may still be
3059
+ // running there — unless the runtime itself is what went: it exited, or
3060
+ // it no longer answers. Those handles served nobody, and the next
3061
+ // message failed on them.
3062
+ const runtimeGone = error?.runtimeGone === true || error?.stall?.unresponsive === true;
3063
+ if ((turn.kind === 'delivered' || runtimeGone) && !isModelSwitchUnconfirmedError(error)) {
3064
+ if (session || client) {
3065
+ const tree = runtimeTree;
3066
+ turn.teardown = stopRuntime(runtimeGone ? 'runtime-gone' : 'turn-failure').catch(() => {});
3067
+ // What the runtime had running, as far as it is known now
3068
+ // (`stopRuntime` has just read the list where that takes no time):
3069
+ // the failure note is published before the teardown ends, and says
3070
+ // what is being stopped.
3071
+ turn.commandsLeft = tree ? tree.commandCount() : 0;
3072
+ } else {
3073
+ // Nothing to stop yet: the runtime was still starting. What it opens
3074
+ // for this turn from here on is let go rather than adopted.
3075
+ runtimeGeneration += 1;
3076
+ }
3077
+ }
2599
3078
  beginPublishing(turn);
2600
3079
  turn.resolvePrimaryReady();
2601
3080
  syncDeliveryReadiness();
@@ -2606,29 +3085,83 @@ export function createCopilotSdkSessionRunner({
2606
3085
  clearTimeout(turn.stallTimer);
2607
3086
  turn.stallTimer = null;
2608
3087
  };
3088
+ // The window restarts: the turn opens, or a hold on it ends.
2609
3089
  turn.armStall = () => {
2610
- if (!(turnStallTimeoutMs > 0) || turn.settled) return;
2611
- turn.disarmStall();
2612
- turn.stallTimer = setTimeout(() => {
2613
- // A human staring at a question card is not a stalled runtime. Re-arm
2614
- // rather than fail: the card has its own (much longer) timeout, and
2615
- // failing the row here would settle it while the answer is still coming.
2616
- // A remote_relay call waiting on another relay is the same kind of
2617
- // silence; the relay bounds that wait itself.
2618
- if (pendingHumanRequests > 0 || pendingRelayToolCalls > 0) {
2619
- turn.armStall();
2620
- return;
2621
- }
2622
- turn.fail(new Error(
2623
- `copilot worker watchdog: the runtime produced no events for ${Math.round(turnStallTimeoutMs / 1000)}s; `
2624
- + 'the row is failed — resend the message to retry',
2625
- ));
2626
- }, turnStallTimeoutMs);
2627
- turn.stallTimer.unref?.();
3090
+ if (!(stallWindows.idle > 0) || turn.settled) return;
3091
+ turn.liveness.traffic('armed');
3092
+ // A check in progress schedules the next one itself.
3093
+ if (!turn.stallTimer && !turn.stallChecking) scheduleStallCheck(turn, turn.liveness.check().waitMs);
2628
3094
  };
2629
3095
  return turn;
2630
3096
  }
2631
3097
 
3098
+ /**
3099
+ * The stall watchdog. One timer per turn, never re-armed by traffic: an
3100
+ * event only moves the turn's clock, and a check that comes early looks
3101
+ * again when the window is due (a model request emits thousands of deltas).
3102
+ */
3103
+ function scheduleStallCheck(turn, waitMs) {
3104
+ turn.stallTimer = setTimeout(() => {
3105
+ turn.stallTimer = null;
3106
+ turn.stallChecking = true;
3107
+ void checkStall(turn)
3108
+ .catch((error) => { dbg('stall check failed', error?.message || String(error)); })
3109
+ .finally(() => {
3110
+ turn.stallChecking = false;
3111
+ // A check that ended without deciding (it threw) must not be the last.
3112
+ if (!turn.settled && !turn.stallTimer) scheduleStallCheck(turn, turn.liveness.check().waitMs || stallWindows.idle);
3113
+ });
3114
+ }, Math.max(1, waitMs));
3115
+ turn.stallTimer.unref?.();
3116
+ }
3117
+
3118
+ async function checkStall(turn) {
3119
+ if (turn.settled) return;
3120
+ // A human staring at a question card is not a stalled runtime: the card
3121
+ // has its own (much longer) timeout, and failing the row here would
3122
+ // settle it while the answer is still coming. A remote_relay call waiting
3123
+ // on another relay is the same kind of silence; the relay bounds that
3124
+ // wait itself. So is a compaction, bounded by its stale cap, and the wait
3125
+ // for the previous runtime to stop, bounded by the teardown's own limits.
3126
+ if (pendingHumanRequests > 0 || pendingRelayToolCalls > 0 || isCompacting() || runtimeStopping) {
3127
+ turn.liveness.traffic('held');
3128
+ scheduleStallCheck(turn, turn.liveness.check().waitMs);
3129
+ return;
3130
+ }
3131
+ let verdict = turn.liveness.check();
3132
+ let unresponsive = false;
3133
+ if (!verdict.stalled && verdict.quietMs >= stallWindows.idle) {
3134
+ // Quiet for longer than a runtime with nothing to do may be, and kept
3135
+ // only because a tool or a model request is in flight. That is worth
3136
+ // one question: is the runtime still there?
3137
+ const probing = client;
3138
+ const ask = () => probeRuntime(
3139
+ typeof probing?.ping === 'function' ? () => probing.ping('stall-check') : null,
3140
+ stallProbeTimeoutMs,
3141
+ );
3142
+ let answer = await ask();
3143
+ // Asked twice before a turn with minutes left is given up: one ping
3144
+ // that came at a bad moment is not a runtime that is gone.
3145
+ if (answer === 'gone' && !turn.settled) answer = await ask();
3146
+ if (turn.settled) return;
3147
+ verdict = turn.liveness.check();
3148
+ unresponsive = answer === 'gone' && verdict.quietMs >= stallWindows.idle;
3149
+ if (!unresponsive && !verdict.stalled) dbg('turn is quiet, the runtime answers; keeping it', turn.liveness.describe());
3150
+ }
3151
+ if (!verdict.stalled && !unresponsive) {
3152
+ scheduleStallCheck(turn, verdict.waitMs);
3153
+ return;
3154
+ }
3155
+ dbg(`turn stalled${unresponsive ? ' (runtime not answering)' : ''}`, turn.liveness.describe());
3156
+ turn.fail(createTurnStalledError({
3157
+ agentLabel: SETTLE_AGENT_LABEL,
3158
+ quietMs: verdict.quietMs,
3159
+ phase: verdict.phase,
3160
+ tools: turn.liveness.toolsInFlight(),
3161
+ unresponsive,
3162
+ }));
3163
+ }
3164
+
2632
3165
  // ----------------------------------------------------- prompt boundaries --
2633
3166
 
2634
3167
  function claimSegment(turn, index, owner) {
@@ -2754,6 +3287,21 @@ export function createCopilotSdkSessionRunner({
2754
3287
  .catch((error) => { dbg('silent continuation drop failed', entry.message.id, error?.message || String(error)); });
2755
3288
  }
2756
3289
 
3290
+ /**
3291
+ * A row whose run is over and whose outcome reached the relay neither as a
3292
+ * response nor as a requeue. Releasing it as it is would leave it to the
3293
+ * relay's recovery, which runs the prompt again or fails it as a blocked
3294
+ * replay, whichever it finds. It is handed to the heartbeat instead, to be
3295
+ * failed with a note that says what happened: the run is not repeated.
3296
+ */
3297
+ function noteResponseOutcome(message, outcome, variant = 'replyLost') {
3298
+ const id = String(message?.id || '').trim();
3299
+ if (outcome !== 'failed' || !id) return;
3300
+ dbg('the outcome could not be saved or requeued; handing the row to the heartbeat', id, variant);
3301
+ claimSettling(message, variant);
3302
+ settleFailedTerminals.set(id, buildSteerSettleFailure(message, { variant, agentLabel: SETTLE_AGENT_LABEL }));
3303
+ }
3304
+
2757
3305
  /**
2758
3306
  * Decide and publish one row's outcome. Idempotent per entry. A run owner's
2759
3307
  * settle also settles the steers folded into its run (they were answered by
@@ -2782,11 +3330,11 @@ export function createCopilotSdkSessionRunner({
2782
3330
  await publishPlanBoard(turn, text, 'plan-mode-fallback');
2783
3331
  }
2784
3332
  await publishFinalStream(entry.message, published);
2785
- await publishResponse(entry.message, {
3333
+ noteResponseOutcome(entry.message, await publishResponse(entry.message, {
2786
3334
  text: published,
2787
3335
  model,
2788
3336
  consumedSteerIds: folded.map((other) => other.message),
2789
- });
3337
+ }));
2790
3338
  break;
2791
3339
  }
2792
3340
  case 'aborted-by-user': {
@@ -2825,23 +3373,38 @@ export function createCopilotSdkSessionRunner({
2825
3373
  const own = finalTextFor(turn, entry, turn.result);
2826
3374
  const text = own ? `${own}\n\n${RUNTIME_INTERRUPTED_NOTE}` : RUNTIME_INTERRUPTED_NOTE;
2827
3375
  await publishFinalStream(entry.message, text);
2828
- await publishResponse(entry.message, { text, model, consumedSteerIds: folded.map((other) => other.message) });
3376
+ noteResponseOutcome(
3377
+ entry.message,
3378
+ await publishResponse(entry.message, { text, model, consumedSteerIds: folded.map((other) => other.message) }),
3379
+ );
2829
3380
  break;
2830
3381
  }
2831
3382
  case 'error': {
2832
3383
  const { classified } = outcome;
2833
- await publishFinalStream(entry.message, entry.state.lastStreamedText);
2834
- await publishResponse(entry.message, {
2835
- text: classified.text,
3384
+ const written = finalTextFor(turn, entry, turn.result);
3385
+ const terminalError = terminalErrorRecord(entry.message, classified, {
3386
+ toolCalls: turn.toolCalls,
3387
+ commandsLeft: commandsLeftRunning(turn),
3388
+ });
3389
+ await publishFinalStream(entry.message, written || entry.state.lastStreamedText);
3390
+ noteResponseOutcome(entry.message, await publishResponse(entry.message, {
3391
+ text: terminalError.message,
2836
3392
  model,
2837
- terminalError: terminalErrorRecord(entry.message, classified),
3393
+ terminalError,
3394
+ partialText: written,
2838
3395
  consumedSteerIds: folded.map((other) => other.message),
2839
- });
3396
+ }), 'failureLost');
2840
3397
  break;
2841
3398
  }
2842
3399
  case 'folded':
2843
3400
  await publishSettleMarker(entry.message, { text: STEER_FOLDED_TEXT, model, kind: 'folded', variant: 'folded' });
2844
3401
  break;
3402
+ case 'failed-with-turn':
3403
+ // The runtime had taken this steer into a run that failed. The
3404
+ // failure is told on that run's reply; this row only says where to
3405
+ // look, and offers Resend like the steer of a stopped turn.
3406
+ await publishSettleMarker(entry.message, { text: STEER_TURN_FAILED_TEXT, model, kind: 'stopped', variant: 'turnFailed' });
3407
+ break;
2845
3408
  case 'stopped':
2846
3409
  await publishSettleMarker(entry.message, { text: STEER_STOPPED_TEXT, model, kind: 'stopped', variant: 'stopped' });
2847
3410
  break;
@@ -2850,7 +3413,11 @@ export function createCopilotSdkSessionRunner({
2850
3413
  // it will be answered at the start of the next interaction — a
2851
3414
  // requeue would run it twice.
2852
3415
  await publishFinalStream(entry.message, STEERED_ROW_MERGED_NOTE);
2853
- await publishResponse(entry.message, { text: STEERED_ROW_MERGED_NOTE, model });
3416
+ noteResponseOutcome(
3417
+ entry.message,
3418
+ await publishResponse(entry.message, { text: STEERED_ROW_MERGED_NOTE, model, requeueOnFailure: false }),
3419
+ 'folded',
3420
+ );
2854
3421
  break;
2855
3422
  case 'cancelled':
2856
3423
  case 'discarded':
@@ -2866,7 +3433,7 @@ export function createCopilotSdkSessionRunner({
2866
3433
  for (const other of folded) {
2867
3434
  const foldOutcome = outcome.type === 'completed' || outcome.type === 'merged-note'
2868
3435
  ? { type: 'folded' }
2869
- : outcome.type === 'error' ? outcome : { type: 'stopped' };
3436
+ : outcome.type === 'error' ? { type: 'failed-with-turn' } : { type: 'stopped' };
2870
3437
  await settleEntry(turn, other, foldOutcome);
2871
3438
  }
2872
3439
  }
@@ -2879,7 +3446,8 @@ export function createCopilotSdkSessionRunner({
2879
3446
  const itemId = queuedLane.get(entry.runtimeId);
2880
3447
  if (!itemId) continue;
2881
3448
  try {
2882
- await session.rpc.queue.removeAt({ id: itemId });
3449
+ const target = session;
3450
+ await askRuntime('queue.removeAt', () => target.rpc.queue.removeAt({ id: itemId }));
2883
3451
  dbg('removed a stopped prompt from the runtime queue', entry.message.id);
2884
3452
  } catch (error) {
2885
3453
  dbg('queue.removeAt after Stop failed', entry.message.id, error?.message || String(error));
@@ -2917,21 +3485,36 @@ export function createCopilotSdkSessionRunner({
2917
3485
  const pending = turn.entries.filter((entry) => !entry.settled && entry.turn === turn);
2918
3486
  if (!pending.length) return;
2919
3487
 
2920
- if (failure) {
3488
+ // A turn the user stopped ends as stopped, whatever happened to it
3489
+ // afterwards: a runtime that died or went silent under a Stop used to put
3490
+ // the watchdog's failure on every row instead.
3491
+ if (failure && !turn.aborted) {
2921
3492
  const classified = classifyCopilotTurnException(failure);
2922
3493
  dbg('copilot turn failed', turn.message?.id || '(continuation)', classified.detail);
3494
+ const running = turn.currentOwner || turn.primaryEntry;
2923
3495
  for (const entry of pending) {
2924
3496
  if (entry.settled) continue;
2925
- await settleEntry(turn, entry, { type: 'error', classified });
3497
+ // The failure is told once, on the row whose run it ended. A message
3498
+ // pushed into the turn gets the marker, whether the runtime had
3499
+ // folded it in (settled with its run's row) or never started it.
3500
+ const ownsRun = entry === running || entry.role !== 'steer' || (entry.consumed && !entry.foldedInto);
3501
+ // A steer whose `send()` has not come back never reached the runtime
3502
+ // as far as anyone knows. Its send is about to fail on the teardown
3503
+ // and requeues the row from there; a marker now would end a message
3504
+ // that can simply be delivered again.
3505
+ if (!ownsRun && !entry.runtimeId && !entry.consumed) continue;
3506
+ await settleEntry(turn, entry, ownsRun ? { type: 'error', classified } : { type: 'failed-with-turn' });
2926
3507
  }
2927
3508
  return;
2928
3509
  }
2929
3510
  if (result?.isError) {
2930
3511
  const classified = classifyCopilotSessionError(result.errorData || { message: result.errorMessage });
2931
3512
  dbg('turn failed', turn.message?.id || '(continuation)', classified.stableCode);
3513
+ const running = turn.currentOwner || turn.primaryEntry;
2932
3514
  for (const entry of pending) {
2933
3515
  if (entry.settled) continue;
2934
- await settleEntry(turn, entry, { type: 'error', classified });
3516
+ const ownsRun = entry === running || entry.role !== 'steer' || (entry.consumed && !entry.foldedInto);
3517
+ await settleEntry(turn, entry, ownsRun ? { type: 'error', classified } : { type: 'failed-with-turn' });
2935
3518
  }
2936
3519
  return;
2937
3520
  }
@@ -3084,24 +3667,62 @@ export function createCopilotSdkSessionRunner({
3084
3667
  // and stops waiting on the remote, and the watchdog hold is released.
3085
3668
  turn.remoteRelayStop?.abort();
3086
3669
  // While `ensureSession` is still connecting there is no session to abort
3087
- // and this would be a silent no-op; `runTurn` re-checks `turn.aborted`
3088
- // once the session exists and settles there instead.
3089
- if (!session) return;
3670
+ // and no prompt was sent: the turn settles here, as stopped, and `runTurn`
3671
+ // sends nothing once the session exists. Waiting for the session instead
3672
+ // left the Stop without effect for as long as the runtime took to answer.
3673
+ if (!session) {
3674
+ if (!turn.primarySent) {
3675
+ dbg('turn stopped before its session was ready', turn.message?.id || '');
3676
+ turn.settle();
3677
+ }
3678
+ return;
3679
+ }
3680
+ // Both requests are bounded. A Stop is what the user reaches for when a
3681
+ // turn is stuck, and a runtime that is stuck does not answer it either:
3682
+ // the Stop then did nothing until the stall watchdog failed the turn.
3683
+ const target = session;
3684
+ const unanswered = Symbol('unanswered');
3685
+ const bounded = (request) => withTimeout(Promise.resolve().then(request), runtimeRpcTimeoutMs, unanswered);
3686
+ let answered = true;
3090
3687
  if (sessionRpc.interruptMainTurn) {
3091
3688
  try {
3092
- const outcome = await session.rpc.interruptMainTurn({ flushQueued: false });
3093
- if (outcome?.interrupted === false) {
3094
- // Nothing was processing: no aborted idle will come. The turn is
3095
- // over as far as the runtime is concerned.
3096
- dbg('interruptMainTurn found no main turn in flight; settling locally', turn.message?.id || '');
3097
- turn.settle();
3689
+ const outcome = await bounded(() => target.rpc.interruptMainTurn({ flushQueued: false }));
3690
+ if (outcome !== unanswered) {
3691
+ if (outcome?.interrupted === false) {
3692
+ // Nothing was processing: no aborted idle will come. The turn is
3693
+ // over as far as the runtime is concerned.
3694
+ dbg('interruptMainTurn found no main turn in flight; settling locally', turn.message?.id || '');
3695
+ turn.settle();
3696
+ }
3697
+ return;
3098
3698
  }
3099
- return;
3699
+ answered = false;
3100
3700
  } catch (error) {
3101
3701
  dbg('interruptMainTurn failed; falling back to abort()', error?.message || String(error));
3102
3702
  }
3103
3703
  }
3104
- await session.abort?.();
3704
+ if (answered) {
3705
+ // A rejection is an answer of a kind the turn cannot use either: the
3706
+ // connection is gone (the runtime exited, or a teardown disposed it).
3707
+ // Thrown on from here, it reached the relay as a Stop that failed, and
3708
+ // the row was settled by nobody.
3709
+ answered = (await bounded(async () => { await target.abort?.(); }).catch((error) => {
3710
+ dbg('abort() failed', error?.message || String(error));
3711
+ return unanswered;
3712
+ })) !== unanswered;
3713
+ }
3714
+ if (answered || turn.settled) return;
3715
+ // No aborted idle is coming from a runtime that does not take the
3716
+ // request. The turn ends here, as stopped (finishTurn), and the runtime
3717
+ // goes with it.
3718
+ dbg('the runtime did not answer the Stop; ending the turn without it', turn.message?.id || '');
3719
+ turn.fail(createTurnStalledError({
3720
+ agentLabel: SETTLE_AGENT_LABEL,
3721
+ quietMs: turn.liveness.quietMs(),
3722
+ phase: turn.liveness.phase(),
3723
+ tools: turn.liveness.toolsInFlight(),
3724
+ unresponsive: true,
3725
+ }));
3105
3726
  }
3106
3727
 
3107
3728
  /**
@@ -3134,8 +3755,23 @@ export function createCopilotSdkSessionRunner({
3134
3755
  }) || null;
3135
3756
  turn.drive = driveTurn(turn);
3136
3757
  try {
3137
- await ensureSession(model, effort);
3138
- if (turn.aborted) {
3758
+ // The watchdog covers the bring-up as well. None of the requests that
3759
+ // open a session (resume, create, the model switch) has a timeout in
3760
+ // the SDK, and a runtime that never answered one held the row in
3761
+ // `processing` with no turn to fail and nothing for a Stop to stop.
3762
+ turn.armStall();
3763
+ const settledFirst = Symbol('turn-settled');
3764
+ const building = ensureSession(model, effort);
3765
+ // Still observed when the turn ends first and nothing awaits it.
3766
+ building.catch(() => {});
3767
+ const built = await Promise.race([
3768
+ building,
3769
+ turn.done.then(() => settledFirst, () => settledFirst),
3770
+ ]);
3771
+ if (built === settledFirst) {
3772
+ // Failed or stopped while the session was being built; nothing is sent.
3773
+ dbg('turn ended before its session was ready', message.id);
3774
+ } else if (turn.aborted) {
3139
3775
  // The abort landed while the session was still being built. Nothing
3140
3776
  // was sent, so there is no runtime turn to interrupt — settle locally
3141
3777
  // rather than sending a prompt the user just cancelled.
@@ -3161,7 +3797,6 @@ export function createCopilotSdkSessionRunner({
3161
3797
  // A send that never resolves (a wedged runtime) must not hold this
3162
3798
  // delivery hostage once the watchdog has failed the turn: the id is
3163
3799
  // still recorded if it ever arrives, but the wait ends with the turn.
3164
- const settledFirst = Symbol('turn-settled');
3165
3800
  const runtimeId = await Promise.race([
3166
3801
  sending,
3167
3802
  turn.done.then(() => settledFirst, () => settledFirst),
@@ -3182,16 +3817,8 @@ export function createCopilotSdkSessionRunner({
3182
3817
  }
3183
3818
  }
3184
3819
  } catch (error) {
3185
- // A failure that killed the session (or came from starting it) leaves a
3186
- // handle nothing else can use; drop it so the next delivery rebuilds and
3187
- // resumes rather than sending into a dead runtime. An UNCONFIRMED MODEL
3188
- // SWITCH is the exception: the session is healthy and merely still on
3189
- // its previous model, and tearing the runtime down over it would kill
3190
- // any live background work for a selection problem the user fixes by
3191
- // picking another model.
3192
- if (!isModelSwitchUnconfirmedError(error)) {
3193
- await stopRuntime('turn-failure').catch(() => {});
3194
- }
3820
+ // A failure that killed the session (or came from starting it); the
3821
+ // turn's own `fail` drops the handles nothing else can use.
3195
3822
  turn.fail(error);
3196
3823
  }
3197
3824
  // `send()` resolving is NOT the turn's completion; the event stream is.
@@ -3237,11 +3864,6 @@ export function createCopilotSdkSessionRunner({
3237
3864
  if (!turn.aborted && !turn.failure && turn.humanRequests > 0) await turn.humansIdle;
3238
3865
  try {
3239
3866
  await quiesceTurn(turn);
3240
- if (turn.failure && turn.kind === 'delivered' && !isModelSwitchUnconfirmedError(turn.failure) && session) {
3241
- // A stall or a runtime death mid-turn (a setup failure already tore
3242
- // the runtime down in runTurn).
3243
- await stopRuntime('turn-failure').catch(() => {});
3244
- }
3245
3867
  if (turn.kind === 'continuation') {
3246
3868
  // Nothing can be published before the row exists. The buffer is
3247
3869
  // drained by the registration itself; this only covers actions that
@@ -3274,6 +3896,8 @@ export function createCopilotSdkSessionRunner({
3274
3896
  // Early settles still in flight must land before the rows stop being
3275
3897
  // reported (settleEntry never rejects, but this is the ordering guard).
3276
3898
  await Promise.allSettled(turn.settlePromises);
3899
+ // The rows are settled; the runner is quiescent once the runtime is gone.
3900
+ if (turn.teardown) await turn.teardown;
3277
3901
  controlPoller?.stop?.(turn.controlState);
3278
3902
  turn.controlState = null;
3279
3903
  // Every path through the turn — published, failed, aborted, threw — has
@@ -3451,25 +4075,54 @@ export function createCopilotSdkSessionRunner({
3451
4075
  const signal = turn.registrationAbort?.signal || null;
3452
4076
  const abandoned = () => turn.discarded || signal?.aborted === true;
3453
4077
  let response = null;
3454
- for (let attempt = 0; attempt < 3 && !response?.messageId; attempt += 1) {
4078
+ // Two ways of not getting a row, counted apart. The relay ANSWERED without
4079
+ // one (it refused, or sent an empty body): three attempts, as ever. The
4080
+ // relay could not be REACHED (it is restarting): the attempts go on, spaced
4081
+ // further apart, for the outage window — the turn's output is buffered
4082
+ // meanwhile and nothing else can tell the user what the runtime did.
4083
+ let refusals = 0;
4084
+ let outageSince = 0;
4085
+ let outageDelayMs = continuationRetryDelayMs;
4086
+ while (!response?.messageId) {
3455
4087
  if (abandoned()) return;
4088
+ let failure = null;
4089
+ // The drive path waits at least as long as the request in flight may
4090
+ // take: a relay that took the connection and is slow to answer is not
4091
+ // one that refused.
4092
+ if (continuationOutageWindowMs > 0) {
4093
+ turn.registrationDeadline = Math.max(turn.registrationDeadline || 0, Date.now() + continuationRequestTimeoutMs);
4094
+ }
3456
4095
  response = await api('POST', '/api/continuation-turn', {
3457
4096
  conversationId: sdkSessionId,
3458
4097
  sdkSessionId,
3459
4098
  relayMode: turn.message.relayMode,
3460
4099
  trigger: CONTINUATION_TRIGGER,
3461
4100
  operationId,
3462
- }).catch((error) => {
4101
+ }, continuationRequestTimeoutMs > 0 ? { signal: AbortSignal.timeout(continuationRequestTimeoutMs) } : {}).catch((error) => {
4102
+ failure = error;
3463
4103
  dbg('continuation turn registration failed', error?.message || String(error));
3464
4104
  return null;
3465
4105
  });
3466
- if (!response?.messageId) {
3467
- if (abandoned()) return;
3468
- await new Promise((resolve) => { setTimeout(resolve, continuationRetryDelayMs); });
4106
+ if (response?.messageId) break;
4107
+ if (abandoned()) return;
4108
+ if (failure && continuationOutageWindowMs > 0 && isTransientRelayError(failure)) {
4109
+ outageSince = outageSince || Date.now();
4110
+ if (Date.now() - outageSince >= continuationOutageWindowMs) break;
4111
+ // The drive path waits for as long as this loop may still succeed.
4112
+ turn.registrationDeadline = outageSince + continuationOutageWindowMs + continuationRequestTimeoutMs;
4113
+ await sleep(outageDelayMs);
4114
+ outageDelayMs = Math.min(outageDelayMs * 2, 8_000);
4115
+ continue;
3469
4116
  }
4117
+ refusals += 1;
4118
+ if (refusals >= 3) break;
4119
+ await sleep(continuationRetryDelayMs);
3470
4120
  }
4121
+ turn.registrationDeadline = 0;
3471
4122
  if (!response?.messageId) {
3472
- abandonContinuationRegistration(turn, 'no relay message id after 3 attempts');
4123
+ abandonContinuationRegistration(turn, outageSince && refusals < 3
4124
+ ? `the relay could not be reached for ${Math.round((Date.now() - outageSince) / 1000)}s`
4125
+ : 'no relay message id after 3 attempts');
3473
4126
  return;
3474
4127
  }
3475
4128
  const messageId = String(response.messageId);
@@ -3518,17 +4171,24 @@ export function createCopilotSdkSessionRunner({
3518
4171
  */
3519
4172
  async function awaitContinuationRow(turn, timeoutMs = continuationRegistrationTimeoutMs) {
3520
4173
  if (turn.registered || turn.discarded) return;
3521
- let timer = null;
3522
- const expired = new Promise((resolve) => {
3523
- timer = setTimeout(() => resolve('expired'), timeoutMs);
3524
- timer.unref?.();
3525
- });
3526
- // `turn.registration` is the caught chain and never rejects.
3527
- const outcome = await Promise.race([turn.registration || Promise.resolve(), expired]);
3528
- clearTimeout(timer);
3529
- if (outcome === 'expired' && !turn.registered && !turn.message.id) {
3530
- abandonContinuationRegistration(turn, 'registration outlived the local deadline');
4174
+ let waitMs = timeoutMs;
4175
+ while (true) {
4176
+ let timer = null;
4177
+ const expired = new Promise((resolve) => {
4178
+ timer = setTimeout(() => resolve('expired'), waitMs);
4179
+ timer.unref?.();
4180
+ });
4181
+ // `turn.registration` is the caught chain and never rejects.
4182
+ const outcome = await Promise.race([turn.registration || Promise.resolve(), expired]);
4183
+ clearTimeout(timer);
4184
+ if (outcome !== 'expired' || turn.registered || turn.message.id) return;
4185
+ // Still trying against a relay that cannot be reached: the deadline is
4186
+ // the registration's own, not this wait's.
4187
+ const remainingMs = (turn.registrationDeadline || 0) - Date.now();
4188
+ if (remainingMs <= 0) break;
4189
+ waitMs = remainingMs;
3531
4190
  }
4191
+ abandonContinuationRegistration(turn, 'registration outlived the local deadline');
3532
4192
  }
3533
4193
 
3534
4194
  // --------------------------------------------------------------- steering --
@@ -3564,6 +4224,11 @@ export function createCopilotSdkSessionRunner({
3564
4224
  dbg('turn settled while steering was preparing; running it as a fresh turn', message.id);
3565
4225
  return NOT_STEERED;
3566
4226
  }
4227
+ // The same for a tool that started meanwhile: the message goes back to
4228
+ // the queue and comes again when the tool has ended.
4229
+ if (mainToolsInFlight(turn).length > 0 && canHandBackHeldDelivery() !== false) {
4230
+ return handBackHeldDelivery(message);
4231
+ }
3567
4232
  const entry = createEntry(turn, message, 'steer');
3568
4233
  dbg('steering a mid-turn delivery into the running turn', message.id);
3569
4234
  let runtimeId;
@@ -3611,7 +4276,8 @@ export function createCopilotSdkSessionRunner({
3611
4276
  * would settle the continuation's reply against no row while its buffered
3612
4277
  * output flushed into the steered one); a question card, ask-mode approval
3613
4278
  * or elicitation is open (the relay must not push past a card); a
3614
- * compaction is running.
4279
+ * compaction is running; a tool of the main agent is running (see
4280
+ * mainToolsInFlight).
3615
4281
  */
3616
4282
  function canAcceptSteering() {
3617
4283
  if (disposed || !session) return false;
@@ -3620,9 +4286,60 @@ export function createCopilotSdkSessionRunner({
3620
4286
  if (!turn.primarySent || awaitingRow(turn)) return false;
3621
4287
  if (pendingHumanRequests > 0) return false;
3622
4288
  if (isCompacting()) return false;
4289
+ if (mainToolsInFlight(turn).length > 0) return false;
3623
4290
  return true;
3624
4291
  }
3625
4292
 
4293
+ /**
4294
+ * The tool hold: the names of the tools the MAIN agent of this turn has
4295
+ * started and not finished. While there is one, nothing is steered in.
4296
+ *
4297
+ * The runtime answers a steer that arrives while it waits for a foreground
4298
+ * shell command by ending the tool call ("moved to background by the user")
4299
+ * and asking the model again. Seen live twice (2026-09-29): the turn then
4300
+ * ended with the answer to the steer alone, or failed on an empty reply
4301
+ * ("No response was returned"), and the command's result came later in a
4302
+ * continuation. Held back until the tool boundary, the steer lands where
4303
+ * the runtime would have injected it anyway and the command is left alone.
4304
+ *
4305
+ * There is no time limit on purpose: steering in after a wait would be the
4306
+ * same interruption, only later. A held message shows as queued, and the
4307
+ * end of the turn releases it like every other hold.
4308
+ *
4309
+ * Not counted:
4310
+ * - tools of subagents (their events carry an `agentId`). A background
4311
+ * agent works beside the main turn and for much longer; the main agent
4312
+ * is not waiting for its tools, so a steer interrupts nothing. A
4313
+ * subagent the main agent runs in the foreground is covered by the main
4314
+ * agent's own tool call that started it.
4315
+ * - `remote_relay`, which holds the stall watchdog but never held steering
4316
+ * (see `pendingRelayToolCalls`).
4317
+ */
4318
+ function mainToolsInFlight(turn = activeTurn) {
4319
+ if (!turn || turn.settled) return [];
4320
+ return turn.liveness.toolsOf('root').filter((name) => !isRemoteRelayToolName(name));
4321
+ }
4322
+
4323
+ /** Why the live turn takes no steer right now ('other': it is being stopped). */
4324
+ function steeringHoldReason(turn) {
4325
+ if (pendingHumanRequests > 0) return 'question';
4326
+ if (isCompacting()) return 'compaction';
4327
+ if (!turn.primarySent || awaitingRow(turn)) return 'delivery';
4328
+ if (mainToolsInFlight(turn).length > 0) return 'tool';
4329
+ return 'other';
4330
+ }
4331
+
4332
+ /**
4333
+ * The tool hold ended (the tool did, or the turn) and deliveries were
4334
+ * handed back for it: say so once. The relay sends them again on the
4335
+ * readiness `syncDeliveryReadiness` reports.
4336
+ */
4337
+ function noteRunningToolHoldEnd() {
4338
+ if (heldForRunningTool < 1 || mainToolsInFlight().length > 0) return;
4339
+ dbg('the running tool ended; steering no longer held for it', `handed back=${heldForRunningTool}`);
4340
+ heldForRunningTool = 0;
4341
+ }
4342
+
3626
4343
  /** A continuation whose synthetic row is still being minted (and not given up on). */
3627
4344
  function awaitingRow(turn) {
3628
4345
  return turn.kind === 'continuation' && !turn.registered && !turn.discarded;
@@ -3643,6 +4360,7 @@ export function createCopilotSdkSessionRunner({
3643
4360
  }
3644
4361
 
3645
4362
  function syncDeliveryReadiness() {
4363
+ noteRunningToolHoldEnd();
3646
4364
  const ready = !isDeliveryHeld();
3647
4365
  if (ready === lastReportedDeliveryReady) return;
3648
4366
  lastReportedDeliveryReady = ready;
@@ -3660,7 +4378,14 @@ export function createCopilotSdkSessionRunner({
3660
4378
  * hold instead would bypass the question card or land inside a compaction.
3661
4379
  */
3662
4380
  async function handBackHeldDelivery(message) {
3663
- dbg('steering held; handing the delivery back to the queue', message.id || '(no id)');
4381
+ const turn = activeTurn;
4382
+ if (turn && !turn.settled && steeringHoldReason(turn) === 'tool') {
4383
+ heldForRunningTool += 1;
4384
+ const tools = [...new Set(mainToolsInFlight(turn).map((name) => name || 'tool'))];
4385
+ dbg('steering held while a tool runs; handing the delivery back to the queue', message.id || '(no id)', `tools=${tools.join(', ')}`);
4386
+ } else {
4387
+ dbg('steering held; handing the delivery back to the queue', message.id || '(no id)');
4388
+ }
3664
4389
  if (!message.id) return false;
3665
4390
  await api('POST', '/api/requeue', {
3666
4391
  messageId: message.id,
@@ -3708,13 +4433,7 @@ export function createCopilotSdkSessionRunner({
3708
4433
  const turn = activeTurn;
3709
4434
  const turnActive = Boolean(turn && !turn.settled);
3710
4435
  const canSteer = canAcceptSteering();
3711
- let holdReason = null;
3712
- if (turnActive && !canSteer) {
3713
- if (pendingHumanRequests > 0) holdReason = 'question';
3714
- else if (isCompacting()) holdReason = 'compaction';
3715
- else if (!turn.primarySent || awaitingRow(turn)) holdReason = 'delivery';
3716
- else holdReason = 'other';
3717
- }
4436
+ const holdReason = turnActive && !canSteer ? steeringHoldReason(turn) : null;
3718
4437
  const messageId = String(turn?.currentOwner?.message?.id || turn?.message?.id || '').trim() || null;
3719
4438
  return { turnActive, canSteer, holdReason, messageId, supported: true, cancellableIds: cancellableIds() };
3720
4439
  }
@@ -3762,6 +4481,18 @@ export function createCopilotSdkSessionRunner({
3762
4481
  }
3763
4482
  }
3764
4483
 
4484
+ /**
4485
+ * A request to the runtime that has to come back. Rejects when the runtime
4486
+ * does not answer in time, so the caller's own failure path runs instead of
4487
+ * a chain that waits for ever.
4488
+ */
4489
+ async function askRuntime(label, request) {
4490
+ const unanswered = Symbol('unanswered');
4491
+ const answer = await withTimeout(Promise.resolve().then(request), runtimeRpcTimeoutMs, unanswered);
4492
+ if (answer === unanswered) throw new Error(`the runtime did not answer ${label} within ${runtimeRpcTimeoutMs}ms`);
4493
+ return answer;
4494
+ }
4495
+
3765
4496
  function withTimeout(promise, ms, fallback) {
3766
4497
  let timer = null;
3767
4498
  return Promise.race([
@@ -3850,12 +4581,12 @@ export function createCopilotSdkSessionRunner({
3850
4581
  async function handlePendingPayload(pending) {
3851
4582
  const message = pending?.message || null;
3852
4583
  if (!message) return false;
3853
- // Held (a card is open, a compaction is running, the turn-opening prompt
3854
- // is not sent yet): hand the row back with no retry penalty; the relay
3855
- // re-delivers it when the hold ends. An older relay (the worker updated on
3856
- // disk before the relay restarted) treats the hand-back as a failed
3857
- // delivery — retry, backoff, worker marked errored — so against one the
3858
- // message is pushed the legacy way.
4584
+ // Held (a card is open, a compaction is running, a tool of the main agent
4585
+ // is running, the turn-opening prompt is not sent yet): hand the row back
4586
+ // with no retry penalty; the relay re-delivers it when the hold ends. An
4587
+ // older relay (the worker updated on disk before the relay restarted)
4588
+ // treats the hand-back as a failed delivery — retry, backoff, worker
4589
+ // marked errored — so against one the message is pushed the legacy way.
3859
4590
  if (isDeliveryHeld() && canHandBackHeldDelivery() !== false) return handBackHeldDelivery(message);
3860
4591
  // Legacy path only: never send a second prompt before the turn-opening one
3861
4592
  // (it would invert the runs), and never start a second turn beside one