@oar-sh/oar 0.9.4 → 0.9.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/extensions/web-relay/polling/polling-loop.mjs +12 -32
- package/.github/extensions/web-relay/skills/question-bridge.mjs +14 -39
- package/CHANGELOG.md +199 -0
- package/README.md +35 -6
- package/package.json +2 -1
- package/server/claude-worker/claude-remote-relay-tool.mjs +1 -0
- package/server/claude-worker/claude-session-process.mjs +411 -19
- package/server/claude-worker/claude-session-worker.mjs +5 -0
- package/server/claude-worker/claude-turn-failure.mjs +92 -0
- package/server/claude-worker/claude-turn-publisher.mjs +52 -3
- package/server/claude-worker/sdk-message-normalizer.mjs +13 -1
- package/server/copilot-worker/copilot-model-switch.mjs +29 -3
- package/server/copilot-worker/copilot-question-bridge.mjs +2 -4
- package/server/copilot-worker/copilot-runtime-tree.mjs +91 -0
- package/server/copilot-worker/copilot-sdk-adapter.mjs +48 -8
- package/server/copilot-worker/copilot-sdk-session-process.mjs +897 -166
- package/server/copilot-worker/copilot-sdk-session-worker.mjs +5 -0
- package/server/cursor-worker/cursor-sdk-adapter.mjs +80 -13
- package/server/cursor-worker/cursor-session-worker.mjs +5 -0
- package/server/cursor-worker/cursor-turn-runner.mjs +5 -2
- package/server/db-schema.mjs +7 -0
- package/server/grok-worker/acp-client.mjs +68 -12
- package/server/grok-worker/grok-sdk-adapter.mjs +4 -0
- package/server/grok-worker/grok-session-worker.mjs +5 -0
- package/server/grok-worker/grok-turn-runner.mjs +6 -2
- package/server/public/app/api-client.js +17 -2
- package/server/public/app/bootstrap.js +3 -0
- package/server/public/app/composer-control-state.mjs +4 -3
- package/server/public/app/conversation-view.js +97 -4
- package/server/public/app/journal-view.js +6 -0
- package/server/public/app/markdown-strikethrough.mjs +31 -0
- package/server/public/app/relay-board-inline.mjs +70 -0
- package/server/public/app/relay-board-view.js +157 -61
- package/server/public/app/relay-error-ctas.mjs +12 -3
- package/server/public/app/socket-handlers.js +9 -0
- package/server/public/app/steered-turn-merge.mjs +10 -1
- package/server/public/app/store.js +4 -1
- package/server/public/app/subagent-fold.mjs +61 -0
- package/server/public/app/usage-limit-ui.js +225 -0
- package/server/public/index.html +97 -4
- package/server/repositories/message-repository.mjs +10 -1
- package/server/repositories/question-repository.mjs +1 -0
- package/server/repositories/remote-relay-repository.mjs +30 -0
- package/server/routes/messages-routes.mjs +93 -17
- package/server/routes/relay-board-routes.mjs +56 -2
- package/server/routes/sessions-routes.mjs +40 -2
- package/server/server-runtime.mjs +33 -2
- package/server/services/preview-proxy-server.mjs +64 -36
- package/server/services/process-list-service.mjs +122 -0
- package/server/services/relay-console-log.mjs +199 -0
- package/server/services/remote-relay-caller-context.mjs +45 -20
- package/server/services/remote-relay-dispatcher.mjs +328 -21
- package/server/services/remote-relay-inbound.mjs +35 -0
- package/server/services/session-worker-launch-service.mjs +17 -2
- package/server/services/session-worker-process-service.mjs +85 -32
- package/server/services/session-worker-registry-service.mjs +1 -1
- package/server/services/session-worker-supervisor-service.mjs +33 -1
- package/server/services/usage-limit-pause-service.mjs +441 -0
- package/shared/ask-user-bridge.mjs +14 -39
- package/shared/claude-usage-limit.mjs +257 -0
- package/shared/control-poller.mjs +69 -20
- package/shared/failure-note-text.mjs +38 -0
- package/shared/question-wait.mjs +108 -0
- package/shared/remote-relay-contract.mjs +33 -2
- package/shared/steer-settle-failure.mjs +22 -0
- package/shared/steer-settle-markers.mjs +24 -0
- package/shared/worker-runtime/process-tree.mjs +483 -0
- package/shared/worker-runtime/relay-errors.mjs +31 -0
- package/shared/worker-runtime/send-and-wait-errors.mjs +4 -4
- package/shared/worker-runtime/turn-liveness.mjs +249 -0
- package/shared/worker-runtime/worker-log-file.mjs +97 -0
|
@@ -48,6 +48,7 @@ import { randomUUID } from 'crypto';
|
|
|
48
48
|
|
|
49
49
|
import {
|
|
50
50
|
USER_INPUT_UNSUPPORTED_ANSWER,
|
|
51
|
+
COPILOT_TURN_CONTINUE_GUIDANCE,
|
|
51
52
|
classifyCopilotSessionError,
|
|
52
53
|
classifyCopilotTurnException,
|
|
53
54
|
copilotAgentModeForRelayMode,
|
|
@@ -76,6 +77,11 @@ import {
|
|
|
76
77
|
normalizeRelayEffort,
|
|
77
78
|
} from './copilot-model-switch.mjs';
|
|
78
79
|
import { createCopilotQuestionBridge } from './copilot-question-bridge.mjs';
|
|
80
|
+
import {
|
|
81
|
+
describeCommandsBeingStopped,
|
|
82
|
+
runtimeStopLeavesCommands,
|
|
83
|
+
watchCopilotRuntimeTree,
|
|
84
|
+
} from './copilot-runtime-tree.mjs';
|
|
79
85
|
import {
|
|
80
86
|
RELAY_ASK_USER_TOOL_DESCRIPTION,
|
|
81
87
|
RELAY_ASK_USER_TOOL_NAME,
|
|
@@ -85,7 +91,7 @@ import {
|
|
|
85
91
|
parseAskUserToolArguments,
|
|
86
92
|
} from './copilot-ask-user-tool.mjs';
|
|
87
93
|
import { buildCopilotRemoteRelayTool } from './copilot-remote-relay-tool.mjs';
|
|
88
|
-
import { createRemoteRelayToolGate } from '../../shared/remote-relay-tool-core.mjs';
|
|
94
|
+
import { createRemoteRelayToolGate, isRemoteRelayToolName } from '../../shared/remote-relay-tool-core.mjs';
|
|
89
95
|
import {
|
|
90
96
|
EXIT_PLAN_BOARD_POSTED_FEEDBACK,
|
|
91
97
|
EXIT_PLAN_NO_BOARD_FEEDBACK,
|
|
@@ -102,10 +108,17 @@ import {
|
|
|
102
108
|
import { EMPTY_TURN_COMPLETION_NOTE } from '../../shared/empty-turn-completion.mjs';
|
|
103
109
|
import { buildModelSnapshotFields, extractModelDescriptors } from '../../shared/model-descriptors.mjs';
|
|
104
110
|
import { buildRelayStopFailure } from '../../shared/relay-stop-failure.mjs';
|
|
105
|
-
import { STEER_FOLDED_TEXT, STEER_STOPPED_TEXT } from '../../shared/steer-settle-markers.mjs';
|
|
111
|
+
import { STEER_FOLDED_TEXT, STEER_STOPPED_TEXT, STEER_TURN_FAILED_TEXT } from '../../shared/steer-settle-markers.mjs';
|
|
106
112
|
import { buildSteerSettleFailure } from '../../shared/steer-settle-failure.mjs';
|
|
107
113
|
import { shouldEmitStreamUpdate } from '../../shared/stream-emit-gating.mjs';
|
|
108
114
|
import { withSteerNote } from '../../shared/steer-note.mjs';
|
|
115
|
+
import { isTransientRelayError } from '../../shared/worker-runtime/relay-errors.mjs';
|
|
116
|
+
import {
|
|
117
|
+
createTurnLiveness,
|
|
118
|
+
createTurnStalledError,
|
|
119
|
+
probeRuntime,
|
|
120
|
+
resolveTurnStallWindows,
|
|
121
|
+
} from '../../shared/worker-runtime/turn-liveness.mjs';
|
|
109
122
|
|
|
110
123
|
// How long the runtime may sit with no session activity before the worker
|
|
111
124
|
// closes it. The worker process itself stays up and reconnects lazily on the
|
|
@@ -114,11 +127,18 @@ import { withSteerNote } from '../../shared/steer-note.mjs';
|
|
|
114
127
|
// holding a runtime subprocess per idle conversation is not.
|
|
115
128
|
const DEFAULT_IDLE_SHUTDOWN_MS = 10 * 60_000;
|
|
116
129
|
const DEFAULT_LIFECYCLE_POLL_MS = 5_000;
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
130
|
+
/** How long the runtime may take to answer the stall watchdog's ping. */
|
|
131
|
+
const DEFAULT_STALL_PROBE_TIMEOUT_MS = 5_000;
|
|
132
|
+
/**
|
|
133
|
+
* How long the runtime may take to answer a request of the worker's own: a
|
|
134
|
+
* Stop, a read of its task registry or its message queue. None of them has a
|
|
135
|
+
* timeout in the SDK, and a healthy runtime answers within milliseconds even
|
|
136
|
+
* while it is busy (it answered a ping in 2 ms through a 150 s silent tool
|
|
137
|
+
* call, runtime 1.0.88).
|
|
138
|
+
*/
|
|
139
|
+
const DEFAULT_RUNTIME_RPC_TIMEOUT_MS = 10_000;
|
|
140
|
+
/** Failed reads of the task registry in a row before its last answer stops counting. */
|
|
141
|
+
const MAX_TASK_REFRESH_FAILURES = 5;
|
|
122
142
|
/**
|
|
123
143
|
* How long live background shells ALONE may keep the runtime up (0 = no limit).
|
|
124
144
|
*
|
|
@@ -143,10 +163,47 @@ const DEFAULT_BACKGROUND_TASK_TIMEOUT_MS = 30 * 60_000;
|
|
|
143
163
|
* worker's `notificationGraceMs`, same value.
|
|
144
164
|
*/
|
|
145
165
|
const DEFAULT_CONTINUATION_GRACE_MS = 60_000;
|
|
146
|
-
/**
|
|
166
|
+
/**
|
|
167
|
+
* Retry spacing for the synthetic-row registration: 3 attempts while the relay
|
|
168
|
+
* answers without minting a row, and the first step of the backoff while it
|
|
169
|
+
* cannot be reached at all.
|
|
170
|
+
*/
|
|
147
171
|
const DEFAULT_CONTINUATION_RETRY_DELAY_MS = 500;
|
|
172
|
+
/**
|
|
173
|
+
* How long the registration keeps trying while the relay cannot be reached. A
|
|
174
|
+
* relay restart takes seconds; three attempts half a second apart gave up
|
|
175
|
+
* inside it, and the whole continuation (the report of a background agent
|
|
176
|
+
* that had worked for an hour) was dropped.
|
|
177
|
+
*/
|
|
178
|
+
const DEFAULT_CONTINUATION_OUTAGE_WINDOW_MS = 120_000;
|
|
179
|
+
/** One registration request; the transport itself would wait 300 s. */
|
|
180
|
+
const DEFAULT_CONTINUATION_REQUEST_TIMEOUT_MS = 15_000;
|
|
148
181
|
/** How long a continuation's actions may buffer before its row is abandoned. */
|
|
149
182
|
const CONTINUATION_REGISTRATION_TIMEOUT_MS = 10_000;
|
|
183
|
+
/**
|
|
184
|
+
* How long a finished reply is offered to a relay that cannot be reached
|
|
185
|
+
* before the row is given up. The reply exists once; a requeue runs the
|
|
186
|
+
* prompt again, in a session that already holds it and its answer.
|
|
187
|
+
*/
|
|
188
|
+
const DEFAULT_RESPONSE_RETRY_WINDOW_MS = 5 * 60_000;
|
|
189
|
+
/**
|
|
190
|
+
* One attempt to save a reply, or to requeue its row. Without a limit of its
|
|
191
|
+
* own an attempt lasts as long as the transport lets it (300 s), which is the
|
|
192
|
+
* whole window: a relay that took the connection and did not answer got one
|
|
193
|
+
* attempt and then a requeue. Generous, because the route may hold a reply
|
|
194
|
+
* for a while before it answers (a question card that was just answered).
|
|
195
|
+
*/
|
|
196
|
+
const DEFAULT_RESPONSE_REQUEST_TIMEOUT_MS = 90_000;
|
|
197
|
+
/**
|
|
198
|
+
* Live output (stream, thoughts, activity lines) is posted from the dispatch
|
|
199
|
+
* chain, one event after the other. A relay that accepts the connection and
|
|
200
|
+
* then does not answer held every later event, the turn's terminator
|
|
201
|
+
* included, for the transport's 300 s. One post waits this long; after one
|
|
202
|
+
* that ran out, live output is skipped for the pause below. The reply itself
|
|
203
|
+
* does not go this way.
|
|
204
|
+
*/
|
|
205
|
+
const DEFAULT_LIVE_POST_TIMEOUT_MS = 10_000;
|
|
206
|
+
const DEFAULT_LIVE_POST_PAUSE_MS = 30_000;
|
|
150
207
|
/** Cap on activity lines carried between turns, so a chatty runtime cannot grow them. */
|
|
151
208
|
const MAX_PENDING_ACTIVITIES = 20;
|
|
152
209
|
/**
|
|
@@ -188,7 +245,16 @@ const DEFAULT_TASK_PUBLISH_THROTTLE_MS = 2_000;
|
|
|
188
245
|
* Claude worker's fold grace, for the same reason.
|
|
189
246
|
*/
|
|
190
247
|
const DEFAULT_ORPHAN_GRACE_MS = 2_000;
|
|
191
|
-
|
|
248
|
+
/**
|
|
249
|
+
* How long a runtime teardown may wait on the runtime itself. A healthy one
|
|
250
|
+
* answers `session.detach` and stops in well under a second; one whose model
|
|
251
|
+
* stream had stalled took 180 s to answer the detach alone (runtime 1.0.88,
|
|
252
|
+
* 2026-09-27), and the SDK puts no timeout on that request. Past the first
|
|
253
|
+
* window the session is left to the stop, past the second the process is
|
|
254
|
+
* killed (`forceStop`).
|
|
255
|
+
*/
|
|
256
|
+
const DEFAULT_RUNTIME_DETACH_TIMEOUT_MS = 10_000;
|
|
257
|
+
const DEFAULT_RUNTIME_STOP_TIMEOUT_MS = 25_000;
|
|
192
258
|
/**
|
|
193
259
|
* Appended to the partial answer when the RUNTIME interrupted the turn on its
|
|
194
260
|
* own (as opposed to the user aborting through the relay). The row has to be
|
|
@@ -325,11 +391,19 @@ export function createCopilotSdkSessionRunner({
|
|
|
325
391
|
infiniteSessionConfig = DEFAULT_INFINITE_SESSION_CONFIG,
|
|
326
392
|
idleShutdownMs = DEFAULT_IDLE_SHUTDOWN_MS,
|
|
327
393
|
lifecyclePollMs = DEFAULT_LIFECYCLE_POLL_MS,
|
|
328
|
-
//
|
|
329
|
-
//
|
|
330
|
-
//
|
|
331
|
-
//
|
|
332
|
-
|
|
394
|
+
// How long a turn may go without a single event before it fails terminally
|
|
395
|
+
// (shared/worker-runtime/turn-liveness.mjs): `turnStallTimeoutMs` with
|
|
396
|
+
// nothing in flight, the two others while a model request or a tool is.
|
|
397
|
+
// 0 disables (the background-task timeout's 0 = no-limit convention).
|
|
398
|
+
// Emphatically NOT off by default: the worker's 10s heartbeat keeps renewing
|
|
399
|
+
// the relay's processing lease (messages-routes.mjs), so without a ceiling a
|
|
400
|
+
// turn whose runtime went quiet holds its queue row open until the relay's
|
|
401
|
+
// turn ceiling.
|
|
402
|
+
turnStallTimeoutMs = undefined,
|
|
403
|
+
turnStallModelTimeoutMs = undefined,
|
|
404
|
+
turnStallToolTimeoutMs = undefined,
|
|
405
|
+
stallProbeTimeoutMs = DEFAULT_STALL_PROBE_TIMEOUT_MS,
|
|
406
|
+
runtimeRpcTimeoutMs = DEFAULT_RUNTIME_RPC_TIMEOUT_MS,
|
|
333
407
|
// How long a `deferred: true` model switch may wait for its
|
|
334
408
|
// `session.model_change` drain before the explicit selection counts as
|
|
335
409
|
// unconfirmed and fails the row (`COPILOT_SDK_RELAY_MODEL_SWITCH_TIMEOUT_MS`).
|
|
@@ -344,11 +418,24 @@ export function createCopilotSdkSessionRunner({
|
|
|
344
418
|
// How long a settled continuation waits for its row before giving up on it.
|
|
345
419
|
// Must outlast the registration's own retries; a test shortens it.
|
|
346
420
|
continuationRegistrationTimeoutMs = CONTINUATION_REGISTRATION_TIMEOUT_MS,
|
|
421
|
+
continuationOutageWindowMs = DEFAULT_CONTINUATION_OUTAGE_WINDOW_MS,
|
|
422
|
+
continuationRequestTimeoutMs = DEFAULT_CONTINUATION_REQUEST_TIMEOUT_MS,
|
|
423
|
+
responseRetryWindowMs = DEFAULT_RESPONSE_RETRY_WINDOW_MS,
|
|
424
|
+
responseRequestTimeoutMs = DEFAULT_RESPONSE_REQUEST_TIMEOUT_MS,
|
|
425
|
+
livePostTimeoutMs = DEFAULT_LIVE_POST_TIMEOUT_MS,
|
|
426
|
+
livePostPauseMs = DEFAULT_LIVE_POST_PAUSE_MS,
|
|
347
427
|
compactionStaleMs = DEFAULT_COMPACTION_STALE_MS,
|
|
348
428
|
settleRetryDelaysMs = DEFAULT_SETTLE_RETRY_DELAYS_MS,
|
|
349
429
|
maxTerminalSettleAttempts = DEFAULT_MAX_TERMINAL_SETTLE_ATTEMPTS,
|
|
350
430
|
queueLaneRefreshMs = DEFAULT_QUEUE_LANE_REFRESH_MS,
|
|
351
431
|
orphanGraceMs = DEFAULT_ORPHAN_GRACE_MS,
|
|
432
|
+
runtimeDetachTimeoutMs = DEFAULT_RUNTIME_DETACH_TIMEOUT_MS,
|
|
433
|
+
runtimeStopTimeoutMs = DEFAULT_RUNTIME_STOP_TIMEOUT_MS,
|
|
434
|
+
// The watch over the runtime's process tree (copilot-runtime-tree.mjs): what
|
|
435
|
+
// a runtime that was killed leaves running is stopped. Tests pass one with a
|
|
436
|
+
// fake process list; the fake client has no process, so nothing is watched
|
|
437
|
+
// without one.
|
|
438
|
+
watchRuntimeTreeImpl = watchCopilotRuntimeTree,
|
|
352
439
|
// Register the relay's own `ask_user` (with `multi_select`) over the
|
|
353
440
|
// runtime's built-in one. A runtime that refuses the override falls back
|
|
354
441
|
// to the built-in for the rest of this worker's life.
|
|
@@ -378,6 +465,12 @@ export function createCopilotSdkSessionRunner({
|
|
|
378
465
|
let client = null;
|
|
379
466
|
let session = null;
|
|
380
467
|
let sdkPaths = null;
|
|
468
|
+
const stallWindows = resolveTurnStallWindows({
|
|
469
|
+
env,
|
|
470
|
+
idleMs: turnStallTimeoutMs,
|
|
471
|
+
modelMs: turnStallModelTimeoutMs,
|
|
472
|
+
toolMs: turnStallToolTimeoutMs,
|
|
473
|
+
});
|
|
381
474
|
// The single owner of "what model/effort is the live session CONFIRMED to be
|
|
382
475
|
// on" (audit #9/#13): tracks the last confirmed pair, caches the per-session
|
|
383
476
|
// model catalog for effort validation, and observes `session.model_change`
|
|
@@ -415,8 +508,20 @@ export function createCopilotSdkSessionRunner({
|
|
|
415
508
|
// test seam only: the turn path never awaits a snapshot.
|
|
416
509
|
let modelSnapshotChain = Promise.resolve();
|
|
417
510
|
let starting = null;
|
|
511
|
+
// The teardown of the previous runtime while it is still in flight. The next
|
|
512
|
+
// runtime waits for it: the relay session id IS the SDK session id, so a
|
|
513
|
+
// second runtime resuming the session while the first still holds it would
|
|
514
|
+
// have two processes writing one event log.
|
|
515
|
+
let runtimeStopping = null;
|
|
516
|
+
// Counts runtime stops, so a session that finishes opening after its
|
|
517
|
+
// runtime was stopped is recognised and let go instead of adopted.
|
|
518
|
+
let runtimeGeneration = 0;
|
|
519
|
+
// The session open in flight, if any (see ensureSession).
|
|
520
|
+
let opening = null;
|
|
418
521
|
let disposed = false;
|
|
419
522
|
let detachRuntimeExit = () => {};
|
|
523
|
+
// The process tree of the live runtime, or null where its pid is not known.
|
|
524
|
+
let runtimeTree = null;
|
|
420
525
|
// A compaction in progress (`session.compaction_start` seen, no `_complete`
|
|
421
526
|
// yet). Holds steering: a message pushed now would land inside the
|
|
422
527
|
// compaction's replay. `compactingSince` backs the stale cap.
|
|
@@ -424,6 +529,10 @@ export function createCopilotSdkSessionRunner({
|
|
|
424
529
|
// The last readiness value reported to `onDeliveryReadinessChange`, so the
|
|
425
530
|
// link only hears about flips.
|
|
426
531
|
let lastReportedDeliveryReady = null;
|
|
532
|
+
// Deliveries handed back because a tool of the main agent was running (see
|
|
533
|
+
// mainToolsInFlight), counted until that hold ends. Only for the log: the
|
|
534
|
+
// messages themselves wait in the relay's queue.
|
|
535
|
+
let heldForRunningTool = 0;
|
|
427
536
|
// Rows whose prompt the runtime consumed and whose settle marker is being
|
|
428
537
|
// saved (id → { message, variant }). Reported by the heartbeat WITH a
|
|
429
538
|
// terminal error until the publish lands, so recovery can never re-run them
|
|
@@ -507,6 +616,15 @@ export function createCopilotSdkSessionRunner({
|
|
|
507
616
|
const agentDetails = new Map();
|
|
508
617
|
let taskRefreshTimer = null;
|
|
509
618
|
let taskRefreshChain = Promise.resolve();
|
|
619
|
+
// Reads of the registry that failed in a row. While there are some, "no
|
|
620
|
+
// live task" is not known but assumed, and the runtime is not closed over
|
|
621
|
+
// it: an idle shutdown on the strength of a failed read killed the shells
|
|
622
|
+
// and agents the read would have found.
|
|
623
|
+
let taskRefreshFailures = 0;
|
|
624
|
+
// A read that is on its way counts the same: its answer is not in yet.
|
|
625
|
+
let taskRefreshInFlight = false;
|
|
626
|
+
// Reads that are due but have not asked the runtime yet.
|
|
627
|
+
let taskRefreshQueued = 0;
|
|
510
628
|
let lastTaskRefreshAt = 0;
|
|
511
629
|
let taskPublishTimer = null;
|
|
512
630
|
let lastPublishedTasksSignature = '';
|
|
@@ -703,16 +821,38 @@ export function createCopilotSdkSessionRunner({
|
|
|
703
821
|
|
|
704
822
|
// ---------------------------------------------------------------- publish --
|
|
705
823
|
|
|
824
|
+
// Until when live output is skipped (see DEFAULT_LIVE_POST_TIMEOUT_MS).
|
|
825
|
+
let livePostsPausedUntil = 0;
|
|
826
|
+
|
|
827
|
+
/**
|
|
828
|
+
* One post of live output. Never rejects and never waits longer than
|
|
829
|
+
* `livePostTimeoutMs`; `droppable` output is skipped while the relay is
|
|
830
|
+
* known not to answer. A subagent's status is not droppable: a run left
|
|
831
|
+
* "running" in the transcript is worse than a late post.
|
|
832
|
+
*/
|
|
833
|
+
async function postLive(routePath, body, { droppable = true } = {}) {
|
|
834
|
+
if (droppable && Date.now() < livePostsPausedUntil) return;
|
|
835
|
+
try {
|
|
836
|
+
await api('POST', routePath, body, livePostTimeoutMs > 0 ? { signal: AbortSignal.timeout(livePostTimeoutMs) } : {});
|
|
837
|
+
} catch (error) {
|
|
838
|
+
if (error?.name !== 'TimeoutError') return;
|
|
839
|
+
if (Date.now() >= livePostsPausedUntil) {
|
|
840
|
+
dbg(`the relay did not answer ${routePath} within ${livePostTimeoutMs}ms; live output is skipped for ${livePostPauseMs}ms`);
|
|
841
|
+
}
|
|
842
|
+
livePostsPausedUntil = Date.now() + livePostPauseMs;
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
706
846
|
async function postActivity(message, text, subagentRunId = null) {
|
|
707
847
|
if (!text) return;
|
|
708
|
-
await
|
|
848
|
+
await postLive('/api/activity', {
|
|
709
849
|
messageId: message.id,
|
|
710
850
|
conversationId: message.conversationId,
|
|
711
851
|
mode: message.relayMode || 'agent',
|
|
712
852
|
text,
|
|
713
853
|
...(subagentRunId ? { subagentRunId } : {}),
|
|
714
854
|
...attemptFields(message),
|
|
715
|
-
})
|
|
855
|
+
});
|
|
716
856
|
}
|
|
717
857
|
|
|
718
858
|
async function dispatchAction(message, action, state) {
|
|
@@ -726,7 +866,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
726
866
|
// Only main-thread text can stand in for the answer on the abort/error
|
|
727
867
|
// fallback paths; subagent text would publish as the reply.
|
|
728
868
|
if (!payload.subagentRunId) state.lastStreamedText = payload.text;
|
|
729
|
-
await
|
|
869
|
+
await postLive('/api/stream', {
|
|
730
870
|
messageId: message.id,
|
|
731
871
|
conversationId: message.conversationId,
|
|
732
872
|
mode: message.relayMode || 'agent',
|
|
@@ -734,11 +874,11 @@ export function createCopilotSdkSessionRunner({
|
|
|
734
874
|
done: payload.done === true,
|
|
735
875
|
...(payload.subagentRunId ? { subagentRunId: payload.subagentRunId } : {}),
|
|
736
876
|
...attemptFields(message),
|
|
737
|
-
})
|
|
877
|
+
});
|
|
738
878
|
return;
|
|
739
879
|
}
|
|
740
880
|
if (channel === 'thought') {
|
|
741
|
-
await
|
|
881
|
+
await postLive('/api/thought', {
|
|
742
882
|
messageId: message.id,
|
|
743
883
|
conversationId: message.conversationId,
|
|
744
884
|
mode: message.relayMode || 'agent',
|
|
@@ -747,7 +887,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
747
887
|
done: payload.done === true,
|
|
748
888
|
...(payload.subagentRunId ? { subagentRunId: payload.subagentRunId } : {}),
|
|
749
889
|
...attemptFields(message),
|
|
750
|
-
})
|
|
890
|
+
});
|
|
751
891
|
return;
|
|
752
892
|
}
|
|
753
893
|
if (channel === 'activity') {
|
|
@@ -758,7 +898,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
758
898
|
// Same body as every sibling worker's, so the lane bubbles, the
|
|
759
899
|
// `subagent_status` broadcast and the UI's grouping behave identically
|
|
760
900
|
// whichever provider produced the run.
|
|
761
|
-
await
|
|
901
|
+
await postLive('/api/subagent-run', {
|
|
762
902
|
messageId: message.id,
|
|
763
903
|
conversationId: message.conversationId,
|
|
764
904
|
subagentRunId: payload.subagentRunId,
|
|
@@ -766,7 +906,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
766
906
|
...(payload.displayName ? { displayName: payload.displayName } : {}),
|
|
767
907
|
status: payload.status,
|
|
768
908
|
...attemptFields(message),
|
|
769
|
-
}
|
|
909
|
+
}, { droppable: false });
|
|
770
910
|
}
|
|
771
911
|
}
|
|
772
912
|
|
|
@@ -882,14 +1022,14 @@ export function createCopilotSdkSessionRunner({
|
|
|
882
1022
|
}
|
|
883
1023
|
|
|
884
1024
|
async function publishFinalStream(message, text) {
|
|
885
|
-
await
|
|
1025
|
+
await postLive('/api/stream', {
|
|
886
1026
|
messageId: message.id,
|
|
887
1027
|
conversationId: message.conversationId,
|
|
888
1028
|
mode: message.relayMode || 'agent',
|
|
889
1029
|
text: String(text || ''),
|
|
890
1030
|
done: true,
|
|
891
1031
|
...attemptFields(message),
|
|
892
|
-
}
|
|
1032
|
+
}, { droppable: false });
|
|
893
1033
|
}
|
|
894
1034
|
|
|
895
1035
|
/**
|
|
@@ -906,48 +1046,95 @@ export function createCopilotSdkSessionRunner({
|
|
|
906
1046
|
*/
|
|
907
1047
|
async function publishResponse(message, {
|
|
908
1048
|
text, model, terminalError = null, modelOrigin, kind = null, consumedSteerIds = null,
|
|
909
|
-
requeueOnFailure = true,
|
|
1049
|
+
requeueOnFailure = true, partialText = '',
|
|
910
1050
|
}) {
|
|
911
1051
|
const consumed = Array.isArray(consumedSteerIds)
|
|
912
1052
|
? consumedSteerIds
|
|
913
1053
|
.map((entry) => ({ id: String(entry?.id || '').trim(), attemptId: entry?.attemptId || null }))
|
|
914
1054
|
.filter((entry) => entry.id)
|
|
915
1055
|
: [];
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
})
|
|
929
|
-
|
|
930
|
-
}
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
1056
|
+
const body = {
|
|
1057
|
+
messageId: message.id,
|
|
1058
|
+
conversationId: message.conversationId,
|
|
1059
|
+
text: String(text || ''),
|
|
1060
|
+
model: model || null,
|
|
1061
|
+
modelOrigin: modelOrigin
|
|
1062
|
+
|| (String(message?.model || '').trim().toLowerCase() === 'auto' ? 'auto' : 'manual'),
|
|
1063
|
+
...(terminalError ? { terminalError } : {}),
|
|
1064
|
+
// What the turn had written before it failed; the relay keeps it
|
|
1065
|
+
// above the failure note.
|
|
1066
|
+
...(terminalError && String(partialText || '').trim() ? { partialText: String(partialText).trim() } : {}),
|
|
1067
|
+
...(kind ? { kind } : {}),
|
|
1068
|
+
...(consumed.length ? { consumedSteerIds: consumed } : {}),
|
|
1069
|
+
...attemptFields(message),
|
|
1070
|
+
};
|
|
1071
|
+
// A row's own reply is offered again while the relay cannot be reached:
|
|
1072
|
+
// it exists once, and the route answers a repeat of one it already saved
|
|
1073
|
+
// with `already_done`. Settle markers carry their own ladder
|
|
1074
|
+
// (publishSettleMarker), and a Stop's note is not worth a wait.
|
|
1075
|
+
const patient = !kind && requeueOnFailure;
|
|
1076
|
+
const requestLimit = () => (responseRequestTimeoutMs > 0 ? { signal: AbortSignal.timeout(responseRequestTimeoutMs) } : {});
|
|
1077
|
+
const startedAt = Date.now();
|
|
1078
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
1079
|
+
try {
|
|
1080
|
+
await api('POST', '/api/response', body, requestLimit());
|
|
1081
|
+
if (attempt) dbg('response saved after the relay came back', message.id, `attempt ${attempt + 1}`);
|
|
1082
|
+
return 'published';
|
|
1083
|
+
} catch (error) {
|
|
1084
|
+
// A stale_attempt 409 means the row already moved on to another attempt
|
|
1085
|
+
// (requeued and re-delivered); the requeue would 409 the same way, and
|
|
1086
|
+
// the newer attempt owes the row its answer — drop this one.
|
|
1087
|
+
if (isStaleAttemptError(error)) {
|
|
1088
|
+
dbg('response refused as stale_attempt; dropping', message.id);
|
|
1089
|
+
return 'stale';
|
|
1090
|
+
}
|
|
1091
|
+
if (patient && !disposed && isTransientRelayError(error) && Date.now() - startedAt < responseRetryWindowMs) {
|
|
1092
|
+
if (!attempt) dbg('the relay did not take the response; trying again', message.id, error?.message || String(error));
|
|
1093
|
+
await sleep(settleRetryDelaysMs[Math.min(attempt, settleRetryDelaysMs.length - 1)] ?? 8_000);
|
|
1094
|
+
continue;
|
|
1095
|
+
}
|
|
1096
|
+
dbg('response could not be saved', message.id, error?.message || String(error));
|
|
1097
|
+
if (kind || !requeueOnFailure) return 'failed';
|
|
1098
|
+
const requeued = await api('POST', '/api/requeue', { messageId: message.id, ...attemptFields(message) }, requestLimit())
|
|
1099
|
+
.then(() => true, () => false);
|
|
1100
|
+
return requeued ? 'requeued' : 'failed';
|
|
937
1101
|
}
|
|
938
|
-
if (kind || !requeueOnFailure) return 'failed';
|
|
939
|
-
const requeued = await api('POST', '/api/requeue', { messageId: message.id, ...attemptFields(message) })
|
|
940
|
-
.then(() => true, () => false);
|
|
941
|
-
return requeued ? 'requeued' : 'failed';
|
|
942
1102
|
}
|
|
943
1103
|
}
|
|
944
1104
|
|
|
945
|
-
|
|
1105
|
+
/**
|
|
1106
|
+
* How many commands the runtime of a failed turn had running when its
|
|
1107
|
+
* teardown began, as far as it was known at that moment: the note does not
|
|
1108
|
+
* wait for a process list. On Linux the list is read at once; on Windows
|
|
1109
|
+
* it takes a PowerShell start, so the note there knows what the last read
|
|
1110
|
+
* knew. 0 where nothing is known or the runtime was kept.
|
|
1111
|
+
*/
|
|
1112
|
+
function commandsLeftRunning(turn) {
|
|
1113
|
+
return Number(turn?.commandsLeft) || 0;
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
/**
|
|
1117
|
+
* `toolCalls` is how many tool calls the failed turn had started. They ran,
|
|
1118
|
+
* and what they changed stays changed, so the note says so: a failure that
|
|
1119
|
+
* reads as if nothing happened sends the user back to the start of work
|
|
1120
|
+
* that is mostly done.
|
|
1121
|
+
*/
|
|
1122
|
+
function terminalErrorRecord(message, classified, { toolCalls = 0, commandsLeft = 0 } = {}) {
|
|
1123
|
+
const worked = toolCalls > 0;
|
|
1124
|
+
const work = worked
|
|
1125
|
+
? ` Before that the agent made ${toolCalls} tool call${toolCalls === 1 ? '' : 's'}; what they changed is still in place.`
|
|
1126
|
+
: '';
|
|
1127
|
+
// `commandsLeft` is how many commands were running when the turn failed.
|
|
1128
|
+
// They are stopped with the runtime, and a command that was cut short is
|
|
1129
|
+
// something the user has to know whatever the failure was.
|
|
1130
|
+
const stopping = describeCommandsBeingStopped(commandsLeft);
|
|
1131
|
+
const guidance = worked && classified.guidance ? COPILOT_TURN_CONTINUE_GUIDANCE : classified.guidance;
|
|
946
1132
|
return {
|
|
947
1133
|
kind: 'copilot-turn-failed',
|
|
948
1134
|
code: classified.code,
|
|
949
1135
|
stableCode: classified.stableCode,
|
|
950
|
-
message: classified.text
|
|
1136
|
+
message: `${classified.text}${classified.guidance ? work : ''}${stopping ? ` ${stopping}` : ''}`,
|
|
1137
|
+
...(guidance ? { guidance } : {}),
|
|
951
1138
|
failedAt: new Date().toISOString(),
|
|
952
1139
|
queueMessageId: String(message.id || '') || null,
|
|
953
1140
|
};
|
|
@@ -1196,6 +1383,9 @@ export function createCopilotSdkSessionRunner({
|
|
|
1196
1383
|
// days ago.
|
|
1197
1384
|
if (replayGate.isReplay(event)) {
|
|
1198
1385
|
dbg('dropping a replayed event', String(event?.type || ''), String(event?.timestamp || ''));
|
|
1386
|
+
// History, but a sign of life all the same: a long resume replays for
|
|
1387
|
+
// as long as it takes, and the turn waiting on it is not stalled.
|
|
1388
|
+
activeTurn?.liveness?.traffic('replay');
|
|
1199
1389
|
return;
|
|
1200
1390
|
}
|
|
1201
1391
|
touch();
|
|
@@ -1235,6 +1425,10 @@ export function createCopilotSdkSessionRunner({
|
|
|
1235
1425
|
if (!isContinuationOpeningEvent(event)) return;
|
|
1236
1426
|
turn = openContinuationTurn();
|
|
1237
1427
|
}
|
|
1428
|
+
// Liveness is noted on RECEIPT. The dispatch chain below awaits relay
|
|
1429
|
+
// POSTs, and a watchdog fed from there measures the relay, not the
|
|
1430
|
+
// runtime: one slow POST would fail a turn whose runtime never paused.
|
|
1431
|
+
observeLiveness(turn, event);
|
|
1238
1432
|
// The owner is captured HERE, synchronously, but settlement happens later
|
|
1239
1433
|
// on the dispatch chain — so an event can be captured to a turn that is
|
|
1240
1434
|
// settled by the time its link runs. `handleTurnEvent`'s settled guard
|
|
@@ -1247,6 +1441,57 @@ export function createCopilotSdkSessionRunner({
|
|
|
1247
1441
|
.catch((error) => { dbg('event dispatch failed', error?.message || String(error)); });
|
|
1248
1442
|
}
|
|
1249
1443
|
|
|
1444
|
+
/** A stage of the bring-up finished: the turn waiting on it is not stalled. */
|
|
1445
|
+
function liveTurnTraffic(what) {
|
|
1446
|
+
const turn = activeTurn;
|
|
1447
|
+
if (turn && !turn.settled) turn.liveness.traffic(what);
|
|
1448
|
+
}
|
|
1449
|
+
|
|
1450
|
+
/**
|
|
1451
|
+
* What the runtime is busy with, for the stall watchdog: a tool or a model
|
|
1452
|
+
* request that has started and not finished, per agent (a subagent's events
|
|
1453
|
+
* carry its `agentId` on the envelope). `model.call_*` is what runtime
|
|
1454
|
+
* 1.0.88 emits around a request; `assistant.turn_start`/`turn_end` bracket
|
|
1455
|
+
* the same span on runtimes that do not.
|
|
1456
|
+
*/
|
|
1457
|
+
function observeLiveness(turn, event) {
|
|
1458
|
+
const liveness = turn?.liveness;
|
|
1459
|
+
if (!liveness || turn.settled) return;
|
|
1460
|
+
const type = String(event?.type || '');
|
|
1461
|
+
const data = event?.data && typeof event.data === 'object' ? event.data : {};
|
|
1462
|
+
const agent = String(event?.agentId || '').trim() || 'root';
|
|
1463
|
+
liveness.traffic(type);
|
|
1464
|
+
switch (type) {
|
|
1465
|
+
case 'tool.execution_start':
|
|
1466
|
+
liveness.begin('tool', `${agent}:${data.toolCallId || ''}`, data.toolName);
|
|
1467
|
+
break;
|
|
1468
|
+
case 'tool.execution_complete':
|
|
1469
|
+
liveness.end('tool', `${agent}:${data.toolCallId || ''}`);
|
|
1470
|
+
break;
|
|
1471
|
+
case 'assistant.turn_start':
|
|
1472
|
+
case 'model.call_start':
|
|
1473
|
+
liveness.begin('model', agent);
|
|
1474
|
+
break;
|
|
1475
|
+
case 'assistant.message':
|
|
1476
|
+
case 'assistant.turn_end':
|
|
1477
|
+
case 'model.call_finished':
|
|
1478
|
+
case 'model.call_failure':
|
|
1479
|
+
liveness.end('model', agent);
|
|
1480
|
+
break;
|
|
1481
|
+
case 'subagent.completed':
|
|
1482
|
+
case 'subagent.failed':
|
|
1483
|
+
// Whatever the agent had open ended with it.
|
|
1484
|
+
liveness.endAgent(agent);
|
|
1485
|
+
break;
|
|
1486
|
+
default:
|
|
1487
|
+
break;
|
|
1488
|
+
}
|
|
1489
|
+
// A tool of the main agent that starts or ends moves the steering gate
|
|
1490
|
+
// (see mainToolsInFlight). Told on receipt, like the liveness itself: the
|
|
1491
|
+
// relay must stop delivering before the next message is on its way.
|
|
1492
|
+
if (agent === 'root' && type.startsWith('tool.execution_')) syncDeliveryReadiness();
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1250
1495
|
/**
|
|
1251
1496
|
* Keep the detached-shell set current and note when one settles.
|
|
1252
1497
|
*
|
|
@@ -1417,7 +1662,11 @@ export function createCopilotSdkSessionRunner({
|
|
|
1417
1662
|
if (!sessionRpc.tasks || taskRefreshTimer) return;
|
|
1418
1663
|
taskRefreshTimer = setTimeout(() => {
|
|
1419
1664
|
taskRefreshTimer = null;
|
|
1420
|
-
|
|
1665
|
+
taskRefreshQueued += 1;
|
|
1666
|
+
taskRefreshChain = taskRefreshChain
|
|
1667
|
+
.then(() => refreshBackgroundTasks())
|
|
1668
|
+
.catch(() => {})
|
|
1669
|
+
.finally(() => { taskRefreshQueued -= 1; });
|
|
1421
1670
|
}, taskRefreshMs);
|
|
1422
1671
|
taskRefreshTimer.unref?.();
|
|
1423
1672
|
}
|
|
@@ -1434,13 +1683,18 @@ export function createCopilotSdkSessionRunner({
|
|
|
1434
1683
|
if (!target || !sessionRpc.tasks) return;
|
|
1435
1684
|
lastTaskRefreshAt = Date.now();
|
|
1436
1685
|
let listed;
|
|
1686
|
+
taskRefreshInFlight = true;
|
|
1437
1687
|
try {
|
|
1438
|
-
listed = await target.rpc.tasks.list();
|
|
1688
|
+
listed = await askRuntime('tasks.list', () => target.rpc.tasks.list());
|
|
1439
1689
|
} catch (error) {
|
|
1440
1690
|
dbg('tasks.list failed', error?.message || String(error));
|
|
1691
|
+
if (session === target) taskRefreshFailures += 1;
|
|
1441
1692
|
return;
|
|
1693
|
+
} finally {
|
|
1694
|
+
taskRefreshInFlight = false;
|
|
1442
1695
|
}
|
|
1443
1696
|
if (session !== target) return;
|
|
1697
|
+
taskRefreshFailures = 0;
|
|
1444
1698
|
const next = new Map();
|
|
1445
1699
|
for (const task of Array.isArray(listed?.tasks) ? listed.tasks : []) {
|
|
1446
1700
|
const id = String(task?.id || '').trim();
|
|
@@ -1462,7 +1716,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
1462
1716
|
if (typeof target.rpc.tasks.getProgress === 'function') {
|
|
1463
1717
|
for (const task of next.values()) {
|
|
1464
1718
|
try {
|
|
1465
|
-
const progress = (await target.rpc.tasks.getProgress({ id: task.id }))?.progress || null;
|
|
1719
|
+
const progress = (await askRuntime('tasks.getProgress', () => target.rpc.tasks.getProgress({ id: task.id })))?.progress || null;
|
|
1466
1720
|
task.progressSummary = summarizeTaskProgress(progress);
|
|
1467
1721
|
} catch { /* a card without a progress line */ }
|
|
1468
1722
|
}
|
|
@@ -1586,7 +1840,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
1586
1840
|
const target = session;
|
|
1587
1841
|
if (!target || !sessionRpc.queue) return;
|
|
1588
1842
|
try {
|
|
1589
|
-
const snapshot = await target.rpc.queue.pendingItems();
|
|
1843
|
+
const snapshot = await askRuntime('queue.pendingItems', () => target.rpc.queue.pendingItems());
|
|
1590
1844
|
const next = new Map();
|
|
1591
1845
|
for (const item of Array.isArray(snapshot?.items) ? snapshot.items : []) {
|
|
1592
1846
|
const runtimeId = String(item?.messageId || '').trim();
|
|
@@ -1710,6 +1964,9 @@ export function createCopilotSdkSessionRunner({
|
|
|
1710
1964
|
turn = openContinuationTurn();
|
|
1711
1965
|
}
|
|
1712
1966
|
}
|
|
1967
|
+
// Noted at receipt on the turn that owned the stream then; the turn that
|
|
1968
|
+
// takes the event over has to learn from it what is in flight.
|
|
1969
|
+
observeLiveness(turn, event);
|
|
1713
1970
|
await handleTurnEvent(turn, event);
|
|
1714
1971
|
}
|
|
1715
1972
|
|
|
@@ -1718,7 +1975,11 @@ export function createCopilotSdkSessionRunner({
|
|
|
1718
1975
|
await redispatchSettledEvent(event);
|
|
1719
1976
|
return;
|
|
1720
1977
|
}
|
|
1721
|
-
|
|
1978
|
+
if (String(event?.type || '') === 'tool.execution_start') {
|
|
1979
|
+
turn.toolCalls += 1;
|
|
1980
|
+
// A command may be starting: its process is read once it is there.
|
|
1981
|
+
runtimeTree?.commandStarting();
|
|
1982
|
+
}
|
|
1722
1983
|
// Root work ends on the runtime's own terminator, so a question-only
|
|
1723
1984
|
// continuation it joins no longer closes itself (whileAwaitingHuman).
|
|
1724
1985
|
if (isContinuationOpeningEvent(event)) turn.rootWork = true;
|
|
@@ -1876,6 +2137,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
1876
2137
|
}
|
|
1877
2138
|
|
|
1878
2139
|
function buildSessionConfig(model, reasoningEffort = null) {
|
|
2140
|
+
const generation = runtimeGeneration;
|
|
1879
2141
|
// Built once per session, not per request. Reads the mode off the LIVE turn
|
|
1880
2142
|
// rather than closing over the one the session was built with, because one
|
|
1881
2143
|
// session serves every turn and the user can switch modes between them.
|
|
@@ -1936,7 +2198,10 @@ export function createCopilotSdkSessionRunner({
|
|
|
1936
2198
|
clientName,
|
|
1937
2199
|
// The runtime's own defaults, pinned. See DEFAULT_INFINITE_SESSION_CONFIG.
|
|
1938
2200
|
...(infiniteSessionConfig ? { infiniteSessions: { ...infiniteSessionConfig } } : {}),
|
|
1939
|
-
|
|
2201
|
+
// Events of a runtime that is being stopped are not this session's any
|
|
2202
|
+
// more: the SDK keeps delivering them until the detach is answered, and
|
|
2203
|
+
// by then the next delivery may already be waiting as the active turn.
|
|
2204
|
+
onEvent: (event) => { if (generation === runtimeGeneration) routeEvent(event); },
|
|
1940
2205
|
// The relay's own tools (`relaySessionTools`); omitted when there are none.
|
|
1941
2206
|
...(tools.length ? { tools } : {}),
|
|
1942
2207
|
// Permission policy follows the conversation's relay mode: agent and
|
|
@@ -2077,9 +2342,13 @@ export function createCopilotSdkSessionRunner({
|
|
|
2077
2342
|
dbg('copilot runtime exited', detail);
|
|
2078
2343
|
const turn = activeTurn;
|
|
2079
2344
|
if (turn && !turn.settled) {
|
|
2080
|
-
//
|
|
2081
|
-
//
|
|
2082
|
-
|
|
2345
|
+
// The turn's `fail` drops the dead handles, whatever kind of turn it
|
|
2346
|
+
// is, and the drive path publishes the failure record onto every row
|
|
2347
|
+
// the turn owns.
|
|
2348
|
+
turn.fail(Object.assign(
|
|
2349
|
+
new Error(`the Copilot runtime exited before the turn completed (${detail})`),
|
|
2350
|
+
{ runtimeGone: true },
|
|
2351
|
+
));
|
|
2083
2352
|
return;
|
|
2084
2353
|
}
|
|
2085
2354
|
// No turn to fail: drop the dead handles so the next delivery rebuilds
|
|
@@ -2087,7 +2356,43 @@ export function createCopilotSdkSessionRunner({
|
|
|
2087
2356
|
void stopRuntime('runtime-exit').catch(() => {});
|
|
2088
2357
|
}
|
|
2089
2358
|
|
|
2359
|
+
/**
|
|
2360
|
+
* Start watching the commands of a runtime that has just started. Advisory:
|
|
2361
|
+
* a watch that cannot be set up costs the clean-up after a killed runtime,
|
|
2362
|
+
* never the turn.
|
|
2363
|
+
*/
|
|
2364
|
+
function watchRuntimeTree(startedClient) {
|
|
2365
|
+
try {
|
|
2366
|
+
return watchRuntimeTreeImpl({
|
|
2367
|
+
client: startedClient,
|
|
2368
|
+
// Commands run inside a turn, or as background work between turns. A
|
|
2369
|
+
// runtime with neither is not read.
|
|
2370
|
+
isBusy: () => !disposed && (
|
|
2371
|
+
!!activeTurn || publishingTurns.size > 0 || backgroundShells.size() > 0 || backgroundTasks.size > 0
|
|
2372
|
+
),
|
|
2373
|
+
dbg,
|
|
2374
|
+
}) || null;
|
|
2375
|
+
} catch (error) {
|
|
2376
|
+
dbg('the copilot runtime process tree is not watched', error?.message || String(error));
|
|
2377
|
+
return null;
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
|
|
2381
|
+
/**
|
|
2382
|
+
* After the runtime is gone: stop what it left running, where it did not
|
|
2383
|
+
* end in good order (see `runtimeStopLeavesCommands`). Never rejects.
|
|
2384
|
+
*/
|
|
2385
|
+
async function releaseRuntimeTree(tree, reason) {
|
|
2386
|
+
if (!tree) return;
|
|
2387
|
+
if (!runtimeStopLeavesCommands(reason)) {
|
|
2388
|
+
tree.close();
|
|
2389
|
+
return;
|
|
2390
|
+
}
|
|
2391
|
+
await tree.stop(reason);
|
|
2392
|
+
}
|
|
2393
|
+
|
|
2090
2394
|
async function ensureClient() {
|
|
2395
|
+
while (runtimeStopping) await runtimeStopping;
|
|
2091
2396
|
if (client) return client;
|
|
2092
2397
|
if (starting) return starting;
|
|
2093
2398
|
starting = (async () => {
|
|
@@ -2095,6 +2400,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
2095
2400
|
const started = await startClientImpl({ paths: sdkPaths, cwd, clientName, logLevel, dbg });
|
|
2096
2401
|
client = started.client;
|
|
2097
2402
|
detachRuntimeExit = observeRuntimeExit(client, handleRuntimeExit);
|
|
2403
|
+
runtimeTree = watchRuntimeTree(client);
|
|
2098
2404
|
// Diagnostics only, and deliberately not awaited — see startCopilotClient.
|
|
2099
2405
|
Promise.resolve(started.versionReady)
|
|
2100
2406
|
.then((info) => {
|
|
@@ -2166,14 +2472,43 @@ export function createCopilotSdkSessionRunner({
|
|
|
2166
2472
|
modelSwitch.reset();
|
|
2167
2473
|
// Disconnect first so the runtime is not holding two handles on one
|
|
2168
2474
|
// session id while the resume runs.
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2475
|
+
const detached = await withTimeout(
|
|
2476
|
+
Promise.resolve().then(() => closing?.disconnect?.()).then(() => true, (error) => {
|
|
2477
|
+
dbg('session disconnect before model switch failed', error?.message || String(error));
|
|
2478
|
+
return true;
|
|
2479
|
+
}),
|
|
2480
|
+
runtimeDetachTimeoutMs,
|
|
2481
|
+
false,
|
|
2482
|
+
);
|
|
2483
|
+
if (!detached) dbg(`session disconnect before model switch did not answer within ${runtimeDetachTimeoutMs}ms`);
|
|
2484
|
+
return openSessionOnce(model, effort);
|
|
2173
2485
|
}
|
|
2174
2486
|
|
|
2487
|
+
/**
|
|
2488
|
+
* One session open at a time. A turn can end before its bring-up does (the
|
|
2489
|
+
* stall watchdog, a Stop), and the open it started goes on without it; the
|
|
2490
|
+
* next turn must wait for that one rather than issue a second resume of the
|
|
2491
|
+
* same session id on the same client.
|
|
2492
|
+
*/
|
|
2175
2493
|
async function ensureSession(model, effort) {
|
|
2494
|
+
while (opening) await opening.catch(() => {});
|
|
2495
|
+
if (session) return applySelection(model, effort);
|
|
2496
|
+
return openSessionOnce(model, effort);
|
|
2497
|
+
}
|
|
2498
|
+
|
|
2499
|
+
async function openSessionOnce(model, effort) {
|
|
2500
|
+
const open = openFreshSession(model, effort);
|
|
2501
|
+
opening = open;
|
|
2502
|
+
try {
|
|
2503
|
+
return await open;
|
|
2504
|
+
} finally {
|
|
2505
|
+
if (opening === open) opening = null;
|
|
2506
|
+
}
|
|
2507
|
+
}
|
|
2508
|
+
|
|
2509
|
+
async function openFreshSession(model, effort) {
|
|
2176
2510
|
await ensureClient();
|
|
2511
|
+
liveTurnTraffic('client-ready');
|
|
2177
2512
|
if (!session) {
|
|
2178
2513
|
// The tool list is part of the session: wait for the worker's first
|
|
2179
2514
|
// registration answer (bounded, and it fails closed).
|
|
@@ -2186,8 +2521,16 @@ export function createCopilotSdkSessionRunner({
|
|
|
2186
2521
|
// deliberately not cached: once `createSession` succeeds, the state
|
|
2187
2522
|
// exists and the NEXT reconnect must resume it.
|
|
2188
2523
|
let resumed = false;
|
|
2524
|
+
const generation = runtimeGeneration;
|
|
2525
|
+
// The turn this session was being opened for can end first (the stall
|
|
2526
|
+
// watchdog, a Stop), and its failure stops the runtime.
|
|
2527
|
+
const adopt = (opened) => {
|
|
2528
|
+
if (generation === runtimeGeneration) return opened;
|
|
2529
|
+
Promise.resolve().then(() => opened?.disconnect?.()).catch(() => {});
|
|
2530
|
+
throw new Error('the Copilot runtime was stopped while its session was opening');
|
|
2531
|
+
};
|
|
2189
2532
|
try {
|
|
2190
|
-
session = await openSession((cfg) => client.resumeSession(sdkSessionId, cfg), config);
|
|
2533
|
+
session = adopt(await openSession((cfg) => client.resumeSession(sdkSessionId, cfg), config));
|
|
2191
2534
|
resumed = true;
|
|
2192
2535
|
dbg(`resumed copilot session ${sdkSessionId.slice(0, 8)}`);
|
|
2193
2536
|
} catch (error) {
|
|
@@ -2196,14 +2539,15 @@ export function createCopilotSdkSessionRunner({
|
|
|
2196
2539
|
// transient, and starting fresh over live state would silently throw
|
|
2197
2540
|
// the conversation's whole history away — so it fails the turn
|
|
2198
2541
|
// instead, which is retryable and loses nothing.
|
|
2199
|
-
if (!isSessionNotFoundError(error)) {
|
|
2542
|
+
if (!isSessionNotFoundError(error) || generation !== runtimeGeneration) {
|
|
2200
2543
|
dbg('copilot session resume failed transiently', error?.message || String(error));
|
|
2201
2544
|
throw error;
|
|
2202
2545
|
}
|
|
2203
2546
|
dbg('copilot session state not found, creating', error?.message || String(error));
|
|
2204
|
-
session = await openSession((cfg) => client.createSession(cfg), config);
|
|
2547
|
+
session = adopt(await openSession((cfg) => client.createSession(cfg), config));
|
|
2205
2548
|
dbg(`created copilot session ${sdkSessionId.slice(0, 8)}`);
|
|
2206
2549
|
}
|
|
2550
|
+
liveTurnTraffic('session-open');
|
|
2207
2551
|
// Which experimental surfaces THIS runtime has (it auto-updates under
|
|
2208
2552
|
// the relay): decides un-steer and the targeted Stop for the session.
|
|
2209
2553
|
probeSessionRpc(session);
|
|
@@ -2243,6 +2587,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
2243
2587
|
}
|
|
2244
2588
|
|
|
2245
2589
|
async function stopRuntime(reason) {
|
|
2590
|
+
runtimeGeneration += 1;
|
|
2246
2591
|
stopLifecycleTimer();
|
|
2247
2592
|
// The next session is built from a fresh registration answer.
|
|
2248
2593
|
void remoteRelayGate.refresh();
|
|
@@ -2250,8 +2595,10 @@ export function createCopilotSdkSessionRunner({
|
|
|
2250
2595
|
detachRuntimeExit = () => {};
|
|
2251
2596
|
const closingSession = session;
|
|
2252
2597
|
const closingClient = client;
|
|
2598
|
+
const closingTree = runtimeTree;
|
|
2253
2599
|
session = null;
|
|
2254
2600
|
client = null;
|
|
2601
|
+
runtimeTree = null;
|
|
2255
2602
|
modelSwitch.reset();
|
|
2256
2603
|
// Detached shells are children of the runtime process, so stopping it ends
|
|
2257
2604
|
// them: the tracked set is state about a process that no longer exists and
|
|
@@ -2281,17 +2628,86 @@ export function createCopilotSdkSessionRunner({
|
|
|
2281
2628
|
waitingSteers = 0;
|
|
2282
2629
|
queuedItemCount = 0;
|
|
2283
2630
|
steerSendsInFlight = 0;
|
|
2631
|
+
taskRefreshFailures = 0;
|
|
2632
|
+
taskRefreshInFlight = false;
|
|
2633
|
+
// A removal this runtime never answered must not hold the next runtime's.
|
|
2634
|
+
unsteerChain = Promise.resolve();
|
|
2284
2635
|
sessionRpc = { queue: false, interruptMainTurn: false, tasks: false };
|
|
2285
2636
|
askUserOverrideRejected = false;
|
|
2286
2637
|
syncDeliveryReadiness();
|
|
2287
|
-
if (!closingSession && !closingClient)
|
|
2638
|
+
if (!closingSession && !closingClient) {
|
|
2639
|
+
// Nothing of its own to close, but a caller that asked for a stopped
|
|
2640
|
+
// runtime (the worker's shutdown) gets one.
|
|
2641
|
+
closingTree?.close();
|
|
2642
|
+
if (runtimeStopping) await runtimeStopping;
|
|
2643
|
+
return;
|
|
2644
|
+
}
|
|
2288
2645
|
dbg(`stopping the copilot runtime (${reason})`);
|
|
2289
|
-
|
|
2290
|
-
|
|
2646
|
+
// Read before the runtime is told to stop, in this very tick: what is
|
|
2647
|
+
// running now is what the failure note names (`turn.fail`), and a runtime
|
|
2648
|
+
// that has to be killed leaves it behind.
|
|
2649
|
+
const leavesCommands = !!closingTree && runtimeStopLeavesCommands(reason);
|
|
2650
|
+
if (leavesCommands) void closingTree.snapshot();
|
|
2651
|
+
const previous = runtimeStopping;
|
|
2652
|
+
const closing = (async () => {
|
|
2653
|
+
if (previous) await previous;
|
|
2654
|
+
// A runtime that has already ended stops nothing any more, and the
|
|
2655
|
+
// teardown below still waits out its windows on a connection that has
|
|
2656
|
+
// nobody behind it (10 s, measured): its commands are stopped at once.
|
|
2657
|
+
const early = leavesCommands
|
|
2658
|
+
? closingTree.runtimeEnded().then((ended) => (ended ? closingTree.stop(reason) : null))
|
|
2659
|
+
: null;
|
|
2660
|
+
await closeRuntime(closingSession, closingClient, closingTree);
|
|
2661
|
+
await early;
|
|
2662
|
+
// Part of the teardown the next runtime waits for: a command of the old
|
|
2663
|
+
// one must not still be writing into the workspace the new one works in.
|
|
2664
|
+
await releaseRuntimeTree(closingTree, reason);
|
|
2665
|
+
})();
|
|
2666
|
+
runtimeStopping = closing;
|
|
2667
|
+
try {
|
|
2668
|
+
await closing;
|
|
2669
|
+
} finally {
|
|
2670
|
+
if (runtimeStopping === closing) runtimeStopping = null;
|
|
2671
|
+
}
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2674
|
+
/**
|
|
2675
|
+
* Close a runtime whose handles were already dropped. Every step is bounded:
|
|
2676
|
+
* the callers are the failure paths, where the runtime is the thing that
|
|
2677
|
+
* stopped answering. Never rejects.
|
|
2678
|
+
*/
|
|
2679
|
+
async function closeRuntime(closingSession, closingClient, closingTree = null) {
|
|
2680
|
+
const startedAt = Date.now();
|
|
2681
|
+
const bounded = (work, ms) => withTimeout(
|
|
2682
|
+
Promise.resolve().then(work).then((value) => {
|
|
2683
|
+
// `client.stop()` reports what it could not close instead of throwing.
|
|
2684
|
+
for (const error of Array.isArray(value) ? value : []) dbg('runtime stop reported', error?.message || String(error));
|
|
2685
|
+
return 'done';
|
|
2686
|
+
}, (error) => error),
|
|
2687
|
+
ms,
|
|
2688
|
+
'timeout',
|
|
2689
|
+
);
|
|
2690
|
+
if (closingSession?.disconnect) {
|
|
2691
|
+
const detached = await bounded(() => closingSession.disconnect(), runtimeDetachTimeoutMs);
|
|
2692
|
+
if (detached === 'timeout') dbg(`session disconnect did not answer within ${runtimeDetachTimeoutMs}ms; stopping the runtime without it`);
|
|
2693
|
+
else if (detached !== 'done') dbg('session disconnect failed', detached?.message || String(detached));
|
|
2291
2694
|
}
|
|
2292
|
-
|
|
2293
|
-
|
|
2695
|
+
if (closingClient?.stop) {
|
|
2696
|
+
const stopped = await bounded(() => closingClient.stop(), runtimeStopTimeoutMs);
|
|
2697
|
+
if (stopped === 'timeout') {
|
|
2698
|
+
dbg(`client stop did not finish within ${runtimeStopTimeoutMs}ms; killing the runtime`);
|
|
2699
|
+
// The last moment the runtime's commands can be told from anybody
|
|
2700
|
+
// else's: a killed runtime leaves them to pid 1.
|
|
2701
|
+
if (closingTree) await closingTree.snapshot();
|
|
2702
|
+
const killed = closingClient.forceStop
|
|
2703
|
+
? await bounded(() => closingClient.forceStop(), runtimeDetachTimeoutMs)
|
|
2704
|
+
: 'done';
|
|
2705
|
+
if (killed !== 'done') dbg('runtime kill did not finish', killed === 'timeout' ? 'timeout' : killed?.message || String(killed));
|
|
2706
|
+
} else if (stopped !== 'done') {
|
|
2707
|
+
dbg('client stop failed', stopped?.message || String(stopped));
|
|
2708
|
+
}
|
|
2294
2709
|
}
|
|
2710
|
+
dbg(`copilot runtime stopped after ${Date.now() - startedAt}ms`);
|
|
2295
2711
|
}
|
|
2296
2712
|
|
|
2297
2713
|
// -------------------------------------------------------------- lifecycle --
|
|
@@ -2335,8 +2751,23 @@ export function createCopilotSdkSessionRunner({
|
|
|
2335
2751
|
// re-read once per lifecycle poll catches state the events miss (a
|
|
2336
2752
|
// shell that exited after a cancel emits no notification).
|
|
2337
2753
|
const capMs = Number(getBackgroundTaskTimeoutMs()) || 0;
|
|
2754
|
+
if (taskRefreshFailures >= MAX_TASK_REFRESH_FAILURES && backgroundTasks.size) {
|
|
2755
|
+
// The runtime has not answered for its tasks for that many reads in a
|
|
2756
|
+
// row: the set is what it was when it last did, and holds nothing.
|
|
2757
|
+
dbg('the task registry cannot be read; its last answer no longer holds the runtime');
|
|
2758
|
+
backgroundTasks = new Map();
|
|
2759
|
+
publishBackgroundTasks({ immediate: true });
|
|
2760
|
+
}
|
|
2338
2761
|
const live = runningBackgroundTasks();
|
|
2339
|
-
if (!live.length)
|
|
2762
|
+
if (!live.length) {
|
|
2763
|
+
// Nothing live as far as the last read that worked. While a read is
|
|
2764
|
+
// due or under way, or reads fail, that is not known: read again, and
|
|
2765
|
+
// hold the runtime meanwhile.
|
|
2766
|
+
if (taskRefreshTimer || taskRefreshQueued > 0 || taskRefreshInFlight) return true;
|
|
2767
|
+
if (!taskRefreshFailures || taskRefreshFailures >= MAX_TASK_REFRESH_FAILURES) return false;
|
|
2768
|
+
if (now - lastTaskRefreshAt >= lifecyclePollMs) scheduleTaskRefresh();
|
|
2769
|
+
return true;
|
|
2770
|
+
}
|
|
2340
2771
|
if (capMs > 0) {
|
|
2341
2772
|
for (const task of live) {
|
|
2342
2773
|
const startedAt = Date.parse(String(task.startedAt || '')) || agentDetails.get(task.id)?.startedAt || 0;
|
|
@@ -2488,6 +2919,19 @@ export function createCopilotSdkSessionRunner({
|
|
|
2488
2919
|
// The row the user pressed Stop on (from the abort control), when known.
|
|
2489
2920
|
abortedRowId: null,
|
|
2490
2921
|
stallTimer: null,
|
|
2922
|
+
stallChecking: false,
|
|
2923
|
+
liveness: createTurnLiveness({ windows: stallWindows }),
|
|
2924
|
+
// The runtime teardown a failure started (see `fail`).
|
|
2925
|
+
teardown: null,
|
|
2926
|
+
// Until when a continuation's registration may still get its row from a
|
|
2927
|
+
// relay it cannot reach (0: it is not in that state).
|
|
2928
|
+
registrationDeadline: 0,
|
|
2929
|
+
// Tool calls this turn started, whichever agent made them. Told in the
|
|
2930
|
+
// failure note when the turn fails.
|
|
2931
|
+
toolCalls: 0,
|
|
2932
|
+
// What the runtime had running when the turn failed (a promise of the
|
|
2933
|
+
// tree watch's read), for the failure note.
|
|
2934
|
+
commandsLeft: null,
|
|
2491
2935
|
planBoardPosted: false,
|
|
2492
2936
|
// Set when a mutating tool was actually approved and run this turn.
|
|
2493
2937
|
acted: false,
|
|
@@ -2596,6 +3040,41 @@ export function createCopilotSdkSessionRunner({
|
|
|
2596
3040
|
if (turn.settled) return;
|
|
2597
3041
|
turn.settled = true;
|
|
2598
3042
|
turn.disarmStall();
|
|
3043
|
+
// A delivered turn that failed (a stall, a runtime death, a setup
|
|
3044
|
+
// failure) leaves a runtime nothing else can use. The teardown drops the
|
|
3045
|
+
// handles in this very tick, before the turn gives up the event stream
|
|
3046
|
+
// and the worker reads as ready again, so the next delivery rebuilds
|
|
3047
|
+
// instead of sending into it. It is awaited only after the failure is
|
|
3048
|
+
// published (driveTurn): a runtime that stopped answering also takes
|
|
3049
|
+
// its time to stop, and the user waited three minutes for a failure
|
|
3050
|
+
// that was already decided (2026-09-27).
|
|
3051
|
+
//
|
|
3052
|
+
// An UNCONFIRMED MODEL SWITCH is the exception: the session is healthy
|
|
3053
|
+
// and merely still on its previous model, and tearing the runtime down
|
|
3054
|
+
// over it would kill any live background work for a selection problem
|
|
3055
|
+
// the user fixes by picking another model.
|
|
3056
|
+
//
|
|
3057
|
+
// A turn the runtime opened by itself keeps its runtime through a
|
|
3058
|
+
// failure, because the background work that opened it may still be
|
|
3059
|
+
// running there — unless the runtime itself is what went: it exited, or
|
|
3060
|
+
// it no longer answers. Those handles served nobody, and the next
|
|
3061
|
+
// message failed on them.
|
|
3062
|
+
const runtimeGone = error?.runtimeGone === true || error?.stall?.unresponsive === true;
|
|
3063
|
+
if ((turn.kind === 'delivered' || runtimeGone) && !isModelSwitchUnconfirmedError(error)) {
|
|
3064
|
+
if (session || client) {
|
|
3065
|
+
const tree = runtimeTree;
|
|
3066
|
+
turn.teardown = stopRuntime(runtimeGone ? 'runtime-gone' : 'turn-failure').catch(() => {});
|
|
3067
|
+
// What the runtime had running, as far as it is known now
|
|
3068
|
+
// (`stopRuntime` has just read the list where that takes no time):
|
|
3069
|
+
// the failure note is published before the teardown ends, and says
|
|
3070
|
+
// what is being stopped.
|
|
3071
|
+
turn.commandsLeft = tree ? tree.commandCount() : 0;
|
|
3072
|
+
} else {
|
|
3073
|
+
// Nothing to stop yet: the runtime was still starting. What it opens
|
|
3074
|
+
// for this turn from here on is let go rather than adopted.
|
|
3075
|
+
runtimeGeneration += 1;
|
|
3076
|
+
}
|
|
3077
|
+
}
|
|
2599
3078
|
beginPublishing(turn);
|
|
2600
3079
|
turn.resolvePrimaryReady();
|
|
2601
3080
|
syncDeliveryReadiness();
|
|
@@ -2606,29 +3085,83 @@ export function createCopilotSdkSessionRunner({
|
|
|
2606
3085
|
clearTimeout(turn.stallTimer);
|
|
2607
3086
|
turn.stallTimer = null;
|
|
2608
3087
|
};
|
|
3088
|
+
// The window restarts: the turn opens, or a hold on it ends.
|
|
2609
3089
|
turn.armStall = () => {
|
|
2610
|
-
if (!(
|
|
2611
|
-
turn.
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
// rather than fail: the card has its own (much longer) timeout, and
|
|
2615
|
-
// failing the row here would settle it while the answer is still coming.
|
|
2616
|
-
// A remote_relay call waiting on another relay is the same kind of
|
|
2617
|
-
// silence; the relay bounds that wait itself.
|
|
2618
|
-
if (pendingHumanRequests > 0 || pendingRelayToolCalls > 0) {
|
|
2619
|
-
turn.armStall();
|
|
2620
|
-
return;
|
|
2621
|
-
}
|
|
2622
|
-
turn.fail(new Error(
|
|
2623
|
-
`copilot worker watchdog: the runtime produced no events for ${Math.round(turnStallTimeoutMs / 1000)}s; `
|
|
2624
|
-
+ 'the row is failed — resend the message to retry',
|
|
2625
|
-
));
|
|
2626
|
-
}, turnStallTimeoutMs);
|
|
2627
|
-
turn.stallTimer.unref?.();
|
|
3090
|
+
if (!(stallWindows.idle > 0) || turn.settled) return;
|
|
3091
|
+
turn.liveness.traffic('armed');
|
|
3092
|
+
// A check in progress schedules the next one itself.
|
|
3093
|
+
if (!turn.stallTimer && !turn.stallChecking) scheduleStallCheck(turn, turn.liveness.check().waitMs);
|
|
2628
3094
|
};
|
|
2629
3095
|
return turn;
|
|
2630
3096
|
}
|
|
2631
3097
|
|
|
3098
|
+
/**
|
|
3099
|
+
* The stall watchdog. One timer per turn, never re-armed by traffic: an
|
|
3100
|
+
* event only moves the turn's clock, and a check that comes early looks
|
|
3101
|
+
* again when the window is due (a model request emits thousands of deltas).
|
|
3102
|
+
*/
|
|
3103
|
+
function scheduleStallCheck(turn, waitMs) {
|
|
3104
|
+
turn.stallTimer = setTimeout(() => {
|
|
3105
|
+
turn.stallTimer = null;
|
|
3106
|
+
turn.stallChecking = true;
|
|
3107
|
+
void checkStall(turn)
|
|
3108
|
+
.catch((error) => { dbg('stall check failed', error?.message || String(error)); })
|
|
3109
|
+
.finally(() => {
|
|
3110
|
+
turn.stallChecking = false;
|
|
3111
|
+
// A check that ended without deciding (it threw) must not be the last.
|
|
3112
|
+
if (!turn.settled && !turn.stallTimer) scheduleStallCheck(turn, turn.liveness.check().waitMs || stallWindows.idle);
|
|
3113
|
+
});
|
|
3114
|
+
}, Math.max(1, waitMs));
|
|
3115
|
+
turn.stallTimer.unref?.();
|
|
3116
|
+
}
|
|
3117
|
+
|
|
3118
|
+
async function checkStall(turn) {
|
|
3119
|
+
if (turn.settled) return;
|
|
3120
|
+
// A human staring at a question card is not a stalled runtime: the card
|
|
3121
|
+
// has its own (much longer) timeout, and failing the row here would
|
|
3122
|
+
// settle it while the answer is still coming. A remote_relay call waiting
|
|
3123
|
+
// on another relay is the same kind of silence; the relay bounds that
|
|
3124
|
+
// wait itself. So is a compaction, bounded by its stale cap, and the wait
|
|
3125
|
+
// for the previous runtime to stop, bounded by the teardown's own limits.
|
|
3126
|
+
if (pendingHumanRequests > 0 || pendingRelayToolCalls > 0 || isCompacting() || runtimeStopping) {
|
|
3127
|
+
turn.liveness.traffic('held');
|
|
3128
|
+
scheduleStallCheck(turn, turn.liveness.check().waitMs);
|
|
3129
|
+
return;
|
|
3130
|
+
}
|
|
3131
|
+
let verdict = turn.liveness.check();
|
|
3132
|
+
let unresponsive = false;
|
|
3133
|
+
if (!verdict.stalled && verdict.quietMs >= stallWindows.idle) {
|
|
3134
|
+
// Quiet for longer than a runtime with nothing to do may be, and kept
|
|
3135
|
+
// only because a tool or a model request is in flight. That is worth
|
|
3136
|
+
// one question: is the runtime still there?
|
|
3137
|
+
const probing = client;
|
|
3138
|
+
const ask = () => probeRuntime(
|
|
3139
|
+
typeof probing?.ping === 'function' ? () => probing.ping('stall-check') : null,
|
|
3140
|
+
stallProbeTimeoutMs,
|
|
3141
|
+
);
|
|
3142
|
+
let answer = await ask();
|
|
3143
|
+
// Asked twice before a turn with minutes left is given up: one ping
|
|
3144
|
+
// that came at a bad moment is not a runtime that is gone.
|
|
3145
|
+
if (answer === 'gone' && !turn.settled) answer = await ask();
|
|
3146
|
+
if (turn.settled) return;
|
|
3147
|
+
verdict = turn.liveness.check();
|
|
3148
|
+
unresponsive = answer === 'gone' && verdict.quietMs >= stallWindows.idle;
|
|
3149
|
+
if (!unresponsive && !verdict.stalled) dbg('turn is quiet, the runtime answers; keeping it', turn.liveness.describe());
|
|
3150
|
+
}
|
|
3151
|
+
if (!verdict.stalled && !unresponsive) {
|
|
3152
|
+
scheduleStallCheck(turn, verdict.waitMs);
|
|
3153
|
+
return;
|
|
3154
|
+
}
|
|
3155
|
+
dbg(`turn stalled${unresponsive ? ' (runtime not answering)' : ''}`, turn.liveness.describe());
|
|
3156
|
+
turn.fail(createTurnStalledError({
|
|
3157
|
+
agentLabel: SETTLE_AGENT_LABEL,
|
|
3158
|
+
quietMs: verdict.quietMs,
|
|
3159
|
+
phase: verdict.phase,
|
|
3160
|
+
tools: turn.liveness.toolsInFlight(),
|
|
3161
|
+
unresponsive,
|
|
3162
|
+
}));
|
|
3163
|
+
}
|
|
3164
|
+
|
|
2632
3165
|
// ----------------------------------------------------- prompt boundaries --
|
|
2633
3166
|
|
|
2634
3167
|
function claimSegment(turn, index, owner) {
|
|
@@ -2754,6 +3287,21 @@ export function createCopilotSdkSessionRunner({
|
|
|
2754
3287
|
.catch((error) => { dbg('silent continuation drop failed', entry.message.id, error?.message || String(error)); });
|
|
2755
3288
|
}
|
|
2756
3289
|
|
|
3290
|
+
/**
|
|
3291
|
+
* A row whose run is over and whose outcome reached the relay neither as a
|
|
3292
|
+
* response nor as a requeue. Releasing it as it is would leave it to the
|
|
3293
|
+
* relay's recovery, which runs the prompt again or fails it as a blocked
|
|
3294
|
+
* replay, whichever it finds. It is handed to the heartbeat instead, to be
|
|
3295
|
+
* failed with a note that says what happened: the run is not repeated.
|
|
3296
|
+
*/
|
|
3297
|
+
function noteResponseOutcome(message, outcome, variant = 'replyLost') {
|
|
3298
|
+
const id = String(message?.id || '').trim();
|
|
3299
|
+
if (outcome !== 'failed' || !id) return;
|
|
3300
|
+
dbg('the outcome could not be saved or requeued; handing the row to the heartbeat', id, variant);
|
|
3301
|
+
claimSettling(message, variant);
|
|
3302
|
+
settleFailedTerminals.set(id, buildSteerSettleFailure(message, { variant, agentLabel: SETTLE_AGENT_LABEL }));
|
|
3303
|
+
}
|
|
3304
|
+
|
|
2757
3305
|
/**
|
|
2758
3306
|
* Decide and publish one row's outcome. Idempotent per entry. A run owner's
|
|
2759
3307
|
* settle also settles the steers folded into its run (they were answered by
|
|
@@ -2782,11 +3330,11 @@ export function createCopilotSdkSessionRunner({
|
|
|
2782
3330
|
await publishPlanBoard(turn, text, 'plan-mode-fallback');
|
|
2783
3331
|
}
|
|
2784
3332
|
await publishFinalStream(entry.message, published);
|
|
2785
|
-
await publishResponse(entry.message, {
|
|
3333
|
+
noteResponseOutcome(entry.message, await publishResponse(entry.message, {
|
|
2786
3334
|
text: published,
|
|
2787
3335
|
model,
|
|
2788
3336
|
consumedSteerIds: folded.map((other) => other.message),
|
|
2789
|
-
});
|
|
3337
|
+
}));
|
|
2790
3338
|
break;
|
|
2791
3339
|
}
|
|
2792
3340
|
case 'aborted-by-user': {
|
|
@@ -2825,23 +3373,38 @@ export function createCopilotSdkSessionRunner({
|
|
|
2825
3373
|
const own = finalTextFor(turn, entry, turn.result);
|
|
2826
3374
|
const text = own ? `${own}\n\n${RUNTIME_INTERRUPTED_NOTE}` : RUNTIME_INTERRUPTED_NOTE;
|
|
2827
3375
|
await publishFinalStream(entry.message, text);
|
|
2828
|
-
|
|
3376
|
+
noteResponseOutcome(
|
|
3377
|
+
entry.message,
|
|
3378
|
+
await publishResponse(entry.message, { text, model, consumedSteerIds: folded.map((other) => other.message) }),
|
|
3379
|
+
);
|
|
2829
3380
|
break;
|
|
2830
3381
|
}
|
|
2831
3382
|
case 'error': {
|
|
2832
3383
|
const { classified } = outcome;
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
3384
|
+
const written = finalTextFor(turn, entry, turn.result);
|
|
3385
|
+
const terminalError = terminalErrorRecord(entry.message, classified, {
|
|
3386
|
+
toolCalls: turn.toolCalls,
|
|
3387
|
+
commandsLeft: commandsLeftRunning(turn),
|
|
3388
|
+
});
|
|
3389
|
+
await publishFinalStream(entry.message, written || entry.state.lastStreamedText);
|
|
3390
|
+
noteResponseOutcome(entry.message, await publishResponse(entry.message, {
|
|
3391
|
+
text: terminalError.message,
|
|
2836
3392
|
model,
|
|
2837
|
-
terminalError
|
|
3393
|
+
terminalError,
|
|
3394
|
+
partialText: written,
|
|
2838
3395
|
consumedSteerIds: folded.map((other) => other.message),
|
|
2839
|
-
});
|
|
3396
|
+
}), 'failureLost');
|
|
2840
3397
|
break;
|
|
2841
3398
|
}
|
|
2842
3399
|
case 'folded':
|
|
2843
3400
|
await publishSettleMarker(entry.message, { text: STEER_FOLDED_TEXT, model, kind: 'folded', variant: 'folded' });
|
|
2844
3401
|
break;
|
|
3402
|
+
case 'failed-with-turn':
|
|
3403
|
+
// The runtime had taken this steer into a run that failed. The
|
|
3404
|
+
// failure is told on that run's reply; this row only says where to
|
|
3405
|
+
// look, and offers Resend like the steer of a stopped turn.
|
|
3406
|
+
await publishSettleMarker(entry.message, { text: STEER_TURN_FAILED_TEXT, model, kind: 'stopped', variant: 'turnFailed' });
|
|
3407
|
+
break;
|
|
2845
3408
|
case 'stopped':
|
|
2846
3409
|
await publishSettleMarker(entry.message, { text: STEER_STOPPED_TEXT, model, kind: 'stopped', variant: 'stopped' });
|
|
2847
3410
|
break;
|
|
@@ -2850,7 +3413,11 @@ export function createCopilotSdkSessionRunner({
|
|
|
2850
3413
|
// it will be answered at the start of the next interaction — a
|
|
2851
3414
|
// requeue would run it twice.
|
|
2852
3415
|
await publishFinalStream(entry.message, STEERED_ROW_MERGED_NOTE);
|
|
2853
|
-
|
|
3416
|
+
noteResponseOutcome(
|
|
3417
|
+
entry.message,
|
|
3418
|
+
await publishResponse(entry.message, { text: STEERED_ROW_MERGED_NOTE, model, requeueOnFailure: false }),
|
|
3419
|
+
'folded',
|
|
3420
|
+
);
|
|
2854
3421
|
break;
|
|
2855
3422
|
case 'cancelled':
|
|
2856
3423
|
case 'discarded':
|
|
@@ -2866,7 +3433,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
2866
3433
|
for (const other of folded) {
|
|
2867
3434
|
const foldOutcome = outcome.type === 'completed' || outcome.type === 'merged-note'
|
|
2868
3435
|
? { type: 'folded' }
|
|
2869
|
-
: outcome.type === 'error' ?
|
|
3436
|
+
: outcome.type === 'error' ? { type: 'failed-with-turn' } : { type: 'stopped' };
|
|
2870
3437
|
await settleEntry(turn, other, foldOutcome);
|
|
2871
3438
|
}
|
|
2872
3439
|
}
|
|
@@ -2879,7 +3446,8 @@ export function createCopilotSdkSessionRunner({
|
|
|
2879
3446
|
const itemId = queuedLane.get(entry.runtimeId);
|
|
2880
3447
|
if (!itemId) continue;
|
|
2881
3448
|
try {
|
|
2882
|
-
|
|
3449
|
+
const target = session;
|
|
3450
|
+
await askRuntime('queue.removeAt', () => target.rpc.queue.removeAt({ id: itemId }));
|
|
2883
3451
|
dbg('removed a stopped prompt from the runtime queue', entry.message.id);
|
|
2884
3452
|
} catch (error) {
|
|
2885
3453
|
dbg('queue.removeAt after Stop failed', entry.message.id, error?.message || String(error));
|
|
@@ -2917,21 +3485,36 @@ export function createCopilotSdkSessionRunner({
|
|
|
2917
3485
|
const pending = turn.entries.filter((entry) => !entry.settled && entry.turn === turn);
|
|
2918
3486
|
if (!pending.length) return;
|
|
2919
3487
|
|
|
2920
|
-
|
|
3488
|
+
// A turn the user stopped ends as stopped, whatever happened to it
|
|
3489
|
+
// afterwards: a runtime that died or went silent under a Stop used to put
|
|
3490
|
+
// the watchdog's failure on every row instead.
|
|
3491
|
+
if (failure && !turn.aborted) {
|
|
2921
3492
|
const classified = classifyCopilotTurnException(failure);
|
|
2922
3493
|
dbg('copilot turn failed', turn.message?.id || '(continuation)', classified.detail);
|
|
3494
|
+
const running = turn.currentOwner || turn.primaryEntry;
|
|
2923
3495
|
for (const entry of pending) {
|
|
2924
3496
|
if (entry.settled) continue;
|
|
2925
|
-
|
|
3497
|
+
// The failure is told once, on the row whose run it ended. A message
|
|
3498
|
+
// pushed into the turn gets the marker, whether the runtime had
|
|
3499
|
+
// folded it in (settled with its run's row) or never started it.
|
|
3500
|
+
const ownsRun = entry === running || entry.role !== 'steer' || (entry.consumed && !entry.foldedInto);
|
|
3501
|
+
// A steer whose `send()` has not come back never reached the runtime
|
|
3502
|
+
// as far as anyone knows. Its send is about to fail on the teardown
|
|
3503
|
+
// and requeues the row from there; a marker now would end a message
|
|
3504
|
+
// that can simply be delivered again.
|
|
3505
|
+
if (!ownsRun && !entry.runtimeId && !entry.consumed) continue;
|
|
3506
|
+
await settleEntry(turn, entry, ownsRun ? { type: 'error', classified } : { type: 'failed-with-turn' });
|
|
2926
3507
|
}
|
|
2927
3508
|
return;
|
|
2928
3509
|
}
|
|
2929
3510
|
if (result?.isError) {
|
|
2930
3511
|
const classified = classifyCopilotSessionError(result.errorData || { message: result.errorMessage });
|
|
2931
3512
|
dbg('turn failed', turn.message?.id || '(continuation)', classified.stableCode);
|
|
3513
|
+
const running = turn.currentOwner || turn.primaryEntry;
|
|
2932
3514
|
for (const entry of pending) {
|
|
2933
3515
|
if (entry.settled) continue;
|
|
2934
|
-
|
|
3516
|
+
const ownsRun = entry === running || entry.role !== 'steer' || (entry.consumed && !entry.foldedInto);
|
|
3517
|
+
await settleEntry(turn, entry, ownsRun ? { type: 'error', classified } : { type: 'failed-with-turn' });
|
|
2935
3518
|
}
|
|
2936
3519
|
return;
|
|
2937
3520
|
}
|
|
@@ -3084,24 +3667,62 @@ export function createCopilotSdkSessionRunner({
|
|
|
3084
3667
|
// and stops waiting on the remote, and the watchdog hold is released.
|
|
3085
3668
|
turn.remoteRelayStop?.abort();
|
|
3086
3669
|
// While `ensureSession` is still connecting there is no session to abort
|
|
3087
|
-
// and
|
|
3088
|
-
// once the session exists
|
|
3089
|
-
|
|
3670
|
+
// and no prompt was sent: the turn settles here, as stopped, and `runTurn`
|
|
3671
|
+
// sends nothing once the session exists. Waiting for the session instead
|
|
3672
|
+
// left the Stop without effect for as long as the runtime took to answer.
|
|
3673
|
+
if (!session) {
|
|
3674
|
+
if (!turn.primarySent) {
|
|
3675
|
+
dbg('turn stopped before its session was ready', turn.message?.id || '');
|
|
3676
|
+
turn.settle();
|
|
3677
|
+
}
|
|
3678
|
+
return;
|
|
3679
|
+
}
|
|
3680
|
+
// Both requests are bounded. A Stop is what the user reaches for when a
|
|
3681
|
+
// turn is stuck, and a runtime that is stuck does not answer it either:
|
|
3682
|
+
// the Stop then did nothing until the stall watchdog failed the turn.
|
|
3683
|
+
const target = session;
|
|
3684
|
+
const unanswered = Symbol('unanswered');
|
|
3685
|
+
const bounded = (request) => withTimeout(Promise.resolve().then(request), runtimeRpcTimeoutMs, unanswered);
|
|
3686
|
+
let answered = true;
|
|
3090
3687
|
if (sessionRpc.interruptMainTurn) {
|
|
3091
3688
|
try {
|
|
3092
|
-
const outcome = await
|
|
3093
|
-
if (outcome
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3689
|
+
const outcome = await bounded(() => target.rpc.interruptMainTurn({ flushQueued: false }));
|
|
3690
|
+
if (outcome !== unanswered) {
|
|
3691
|
+
if (outcome?.interrupted === false) {
|
|
3692
|
+
// Nothing was processing: no aborted idle will come. The turn is
|
|
3693
|
+
// over as far as the runtime is concerned.
|
|
3694
|
+
dbg('interruptMainTurn found no main turn in flight; settling locally', turn.message?.id || '');
|
|
3695
|
+
turn.settle();
|
|
3696
|
+
}
|
|
3697
|
+
return;
|
|
3098
3698
|
}
|
|
3099
|
-
|
|
3699
|
+
answered = false;
|
|
3100
3700
|
} catch (error) {
|
|
3101
3701
|
dbg('interruptMainTurn failed; falling back to abort()', error?.message || String(error));
|
|
3102
3702
|
}
|
|
3103
3703
|
}
|
|
3104
|
-
|
|
3704
|
+
if (answered) {
|
|
3705
|
+
// A rejection is an answer of a kind the turn cannot use either: the
|
|
3706
|
+
// connection is gone (the runtime exited, or a teardown disposed it).
|
|
3707
|
+
// Thrown on from here, it reached the relay as a Stop that failed, and
|
|
3708
|
+
// the row was settled by nobody.
|
|
3709
|
+
answered = (await bounded(async () => { await target.abort?.(); }).catch((error) => {
|
|
3710
|
+
dbg('abort() failed', error?.message || String(error));
|
|
3711
|
+
return unanswered;
|
|
3712
|
+
})) !== unanswered;
|
|
3713
|
+
}
|
|
3714
|
+
if (answered || turn.settled) return;
|
|
3715
|
+
// No aborted idle is coming from a runtime that does not take the
|
|
3716
|
+
// request. The turn ends here, as stopped (finishTurn), and the runtime
|
|
3717
|
+
// goes with it.
|
|
3718
|
+
dbg('the runtime did not answer the Stop; ending the turn without it', turn.message?.id || '');
|
|
3719
|
+
turn.fail(createTurnStalledError({
|
|
3720
|
+
agentLabel: SETTLE_AGENT_LABEL,
|
|
3721
|
+
quietMs: turn.liveness.quietMs(),
|
|
3722
|
+
phase: turn.liveness.phase(),
|
|
3723
|
+
tools: turn.liveness.toolsInFlight(),
|
|
3724
|
+
unresponsive: true,
|
|
3725
|
+
}));
|
|
3105
3726
|
}
|
|
3106
3727
|
|
|
3107
3728
|
/**
|
|
@@ -3134,8 +3755,23 @@ export function createCopilotSdkSessionRunner({
|
|
|
3134
3755
|
}) || null;
|
|
3135
3756
|
turn.drive = driveTurn(turn);
|
|
3136
3757
|
try {
|
|
3137
|
-
|
|
3138
|
-
|
|
3758
|
+
// The watchdog covers the bring-up as well. None of the requests that
|
|
3759
|
+
// open a session (resume, create, the model switch) has a timeout in
|
|
3760
|
+
// the SDK, and a runtime that never answered one held the row in
|
|
3761
|
+
// `processing` with no turn to fail and nothing for a Stop to stop.
|
|
3762
|
+
turn.armStall();
|
|
3763
|
+
const settledFirst = Symbol('turn-settled');
|
|
3764
|
+
const building = ensureSession(model, effort);
|
|
3765
|
+
// Still observed when the turn ends first and nothing awaits it.
|
|
3766
|
+
building.catch(() => {});
|
|
3767
|
+
const built = await Promise.race([
|
|
3768
|
+
building,
|
|
3769
|
+
turn.done.then(() => settledFirst, () => settledFirst),
|
|
3770
|
+
]);
|
|
3771
|
+
if (built === settledFirst) {
|
|
3772
|
+
// Failed or stopped while the session was being built; nothing is sent.
|
|
3773
|
+
dbg('turn ended before its session was ready', message.id);
|
|
3774
|
+
} else if (turn.aborted) {
|
|
3139
3775
|
// The abort landed while the session was still being built. Nothing
|
|
3140
3776
|
// was sent, so there is no runtime turn to interrupt — settle locally
|
|
3141
3777
|
// rather than sending a prompt the user just cancelled.
|
|
@@ -3161,7 +3797,6 @@ export function createCopilotSdkSessionRunner({
|
|
|
3161
3797
|
// A send that never resolves (a wedged runtime) must not hold this
|
|
3162
3798
|
// delivery hostage once the watchdog has failed the turn: the id is
|
|
3163
3799
|
// still recorded if it ever arrives, but the wait ends with the turn.
|
|
3164
|
-
const settledFirst = Symbol('turn-settled');
|
|
3165
3800
|
const runtimeId = await Promise.race([
|
|
3166
3801
|
sending,
|
|
3167
3802
|
turn.done.then(() => settledFirst, () => settledFirst),
|
|
@@ -3182,16 +3817,8 @@ export function createCopilotSdkSessionRunner({
|
|
|
3182
3817
|
}
|
|
3183
3818
|
}
|
|
3184
3819
|
} catch (error) {
|
|
3185
|
-
// A failure that killed the session (or came from starting it)
|
|
3186
|
-
//
|
|
3187
|
-
// resumes rather than sending into a dead runtime. An UNCONFIRMED MODEL
|
|
3188
|
-
// SWITCH is the exception: the session is healthy and merely still on
|
|
3189
|
-
// its previous model, and tearing the runtime down over it would kill
|
|
3190
|
-
// any live background work for a selection problem the user fixes by
|
|
3191
|
-
// picking another model.
|
|
3192
|
-
if (!isModelSwitchUnconfirmedError(error)) {
|
|
3193
|
-
await stopRuntime('turn-failure').catch(() => {});
|
|
3194
|
-
}
|
|
3820
|
+
// A failure that killed the session (or came from starting it); the
|
|
3821
|
+
// turn's own `fail` drops the handles nothing else can use.
|
|
3195
3822
|
turn.fail(error);
|
|
3196
3823
|
}
|
|
3197
3824
|
// `send()` resolving is NOT the turn's completion; the event stream is.
|
|
@@ -3237,11 +3864,6 @@ export function createCopilotSdkSessionRunner({
|
|
|
3237
3864
|
if (!turn.aborted && !turn.failure && turn.humanRequests > 0) await turn.humansIdle;
|
|
3238
3865
|
try {
|
|
3239
3866
|
await quiesceTurn(turn);
|
|
3240
|
-
if (turn.failure && turn.kind === 'delivered' && !isModelSwitchUnconfirmedError(turn.failure) && session) {
|
|
3241
|
-
// A stall or a runtime death mid-turn (a setup failure already tore
|
|
3242
|
-
// the runtime down in runTurn).
|
|
3243
|
-
await stopRuntime('turn-failure').catch(() => {});
|
|
3244
|
-
}
|
|
3245
3867
|
if (turn.kind === 'continuation') {
|
|
3246
3868
|
// Nothing can be published before the row exists. The buffer is
|
|
3247
3869
|
// drained by the registration itself; this only covers actions that
|
|
@@ -3274,6 +3896,8 @@ export function createCopilotSdkSessionRunner({
|
|
|
3274
3896
|
// Early settles still in flight must land before the rows stop being
|
|
3275
3897
|
// reported (settleEntry never rejects, but this is the ordering guard).
|
|
3276
3898
|
await Promise.allSettled(turn.settlePromises);
|
|
3899
|
+
// The rows are settled; the runner is quiescent once the runtime is gone.
|
|
3900
|
+
if (turn.teardown) await turn.teardown;
|
|
3277
3901
|
controlPoller?.stop?.(turn.controlState);
|
|
3278
3902
|
turn.controlState = null;
|
|
3279
3903
|
// Every path through the turn — published, failed, aborted, threw — has
|
|
@@ -3451,25 +4075,54 @@ export function createCopilotSdkSessionRunner({
|
|
|
3451
4075
|
const signal = turn.registrationAbort?.signal || null;
|
|
3452
4076
|
const abandoned = () => turn.discarded || signal?.aborted === true;
|
|
3453
4077
|
let response = null;
|
|
3454
|
-
|
|
4078
|
+
// Two ways of not getting a row, counted apart. The relay ANSWERED without
|
|
4079
|
+
// one (it refused, or sent an empty body): three attempts, as ever. The
|
|
4080
|
+
// relay could not be REACHED (it is restarting): the attempts go on, spaced
|
|
4081
|
+
// further apart, for the outage window — the turn's output is buffered
|
|
4082
|
+
// meanwhile and nothing else can tell the user what the runtime did.
|
|
4083
|
+
let refusals = 0;
|
|
4084
|
+
let outageSince = 0;
|
|
4085
|
+
let outageDelayMs = continuationRetryDelayMs;
|
|
4086
|
+
while (!response?.messageId) {
|
|
3455
4087
|
if (abandoned()) return;
|
|
4088
|
+
let failure = null;
|
|
4089
|
+
// The drive path waits at least as long as the request in flight may
|
|
4090
|
+
// take: a relay that took the connection and is slow to answer is not
|
|
4091
|
+
// one that refused.
|
|
4092
|
+
if (continuationOutageWindowMs > 0) {
|
|
4093
|
+
turn.registrationDeadline = Math.max(turn.registrationDeadline || 0, Date.now() + continuationRequestTimeoutMs);
|
|
4094
|
+
}
|
|
3456
4095
|
response = await api('POST', '/api/continuation-turn', {
|
|
3457
4096
|
conversationId: sdkSessionId,
|
|
3458
4097
|
sdkSessionId,
|
|
3459
4098
|
relayMode: turn.message.relayMode,
|
|
3460
4099
|
trigger: CONTINUATION_TRIGGER,
|
|
3461
4100
|
operationId,
|
|
3462
|
-
}).catch((error) => {
|
|
4101
|
+
}, continuationRequestTimeoutMs > 0 ? { signal: AbortSignal.timeout(continuationRequestTimeoutMs) } : {}).catch((error) => {
|
|
4102
|
+
failure = error;
|
|
3463
4103
|
dbg('continuation turn registration failed', error?.message || String(error));
|
|
3464
4104
|
return null;
|
|
3465
4105
|
});
|
|
3466
|
-
if (
|
|
3467
|
-
|
|
3468
|
-
|
|
4106
|
+
if (response?.messageId) break;
|
|
4107
|
+
if (abandoned()) return;
|
|
4108
|
+
if (failure && continuationOutageWindowMs > 0 && isTransientRelayError(failure)) {
|
|
4109
|
+
outageSince = outageSince || Date.now();
|
|
4110
|
+
if (Date.now() - outageSince >= continuationOutageWindowMs) break;
|
|
4111
|
+
// The drive path waits for as long as this loop may still succeed.
|
|
4112
|
+
turn.registrationDeadline = outageSince + continuationOutageWindowMs + continuationRequestTimeoutMs;
|
|
4113
|
+
await sleep(outageDelayMs);
|
|
4114
|
+
outageDelayMs = Math.min(outageDelayMs * 2, 8_000);
|
|
4115
|
+
continue;
|
|
3469
4116
|
}
|
|
4117
|
+
refusals += 1;
|
|
4118
|
+
if (refusals >= 3) break;
|
|
4119
|
+
await sleep(continuationRetryDelayMs);
|
|
3470
4120
|
}
|
|
4121
|
+
turn.registrationDeadline = 0;
|
|
3471
4122
|
if (!response?.messageId) {
|
|
3472
|
-
abandonContinuationRegistration(turn,
|
|
4123
|
+
abandonContinuationRegistration(turn, outageSince && refusals < 3
|
|
4124
|
+
? `the relay could not be reached for ${Math.round((Date.now() - outageSince) / 1000)}s`
|
|
4125
|
+
: 'no relay message id after 3 attempts');
|
|
3473
4126
|
return;
|
|
3474
4127
|
}
|
|
3475
4128
|
const messageId = String(response.messageId);
|
|
@@ -3518,17 +4171,24 @@ export function createCopilotSdkSessionRunner({
|
|
|
3518
4171
|
*/
|
|
3519
4172
|
async function awaitContinuationRow(turn, timeoutMs = continuationRegistrationTimeoutMs) {
|
|
3520
4173
|
if (turn.registered || turn.discarded) return;
|
|
3521
|
-
let
|
|
3522
|
-
|
|
3523
|
-
timer =
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
4174
|
+
let waitMs = timeoutMs;
|
|
4175
|
+
while (true) {
|
|
4176
|
+
let timer = null;
|
|
4177
|
+
const expired = new Promise((resolve) => {
|
|
4178
|
+
timer = setTimeout(() => resolve('expired'), waitMs);
|
|
4179
|
+
timer.unref?.();
|
|
4180
|
+
});
|
|
4181
|
+
// `turn.registration` is the caught chain and never rejects.
|
|
4182
|
+
const outcome = await Promise.race([turn.registration || Promise.resolve(), expired]);
|
|
4183
|
+
clearTimeout(timer);
|
|
4184
|
+
if (outcome !== 'expired' || turn.registered || turn.message.id) return;
|
|
4185
|
+
// Still trying against a relay that cannot be reached: the deadline is
|
|
4186
|
+
// the registration's own, not this wait's.
|
|
4187
|
+
const remainingMs = (turn.registrationDeadline || 0) - Date.now();
|
|
4188
|
+
if (remainingMs <= 0) break;
|
|
4189
|
+
waitMs = remainingMs;
|
|
3531
4190
|
}
|
|
4191
|
+
abandonContinuationRegistration(turn, 'registration outlived the local deadline');
|
|
3532
4192
|
}
|
|
3533
4193
|
|
|
3534
4194
|
// --------------------------------------------------------------- steering --
|
|
@@ -3564,6 +4224,11 @@ export function createCopilotSdkSessionRunner({
|
|
|
3564
4224
|
dbg('turn settled while steering was preparing; running it as a fresh turn', message.id);
|
|
3565
4225
|
return NOT_STEERED;
|
|
3566
4226
|
}
|
|
4227
|
+
// The same for a tool that started meanwhile: the message goes back to
|
|
4228
|
+
// the queue and comes again when the tool has ended.
|
|
4229
|
+
if (mainToolsInFlight(turn).length > 0 && canHandBackHeldDelivery() !== false) {
|
|
4230
|
+
return handBackHeldDelivery(message);
|
|
4231
|
+
}
|
|
3567
4232
|
const entry = createEntry(turn, message, 'steer');
|
|
3568
4233
|
dbg('steering a mid-turn delivery into the running turn', message.id);
|
|
3569
4234
|
let runtimeId;
|
|
@@ -3611,7 +4276,8 @@ export function createCopilotSdkSessionRunner({
|
|
|
3611
4276
|
* would settle the continuation's reply against no row while its buffered
|
|
3612
4277
|
* output flushed into the steered one); a question card, ask-mode approval
|
|
3613
4278
|
* or elicitation is open (the relay must not push past a card); a
|
|
3614
|
-
* compaction is running
|
|
4279
|
+
* compaction is running; a tool of the main agent is running (see
|
|
4280
|
+
* mainToolsInFlight).
|
|
3615
4281
|
*/
|
|
3616
4282
|
function canAcceptSteering() {
|
|
3617
4283
|
if (disposed || !session) return false;
|
|
@@ -3620,9 +4286,60 @@ export function createCopilotSdkSessionRunner({
|
|
|
3620
4286
|
if (!turn.primarySent || awaitingRow(turn)) return false;
|
|
3621
4287
|
if (pendingHumanRequests > 0) return false;
|
|
3622
4288
|
if (isCompacting()) return false;
|
|
4289
|
+
if (mainToolsInFlight(turn).length > 0) return false;
|
|
3623
4290
|
return true;
|
|
3624
4291
|
}
|
|
3625
4292
|
|
|
4293
|
+
/**
|
|
4294
|
+
* The tool hold: the names of the tools the MAIN agent of this turn has
|
|
4295
|
+
* started and not finished. While there is one, nothing is steered in.
|
|
4296
|
+
*
|
|
4297
|
+
* The runtime answers a steer that arrives while it waits for a foreground
|
|
4298
|
+
* shell command by ending the tool call ("moved to background by the user")
|
|
4299
|
+
* and asking the model again. Seen live twice (2026-09-29): the turn then
|
|
4300
|
+
* ended with the answer to the steer alone, or failed on an empty reply
|
|
4301
|
+
* ("No response was returned"), and the command's result came later in a
|
|
4302
|
+
* continuation. Held back until the tool boundary, the steer lands where
|
|
4303
|
+
* the runtime would have injected it anyway and the command is left alone.
|
|
4304
|
+
*
|
|
4305
|
+
* There is no time limit on purpose: steering in after a wait would be the
|
|
4306
|
+
* same interruption, only later. A held message shows as queued, and the
|
|
4307
|
+
* end of the turn releases it like every other hold.
|
|
4308
|
+
*
|
|
4309
|
+
* Not counted:
|
|
4310
|
+
* - tools of subagents (their events carry an `agentId`). A background
|
|
4311
|
+
* agent works beside the main turn and for much longer; the main agent
|
|
4312
|
+
* is not waiting for its tools, so a steer interrupts nothing. A
|
|
4313
|
+
* subagent the main agent runs in the foreground is covered by the main
|
|
4314
|
+
* agent's own tool call that started it.
|
|
4315
|
+
* - `remote_relay`, which holds the stall watchdog but never held steering
|
|
4316
|
+
* (see `pendingRelayToolCalls`).
|
|
4317
|
+
*/
|
|
4318
|
+
function mainToolsInFlight(turn = activeTurn) {
|
|
4319
|
+
if (!turn || turn.settled) return [];
|
|
4320
|
+
return turn.liveness.toolsOf('root').filter((name) => !isRemoteRelayToolName(name));
|
|
4321
|
+
}
|
|
4322
|
+
|
|
4323
|
+
/** Why the live turn takes no steer right now ('other': it is being stopped). */
|
|
4324
|
+
function steeringHoldReason(turn) {
|
|
4325
|
+
if (pendingHumanRequests > 0) return 'question';
|
|
4326
|
+
if (isCompacting()) return 'compaction';
|
|
4327
|
+
if (!turn.primarySent || awaitingRow(turn)) return 'delivery';
|
|
4328
|
+
if (mainToolsInFlight(turn).length > 0) return 'tool';
|
|
4329
|
+
return 'other';
|
|
4330
|
+
}
|
|
4331
|
+
|
|
4332
|
+
/**
|
|
4333
|
+
* The tool hold ended (the tool did, or the turn) and deliveries were
|
|
4334
|
+
* handed back for it: say so once. The relay sends them again on the
|
|
4335
|
+
* readiness `syncDeliveryReadiness` reports.
|
|
4336
|
+
*/
|
|
4337
|
+
function noteRunningToolHoldEnd() {
|
|
4338
|
+
if (heldForRunningTool < 1 || mainToolsInFlight().length > 0) return;
|
|
4339
|
+
dbg('the running tool ended; steering no longer held for it', `handed back=${heldForRunningTool}`);
|
|
4340
|
+
heldForRunningTool = 0;
|
|
4341
|
+
}
|
|
4342
|
+
|
|
3626
4343
|
/** A continuation whose synthetic row is still being minted (and not given up on). */
|
|
3627
4344
|
function awaitingRow(turn) {
|
|
3628
4345
|
return turn.kind === 'continuation' && !turn.registered && !turn.discarded;
|
|
@@ -3643,6 +4360,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
3643
4360
|
}
|
|
3644
4361
|
|
|
3645
4362
|
function syncDeliveryReadiness() {
|
|
4363
|
+
noteRunningToolHoldEnd();
|
|
3646
4364
|
const ready = !isDeliveryHeld();
|
|
3647
4365
|
if (ready === lastReportedDeliveryReady) return;
|
|
3648
4366
|
lastReportedDeliveryReady = ready;
|
|
@@ -3660,7 +4378,14 @@ export function createCopilotSdkSessionRunner({
|
|
|
3660
4378
|
* hold instead would bypass the question card or land inside a compaction.
|
|
3661
4379
|
*/
|
|
3662
4380
|
async function handBackHeldDelivery(message) {
|
|
3663
|
-
|
|
4381
|
+
const turn = activeTurn;
|
|
4382
|
+
if (turn && !turn.settled && steeringHoldReason(turn) === 'tool') {
|
|
4383
|
+
heldForRunningTool += 1;
|
|
4384
|
+
const tools = [...new Set(mainToolsInFlight(turn).map((name) => name || 'tool'))];
|
|
4385
|
+
dbg('steering held while a tool runs; handing the delivery back to the queue', message.id || '(no id)', `tools=${tools.join(', ')}`);
|
|
4386
|
+
} else {
|
|
4387
|
+
dbg('steering held; handing the delivery back to the queue', message.id || '(no id)');
|
|
4388
|
+
}
|
|
3664
4389
|
if (!message.id) return false;
|
|
3665
4390
|
await api('POST', '/api/requeue', {
|
|
3666
4391
|
messageId: message.id,
|
|
@@ -3708,13 +4433,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
3708
4433
|
const turn = activeTurn;
|
|
3709
4434
|
const turnActive = Boolean(turn && !turn.settled);
|
|
3710
4435
|
const canSteer = canAcceptSteering();
|
|
3711
|
-
|
|
3712
|
-
if (turnActive && !canSteer) {
|
|
3713
|
-
if (pendingHumanRequests > 0) holdReason = 'question';
|
|
3714
|
-
else if (isCompacting()) holdReason = 'compaction';
|
|
3715
|
-
else if (!turn.primarySent || awaitingRow(turn)) holdReason = 'delivery';
|
|
3716
|
-
else holdReason = 'other';
|
|
3717
|
-
}
|
|
4436
|
+
const holdReason = turnActive && !canSteer ? steeringHoldReason(turn) : null;
|
|
3718
4437
|
const messageId = String(turn?.currentOwner?.message?.id || turn?.message?.id || '').trim() || null;
|
|
3719
4438
|
return { turnActive, canSteer, holdReason, messageId, supported: true, cancellableIds: cancellableIds() };
|
|
3720
4439
|
}
|
|
@@ -3762,6 +4481,18 @@ export function createCopilotSdkSessionRunner({
|
|
|
3762
4481
|
}
|
|
3763
4482
|
}
|
|
3764
4483
|
|
|
4484
|
+
/**
|
|
4485
|
+
* A request to the runtime that has to come back. Rejects when the runtime
|
|
4486
|
+
* does not answer in time, so the caller's own failure path runs instead of
|
|
4487
|
+
* a chain that waits for ever.
|
|
4488
|
+
*/
|
|
4489
|
+
async function askRuntime(label, request) {
|
|
4490
|
+
const unanswered = Symbol('unanswered');
|
|
4491
|
+
const answer = await withTimeout(Promise.resolve().then(request), runtimeRpcTimeoutMs, unanswered);
|
|
4492
|
+
if (answer === unanswered) throw new Error(`the runtime did not answer ${label} within ${runtimeRpcTimeoutMs}ms`);
|
|
4493
|
+
return answer;
|
|
4494
|
+
}
|
|
4495
|
+
|
|
3765
4496
|
function withTimeout(promise, ms, fallback) {
|
|
3766
4497
|
let timer = null;
|
|
3767
4498
|
return Promise.race([
|
|
@@ -3850,12 +4581,12 @@ export function createCopilotSdkSessionRunner({
|
|
|
3850
4581
|
async function handlePendingPayload(pending) {
|
|
3851
4582
|
const message = pending?.message || null;
|
|
3852
4583
|
if (!message) return false;
|
|
3853
|
-
// Held (a card is open, a compaction is running, the
|
|
3854
|
-
// is not sent yet): hand the row back
|
|
3855
|
-
// re-delivers it when the hold ends. An
|
|
3856
|
-
//
|
|
3857
|
-
// delivery — retry, backoff, worker
|
|
3858
|
-
// message is pushed the legacy way.
|
|
4584
|
+
// Held (a card is open, a compaction is running, a tool of the main agent
|
|
4585
|
+
// is running, the turn-opening prompt is not sent yet): hand the row back
|
|
4586
|
+
// with no retry penalty; the relay re-delivers it when the hold ends. An
|
|
4587
|
+
// older relay (the worker updated on disk before the relay restarted)
|
|
4588
|
+
// treats the hand-back as a failed delivery — retry, backoff, worker
|
|
4589
|
+
// marked errored — so against one the message is pushed the legacy way.
|
|
3859
4590
|
if (isDeliveryHeld() && canHandBackHeldDelivery() !== false) return handBackHeldDelivery(message);
|
|
3860
4591
|
// Legacy path only: never send a second prompt before the turn-opening one
|
|
3861
4592
|
// (it would invert the runs), and never start a second turn beside one
|