@oar-sh/oar 0.9.2 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +269 -0
- package/README.md +407 -424
- package/bin/oar.js +25 -10
- package/package.json +7 -4
- package/server/README.md +1 -1
- package/server/claude-worker/claude-session-process.mjs +981 -44
- package/server/claude-worker/claude-session-worker.mjs +45 -4
- package/server/claude-worker/claude-turn-publisher.mjs +81 -20
- package/server/claude-worker/claude-worker-link-wiring.mjs +9 -0
- package/server/copilot-worker/copilot-ask-user-tool.mjs +89 -0
- package/server/copilot-worker/copilot-continuation-signals.mjs +55 -0
- package/server/copilot-worker/copilot-question-bridge.mjs +34 -12
- package/server/copilot-worker/copilot-sdk-event-normalizer.mjs +101 -12
- package/server/copilot-worker/copilot-sdk-session-process.mjs +2269 -492
- package/server/copilot-worker/copilot-sdk-session-worker.mjs +82 -20
- package/server/db-schema.mjs +20 -0
- package/server/migrations/0004-steer-settle-markers.mjs +119 -0
- package/server/public/app/api-client.js +22 -2
- package/server/public/app/ask-user-view.js +102 -13
- package/server/public/app/attachments-view.js +37 -2
- package/server/public/app/background-task-timeout-options.mjs +9 -0
- package/server/public/app/background-tasks-view.mjs +33 -18
- package/server/public/app/bootstrap.js +34 -8
- package/server/public/app/composer-control-state.mjs +52 -34
- package/server/public/app/conversation-draft-sync.mjs +187 -0
- package/server/public/app/conversation-list-filter.mjs +86 -0
- package/server/public/app/conversation-provider-indicator.mjs +8 -0
- package/server/public/app/conversation-view.js +919 -194
- package/server/public/app/emoji-view.js +2 -0
- package/server/public/app/font-scaling.js +4 -37
- package/server/public/app/journal-view.js +145 -5
- package/server/public/app/model-selector-options.mjs +1 -1
- package/server/public/app/question-multi-select.mjs +52 -0
- package/server/public/app/relay-prompt-sanitizer.mjs +17 -2
- package/server/public/app/settings-modal.js +8 -6
- package/server/public/app/socket-handlers.js +19 -0
- package/server/public/app/steered-turn-merge.mjs +100 -0
- package/server/public/app/store.js +126 -15
- package/server/public/app/sync-outbox.mjs +16 -8
- package/server/public/app/tool-activity-emoji.mjs +36 -0
- package/server/public/app/transcript-viewport-anchor.mjs +210 -0
- package/server/public/index.html +206 -9
- package/server/public/sw.js +7 -0
- package/server/relay-tools.md +2 -0
- package/server/repositories/message-repository.mjs +61 -18
- package/server/repositories/question-repository.mjs +15 -0
- package/server/repositories/session-repository.mjs +7 -31
- package/server/routes/messages-routes.mjs +488 -46
- package/server/routes/sessions-routes.mjs +298 -55
- package/server/server-runtime.mjs +65 -25
- package/server/services/live-turn-picker.mjs +47 -0
- package/server/services/relay-prompt-sanitizer.mjs +17 -2
- package/server/services/relay-question-context.mjs +46 -0
- package/server/services/sdk-session-import-service.mjs +49 -7
- package/server/services/session-worker-registry-service.mjs +75 -0
- package/server/services/session-worker-supervisor-service.mjs +7 -1
- package/server/services/session-worker-websocket-service.mjs +23 -1
- package/server/services/steer-resend.mjs +45 -0
- package/shared/background-task-timeout.mjs +12 -4
- package/shared/control-poller.mjs +4 -1
- package/shared/question-multi-select.mjs +16 -0
- package/shared/relay-stop-failure.mjs +15 -0
- package/shared/slash-command-guard.mjs +76 -0
- package/shared/steer-note.mjs +66 -0
- package/shared/steer-settle-failure.mjs +89 -0
- package/shared/steer-settle-markers.mjs +18 -0
- package/shared/worker-crash-guard.mjs +40 -25
- package/shared/worker-runtime/heartbeat.mjs +55 -1
- package/shared/worker-runtime/runner-link-wiring.mjs +44 -0
- package/shared/worker-runtime/worker-websocket-link.mjs +119 -8
- package/server/claude-worker/claude-session-test-harness.mjs +0 -250
- package/server/claude-worker/fixtures/workflow-run-record.json +0 -1
- package/server/copilot-worker/copilot-sdk-test-harness.mjs +0 -382
- package/server/copilot-worker/fixtures/abort-turn.json +0 -588
- package/server/copilot-worker/fixtures/ask-user-turn.json +0 -1154
- package/server/copilot-worker/fixtures/background-timer-continuation.json +0 -445
- package/server/copilot-worker/fixtures/background-timer-turn.json +0 -497
- package/server/copilot-worker/fixtures/happy-turn.json +0 -566
- package/server/copilot-worker/fixtures/quota-turn.json +0 -647
- package/server/copilot-worker/fixtures/reasoning-turn.json +0 -625
- package/server/copilot-worker/fixtures/resume-turn.json +0 -700
- package/server/copilot-worker/fixtures/subagent-turn.json +0 -1378
- package/server/copilot-worker/fixtures/tool-permission-turn.json +0 -1293
- package/server/routes/messages-routes-test-harness.mjs +0 -111
- package/server/services/fixtures/claude-auth-stub.cmd +0 -1
- package/server/services/fixtures/claude-auth-stub.mjs +0 -76
- package/server/services/fixtures/claude-auth-stub.sh +0 -4
- package/server/services/fixtures/cli-install-stub.cmd +0 -1
- package/server/services/fixtures/cli-install-stub.mjs +0 -98
- package/server/services/fixtures/cli-install-stub.sh +0 -4
- package/server/services/fixtures/grok-stub.cmd +0 -1
- package/server/services/fixtures/grok-stub.mjs +0 -89
- package/server/services/fixtures/grok-stub.sh +0 -4
- package/shared/fixtures/copilot-catalog-raw-2026-09-07.json +0 -2363
|
@@ -63,10 +63,12 @@ import { resolveCopilotProviderConfig } from './copilot-byok-provider.mjs';
|
|
|
63
63
|
import {
|
|
64
64
|
createBackgroundShellTracker,
|
|
65
65
|
createReplayGate,
|
|
66
|
+
describeSettledAgent,
|
|
66
67
|
describeSettledShell,
|
|
67
68
|
isContinuationOpeningEvent,
|
|
69
|
+
settledAgentFromNotification,
|
|
68
70
|
} from './copilot-continuation-signals.mjs';
|
|
69
|
-
import { createCopilotEventNormalizer } from './copilot-sdk-event-normalizer.mjs';
|
|
71
|
+
import { createCopilotEventNormalizer, formatToolActivityText } from './copilot-sdk-event-normalizer.mjs';
|
|
70
72
|
import {
|
|
71
73
|
DEFAULT_MODEL_SWITCH_TIMEOUT_MS,
|
|
72
74
|
createCopilotModelSwitcher,
|
|
@@ -74,6 +76,14 @@ import {
|
|
|
74
76
|
normalizeRelayEffort,
|
|
75
77
|
} from './copilot-model-switch.mjs';
|
|
76
78
|
import { createCopilotQuestionBridge } from './copilot-question-bridge.mjs';
|
|
79
|
+
import {
|
|
80
|
+
RELAY_ASK_USER_TOOL_DESCRIPTION,
|
|
81
|
+
RELAY_ASK_USER_TOOL_NAME,
|
|
82
|
+
RELAY_ASK_USER_TOOL_PARAMETERS,
|
|
83
|
+
formatAskUserToolResult,
|
|
84
|
+
isToolOverrideRejection,
|
|
85
|
+
parseAskUserToolArguments,
|
|
86
|
+
} from './copilot-ask-user-tool.mjs';
|
|
77
87
|
import {
|
|
78
88
|
EXIT_PLAN_BOARD_POSTED_FEEDBACK,
|
|
79
89
|
EXIT_PLAN_NO_BOARD_FEEDBACK,
|
|
@@ -89,6 +99,11 @@ import {
|
|
|
89
99
|
} from './copilot-prompt-context.mjs';
|
|
90
100
|
import { EMPTY_TURN_COMPLETION_NOTE } from '../../shared/empty-turn-completion.mjs';
|
|
91
101
|
import { buildModelSnapshotFields, extractModelDescriptors } from '../../shared/model-descriptors.mjs';
|
|
102
|
+
import { buildRelayStopFailure } from '../../shared/relay-stop-failure.mjs';
|
|
103
|
+
import { STEER_FOLDED_TEXT, STEER_STOPPED_TEXT } from '../../shared/steer-settle-markers.mjs';
|
|
104
|
+
import { buildSteerSettleFailure } from '../../shared/steer-settle-failure.mjs';
|
|
105
|
+
import { shouldEmitStreamUpdate } from '../../shared/stream-emit-gating.mjs';
|
|
106
|
+
import { withSteerNote } from '../../shared/steer-note.mjs';
|
|
92
107
|
|
|
93
108
|
// How long the runtime may sit with no session activity before the worker
|
|
94
109
|
// closes it. The worker process itself stays up and reconnects lazily on the
|
|
@@ -132,6 +147,45 @@ const DEFAULT_CONTINUATION_RETRY_DELAY_MS = 500;
|
|
|
132
147
|
const CONTINUATION_REGISTRATION_TIMEOUT_MS = 10_000;
|
|
133
148
|
/** Cap on activity lines carried between turns, so a chatty runtime cannot grow them. */
|
|
134
149
|
const MAX_PENDING_ACTIVITIES = 20;
|
|
150
|
+
/**
|
|
151
|
+
* Backstop for the compaction hold: `session.compaction_start` without a
|
|
152
|
+
* matching `_complete` (a runtime that died mid-compaction and was resumed)
|
|
153
|
+
* must not hold steering forever. Sized far above any plausible compaction,
|
|
154
|
+
* like the Claude worker's `compactionStaleMs`.
|
|
155
|
+
*/
|
|
156
|
+
const DEFAULT_COMPACTION_STALE_MS = 10 * 60_000;
|
|
157
|
+
/**
|
|
158
|
+
* Backoff between attempts to save a consumed steer's settle marker (~31 s in
|
|
159
|
+
* all); after the last one the row fails terminally instead. Same ladder as
|
|
160
|
+
* the Claude worker, because it protects the same invariant: a prompt the
|
|
161
|
+
* runtime already consumed is never re-run.
|
|
162
|
+
*/
|
|
163
|
+
const DEFAULT_SETTLE_RETRY_DELAYS_MS = Object.freeze([1_000, 2_000, 4_000, 8_000, 8_000, 8_000]);
|
|
164
|
+
const DEFAULT_MAX_TERMINAL_SETTLE_ATTEMPTS = 5;
|
|
165
|
+
/** Debounce for re-reading the runtime's queued lane after `pending_messages.modified`. */
|
|
166
|
+
const DEFAULT_QUEUE_LANE_REFRESH_MS = 150;
|
|
167
|
+
/**
|
|
168
|
+
* How long a Stop's acknowledgement is held back for the reply of the row it
|
|
169
|
+
* named when that row's early settle is still publishing (see interruptTurn).
|
|
170
|
+
* Bounded: an unresponsive relay must not hold the Stop itself.
|
|
171
|
+
*/
|
|
172
|
+
const STOPPED_ROW_PUBLISH_WAIT_MS = 5_000;
|
|
173
|
+
/**
|
|
174
|
+
* Debounce for re-reading `rpc.tasks.list()` after `session.background_tasks_changed`
|
|
175
|
+
* (which carries no payload and fires ~20 times per shell call).
|
|
176
|
+
*/
|
|
177
|
+
const DEFAULT_TASK_REFRESH_MS = 250;
|
|
178
|
+
/** Trailing throttle for enrichment-only card updates (a live tool call, tokens). */
|
|
179
|
+
const DEFAULT_TASK_PUBLISH_THROTTLE_MS = 2_000;
|
|
180
|
+
/**
|
|
181
|
+
* How long a pushed prompt the runtime had not started when the turn's idle
|
|
182
|
+
* arrived is kept unsettled, waiting for the runtime to open its run. The
|
|
183
|
+
* drain's idle can race a send that resolved a moment earlier; the runtime
|
|
184
|
+
* then starts the prompt right away (`immediate` when idle) and its
|
|
185
|
+
* `user.message` adopts the row as a turn of its own. Same value as the
|
|
186
|
+
* Claude worker's fold grace, for the same reason.
|
|
187
|
+
*/
|
|
188
|
+
const DEFAULT_ORPHAN_GRACE_MS = 2_000;
|
|
135
189
|
|
|
136
190
|
/**
|
|
137
191
|
* Appended to the partial answer when the RUNTIME interrupted the turn on its
|
|
@@ -145,15 +199,27 @@ export const RUNTIME_INTERRUPTED_NOTE =
|
|
|
145
199
|
|
|
146
200
|
/**
|
|
147
201
|
* Published to a steered queue row whose prompt the runtime accepted but never
|
|
148
|
-
* opened work on before the interaction ended. The prompt is still
|
|
149
|
-
* INSIDE the runtime, so it will be answered at the start of the next
|
|
150
|
-
* requeuing the row would run it twice.
|
|
151
|
-
*
|
|
202
|
+
* opened work on before the interaction ended normally. The prompt is still
|
|
203
|
+
* queued INSIDE the runtime, so it will be answered at the start of the next
|
|
204
|
+
* turn — requeuing the row would run it twice. With `mode: "immediate"` the
|
|
205
|
+
* runtime drains every pushed prompt before it idles, so this is a defensive
|
|
206
|
+
* fallback rather than a routine outcome.
|
|
152
207
|
*/
|
|
153
208
|
export const STEERED_ROW_MERGED_NOTE =
|
|
154
209
|
'_(This message was delivered while the previous turn was still running; the reply continues in '
|
|
155
210
|
+ 'the next turn.)_';
|
|
156
211
|
|
|
212
|
+
/**
|
|
213
|
+
* The reply of a continuation row opened only so a background task's question
|
|
214
|
+
* had a row to hang its card off (see `ensureInteractiveTurn`). The runtime
|
|
215
|
+
* produced no root work for it, so nothing else would ever answer the row.
|
|
216
|
+
*/
|
|
217
|
+
export const BACKGROUND_QUESTION_NOTE =
|
|
218
|
+
'System note: a background task asked for input here; it carries on in the background.';
|
|
219
|
+
|
|
220
|
+
/** The provider name in the at-most-once settle failure wording. */
|
|
221
|
+
const SETTLE_AGENT_LABEL = 'Copilot';
|
|
222
|
+
|
|
157
223
|
/**
|
|
158
224
|
* The `trigger` reported to `POST /api/continuation-turn`, matching the value
|
|
159
225
|
* the Claude worker sends so the relay's `CONTINUATION … trigger=` log line and
|
|
@@ -199,6 +265,22 @@ function isStaleAttemptError(error) {
|
|
|
199
265
|
return detail.includes('stale_attempt');
|
|
200
266
|
}
|
|
201
267
|
|
|
268
|
+
/**
|
|
269
|
+
* Whether publishing `action` makes its row worth keeping (`entry.showedWork`).
|
|
270
|
+
* A bookkeeping line only tracks background work the task card already shows,
|
|
271
|
+
* and so does a line carried in from between turns (a shell or agent the card
|
|
272
|
+
* showed finishing) — unless it reports a failure. The card lists running
|
|
273
|
+
* tasks only, so a failed agent's card vanishes exactly like a finished one's,
|
|
274
|
+
* and a `subagent.failed` between turns reaches no row: that line is the
|
|
275
|
+
* failure's only trace. It renders only through its row's response, so
|
|
276
|
+
* dropping the row would drop the failure with it.
|
|
277
|
+
*/
|
|
278
|
+
function showsWork(action) {
|
|
279
|
+
if (action.channel === 'init') return false;
|
|
280
|
+
if (action.carried) return action.reportsFailure === true;
|
|
281
|
+
return !action.bookkeeping;
|
|
282
|
+
}
|
|
283
|
+
|
|
202
284
|
export function createCopilotSdkSessionRunner({
|
|
203
285
|
api,
|
|
204
286
|
sdkSessionId,
|
|
@@ -220,15 +302,16 @@ export function createCopilotSdkSessionRunner({
|
|
|
220
302
|
resolveProviderConfigImpl = resolveCopilotProviderConfig,
|
|
221
303
|
// Threading seam for `MessageOptions.mode` ("enqueue" | "immediate").
|
|
222
304
|
//
|
|
223
|
-
//
|
|
224
|
-
//
|
|
225
|
-
//
|
|
226
|
-
//
|
|
227
|
-
//
|
|
228
|
-
//
|
|
229
|
-
//
|
|
230
|
-
//
|
|
231
|
-
|
|
305
|
+
// "immediate", always (fake-provider probe against runtime 1.0.88,
|
|
306
|
+
// 2026-09-25). Mid-turn it is a real steer: injected at the next tool
|
|
307
|
+
// boundary as `user.message{delivery:"steering"}` (or, when the model is
|
|
308
|
+
// mid-stream, moved to the FRONT of the queue and run right after as
|
|
309
|
+
// `delivery:"queued"`). When the main loop is idle both modes start a run at
|
|
310
|
+
// once — except that an "enqueue" message sent while a background agent or
|
|
311
|
+
// an attached shell keeps `session.idle` deferred sits FROZEN until that
|
|
312
|
+
// work ends, while an "immediate" one runs immediately. So immediate is the
|
|
313
|
+
// only mode that never strands a message behind background work.
|
|
314
|
+
resolveSendModeImpl = () => 'immediate',
|
|
232
315
|
// Interactive surfaces. Tests inject a fake bridge; nothing here reaches the
|
|
233
316
|
// relay without one.
|
|
234
317
|
createQuestionBridgeImpl = createCopilotQuestionBridge,
|
|
@@ -259,6 +342,32 @@ export function createCopilotSdkSessionRunner({
|
|
|
259
342
|
// How long a settled continuation waits for its row before giving up on it.
|
|
260
343
|
// Must outlast the registration's own retries; a test shortens it.
|
|
261
344
|
continuationRegistrationTimeoutMs = CONTINUATION_REGISTRATION_TIMEOUT_MS,
|
|
345
|
+
compactionStaleMs = DEFAULT_COMPACTION_STALE_MS,
|
|
346
|
+
settleRetryDelaysMs = DEFAULT_SETTLE_RETRY_DELAYS_MS,
|
|
347
|
+
maxTerminalSettleAttempts = DEFAULT_MAX_TERMINAL_SETTLE_ATTEMPTS,
|
|
348
|
+
queueLaneRefreshMs = DEFAULT_QUEUE_LANE_REFRESH_MS,
|
|
349
|
+
orphanGraceMs = DEFAULT_ORPHAN_GRACE_MS,
|
|
350
|
+
// Register the relay's own `ask_user` (with `multi_select`) over the
|
|
351
|
+
// runtime's built-in one. A runtime that refuses the override falls back
|
|
352
|
+
// to the built-in for the rest of this worker's life.
|
|
353
|
+
relayAskUserTool = true,
|
|
354
|
+
taskRefreshMs = DEFAULT_TASK_REFRESH_MS,
|
|
355
|
+
taskPublishThrottleMs = DEFAULT_TASK_PUBLISH_THROTTLE_MS,
|
|
356
|
+
// Called with `true`/`false` whenever the runner flips between accepting
|
|
357
|
+
// deliveries and holding them (see isDeliveryHeld). The worker wires it to
|
|
358
|
+
// the socket link: a hold withdraws the relay's readiness, and its end
|
|
359
|
+
// re-arms it at once so a held message steers into the resumed turn.
|
|
360
|
+
onDeliveryReadinessChange = () => {},
|
|
361
|
+
// False while the connected relay does not understand the `steering-held`
|
|
362
|
+
// hand-back (a worker updated on disk before the relay restarted): a held
|
|
363
|
+
// delivery is then pushed the legacy way instead of being handed back into
|
|
364
|
+
// a retry-and-backoff requeue.
|
|
365
|
+
canHandBackHeldDelivery = () => true,
|
|
366
|
+
// Called when the set of recallable (un-steerable) rows changes. The worker
|
|
367
|
+
// wires it to an immediate heartbeat: the client's Cancel appears from the
|
|
368
|
+
// heartbeat snapshot, and a 10 s wait would often outlast the window in
|
|
369
|
+
// which the runtime still holds the message (live check 2026-09-25).
|
|
370
|
+
onCancellableChange = () => {},
|
|
262
371
|
dbg = () => {},
|
|
263
372
|
} = {}) {
|
|
264
373
|
let client = null;
|
|
@@ -303,12 +412,105 @@ export function createCopilotSdkSessionRunner({
|
|
|
303
412
|
let starting = null;
|
|
304
413
|
let disposed = false;
|
|
305
414
|
let detachRuntimeExit = () => {};
|
|
306
|
-
//
|
|
307
|
-
//
|
|
308
|
-
//
|
|
309
|
-
|
|
310
|
-
//
|
|
311
|
-
|
|
415
|
+
// A compaction in progress (`session.compaction_start` seen, no `_complete`
|
|
416
|
+
// yet). Holds steering: a message pushed now would land inside the
|
|
417
|
+
// compaction's replay. `compactingSince` backs the stale cap.
|
|
418
|
+
let compactingSince = 0;
|
|
419
|
+
// The last readiness value reported to `onDeliveryReadinessChange`, so the
|
|
420
|
+
// link only hears about flips.
|
|
421
|
+
let lastReportedDeliveryReady = null;
|
|
422
|
+
// Rows whose prompt the runtime consumed and whose settle marker is being
|
|
423
|
+
// saved (id → { message, variant }). Reported by the heartbeat WITH a
|
|
424
|
+
// terminal error until the publish lands, so recovery can never re-run them
|
|
425
|
+
// (see publishSettleMarker).
|
|
426
|
+
const settlingMessages = new Map();
|
|
427
|
+
// Settle markers that could not be posted even terminally; the heartbeat
|
|
428
|
+
// hands them to the relay to fail (id → terminalError).
|
|
429
|
+
const settleFailedTerminals = new Map();
|
|
430
|
+
// Per-row chains of `/api/queue-consumed` POSTs, so a mark and a later
|
|
431
|
+
// un-mark for the same row reach the relay in order.
|
|
432
|
+
const consumedMarkChains = new Map();
|
|
433
|
+
// The runtime's queued lane as last read from `rpc.queue.pendingItems`:
|
|
434
|
+
// runtime message id → stable queue item id. What un-steer needs, and what
|
|
435
|
+
// decides which pushed rows the client may still cancel.
|
|
436
|
+
let queuedLane = new Map();
|
|
437
|
+
// The rest of the last queue snapshot: how many immediate messages wait in
|
|
438
|
+
// the runtime's steering lane (entries not yet folded into the running
|
|
439
|
+
// turn), and how many items the queued lane holds. The steering lane has no
|
|
440
|
+
// ids — only its newest entry can be removed (`queue.removeMostRecent`,
|
|
441
|
+
// LIFO across both lanes; probe 2026-09-25) — so these decide whether our
|
|
442
|
+
// newest steer is still recallable.
|
|
443
|
+
let waitingSteers = 0;
|
|
444
|
+
let queuedItemCount = 0;
|
|
445
|
+
// Guards for the LIFO recall (removeMostRecent takes the runtime's NEWEST
|
|
446
|
+
// waiting message, whoever sent it):
|
|
447
|
+
// - `steerSendsInFlight`: steer sends whose `send()` has not resolved yet.
|
|
448
|
+
// The runtime may already count such a message while this worker cannot
|
|
449
|
+
// yet name it, so no "newest" recall is attempted while any is in flight.
|
|
450
|
+
// - `steerSendSeq`: send order across every owned turn, so "our newest
|
|
451
|
+
// unconsumed steer" is decided by when it was sent, not by which turn
|
|
452
|
+
// object happens to hold it.
|
|
453
|
+
// - `unsteerChain`: un-steers run one at a time (each re-reads the queue).
|
|
454
|
+
// Steer sends never wait on an un-steer; the recall check and its remove
|
|
455
|
+
// request run in the same tick, so no send can be issued between them.
|
|
456
|
+
//
|
|
457
|
+
// What else removeMostRecent can take (live probe, runtime 1.0.85,
|
|
458
|
+
// 2026-09-26): only user-facing sends. The runtime's own notifications
|
|
459
|
+
// (agent_idle / agent_completed / shell_completed) never show in either
|
|
460
|
+
// lane — with one pending and nothing of ours queued the remove answered
|
|
461
|
+
// `removed:false`, and the notification was still delivered after a remove
|
|
462
|
+
// that took our steer. A `source:"system"` send is hidden from pendingItems
|
|
463
|
+
// and skipped (the remove took the OLDER user steer; the system one still
|
|
464
|
+
// ran). An `agent-*`-sourced send is listed and removable, but the
|
|
465
|
+
// runtime's inter-agent prompts never passed through the main lanes. So the
|
|
466
|
+
// read-then-remove window cannot drop a runtime message; what it can still
|
|
467
|
+
// hit is a message another client of the session pushed in that window,
|
|
468
|
+
// which the two guards below detect:
|
|
469
|
+
// - `recallWatches`: runtime id → { taken } for a recall in progress. A
|
|
470
|
+
// root `user.message` naming it sets `taken` from the event callback
|
|
471
|
+
// itself — the SDK dispatches a notification before it resolves a later
|
|
472
|
+
// RPC answer, whereas `entry.consumed` is set on the dispatch chain and
|
|
473
|
+
// can lag behind a slow publish.
|
|
474
|
+
// - `recalledRuntimeIds`: runtime id → row id of every prompt pulled back
|
|
475
|
+
// and cancelled. One must never run; if it does, the remove took
|
|
476
|
+
// another message and the row was cancelled wrongly — logged loudly.
|
|
477
|
+
let steerSendsInFlight = 0;
|
|
478
|
+
let lastCancellableSignature = '';
|
|
479
|
+
let steerSendSeq = 0;
|
|
480
|
+
let unsteerChain = Promise.resolve();
|
|
481
|
+
const recallWatches = new Map();
|
|
482
|
+
const recalledRuntimeIds = new Map();
|
|
483
|
+
const MAX_RECALLED_RUNTIME_IDS = 200;
|
|
484
|
+
/** How long an un-steer may wait on the runtime before it gives up. */
|
|
485
|
+
const UNSTEER_TIMEOUT_MS = 5_000;
|
|
486
|
+
let queueLaneTimer = null;
|
|
487
|
+
let queueLaneRefreshChain = Promise.resolve();
|
|
488
|
+
// Whether the session's `rpc.queue` / `rpc.interruptMainTurn` / `rpc.tasks`
|
|
489
|
+
// are present. Probed per session (the runtime auto-updates under the relay
|
|
490
|
+
// and every one of these is @experimental); a missing surface degrades to
|
|
491
|
+
// the previous behaviour, never to an error.
|
|
492
|
+
let sessionRpc = { queue: false, interruptMainTurn: false, tasks: false };
|
|
493
|
+
// Set once a runtime refused the relay's `ask_user` override: every later
|
|
494
|
+
// session is built without it (the built-in + wording heuristic remain).
|
|
495
|
+
let askUserOverrideRejected = false;
|
|
496
|
+
// The runtime's background task registry as last read from
|
|
497
|
+
// `rpc.tasks.list()` (id → TaskInfo), and what the event stream adds per
|
|
498
|
+
// agent that the registry does not carry: the live tool call, a running
|
|
499
|
+
// token count, the model announced at spawn. Both feed the task panel
|
|
500
|
+
// cards; the registry alone decides liveness (the lifecycle pin).
|
|
501
|
+
let backgroundTasks = new Map();
|
|
502
|
+
const agentDetails = new Map();
|
|
503
|
+
let taskRefreshTimer = null;
|
|
504
|
+
let taskRefreshChain = Promise.resolve();
|
|
505
|
+
let lastTaskRefreshAt = 0;
|
|
506
|
+
let taskPublishTimer = null;
|
|
507
|
+
let lastPublishedTasksSignature = '';
|
|
508
|
+
// Cap expiry is bounded: a task whose cancel the runtime refuses (or that
|
|
509
|
+
// keeps failing) this many times is abandoned — dropped from the live set so
|
|
510
|
+
// it stops pinning the runtime — rather than held forever.
|
|
511
|
+
const taskCancelRefusals = new Map();
|
|
512
|
+
const abandonedTasks = new Set();
|
|
513
|
+
const MAX_TASK_CANCEL_REFUSALS = 3;
|
|
312
514
|
// Blocking handlers currently waiting on a human (`ask_user`, an ask-mode
|
|
313
515
|
// tool approval). The runtime emits NO events while blocked in one, and the
|
|
314
516
|
// question timeout is 8 hours against a 120s stall ceiling — so without this
|
|
@@ -334,10 +536,11 @@ export function createCopilotSdkSessionRunner({
|
|
|
334
536
|
// continuation opening, or by the grace expiring on a runtime that decided it
|
|
335
537
|
// had nothing to say.
|
|
336
538
|
let continuationDueSince = 0;
|
|
337
|
-
// Transcript lines produced between turns (a settled shell's
|
|
338
|
-
//
|
|
339
|
-
//
|
|
340
|
-
// makes.
|
|
539
|
+
// Transcript lines produced between turns (a settled shell's or background
|
|
540
|
+
// agent's notification), as `{ text, reportsFailure }`. They belong to the
|
|
541
|
+
// turn they trigger, so they are carried into it rather than dropped — the
|
|
542
|
+
// same trade the Claude worker's `pendingActivities` makes. A line that
|
|
543
|
+
// reports a failure keeps the row it lands on (see `showsWork`).
|
|
341
544
|
let pendingActivities = [];
|
|
342
545
|
// The relay mode of the last delivered turn. A self-initiated turn has no
|
|
343
546
|
// delivery to read a mode off, and it is a continuation OF that turn's work,
|
|
@@ -345,33 +548,73 @@ export function createCopilotSdkSessionRunner({
|
|
|
345
548
|
// plan-board gating behaving the same either side of a background wait.
|
|
346
549
|
let lastRelayMode = 'agent';
|
|
347
550
|
|
|
551
|
+
/**
|
|
552
|
+
* The turn a human request belongs to, opened on demand when none is live.
|
|
553
|
+
*
|
|
554
|
+
* A card needs a `processing` row (`/api/relay-question` 409s otherwise),
|
|
555
|
+
* and a background agent can ask long after the turn that spawned it settled
|
|
556
|
+
* on `assistant.idle`. Nothing else opens a row for that question, so it
|
|
557
|
+
* gets a continuation of its own — the Claude worker opens one before an
|
|
558
|
+
* off-turn `AskUserQuestion` for the same reason. `humanOrigin`: unless root
|
|
559
|
+
* work joins it, it closes itself once its last card is done.
|
|
560
|
+
*/
|
|
561
|
+
function ensureInteractiveTurn() {
|
|
562
|
+
if (activeTurn || !session || disposed) return activeTurn;
|
|
563
|
+
const turn = openContinuationTurn();
|
|
564
|
+
turn.humanOrigin = true;
|
|
565
|
+
return turn;
|
|
566
|
+
}
|
|
567
|
+
|
|
348
568
|
async function whileAwaitingHuman(run) {
|
|
569
|
+
// Captured once: the card belongs to this turn's row even when another
|
|
570
|
+
// turn owns the event stream by the time it is created or answered.
|
|
571
|
+
const turn = ensureInteractiveTurn();
|
|
349
572
|
pendingHumanRequests += 1;
|
|
573
|
+
if (turn) {
|
|
574
|
+
if (turn.humanRequests === 0) turn.humansIdle = new Promise((resolve) => { turn.resolveHumansIdle = resolve; });
|
|
575
|
+
turn.humanRequests += 1;
|
|
576
|
+
turn.askedHuman = true;
|
|
577
|
+
}
|
|
578
|
+
// An open card holds steering: the relay's readiness is withdrawn at once.
|
|
579
|
+
syncDeliveryReadiness();
|
|
350
580
|
try {
|
|
351
|
-
return await run();
|
|
581
|
+
return await run(turn);
|
|
352
582
|
} finally {
|
|
353
583
|
pendingHumanRequests -= 1;
|
|
584
|
+
if (turn) {
|
|
585
|
+
turn.humanRequests -= 1;
|
|
586
|
+
if (turn.humanRequests === 0) {
|
|
587
|
+
turn.resolveHumansIdle();
|
|
588
|
+
// A row opened only for questions: no terminator is coming for it.
|
|
589
|
+
if (turn.humanOrigin && !turn.rootWork && !turn.settled) {
|
|
590
|
+
turn.result = { subtype: 'completed', text: BACKGROUND_QUESTION_NOTE };
|
|
591
|
+
turn.settle();
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
}
|
|
354
595
|
// The clock restarts from the answer, not from before the wait.
|
|
355
596
|
touch();
|
|
356
597
|
activeTurn?.armStall?.();
|
|
598
|
+
// The answer re-arms the relay immediately, so a message queued while
|
|
599
|
+
// the card was open steers into the resumed turn within one round trip.
|
|
600
|
+
syncDeliveryReadiness();
|
|
357
601
|
}
|
|
358
602
|
}
|
|
359
603
|
|
|
360
604
|
/**
|
|
361
|
-
* Hold an interactive callback until
|
|
605
|
+
* Hold an interactive callback until its turn's queue row exists.
|
|
362
606
|
*
|
|
363
607
|
* A continuation becomes the active turn synchronously but gets its row
|
|
364
608
|
* asynchronously, and the runtime can block on `user_input.requested` (or an
|
|
365
609
|
* ask-mode approval) in that gap — the question bridge would then see no
|
|
366
|
-
*
|
|
367
|
-
*
|
|
610
|
+
* row and degrade to an unsupported answer / local rejection for a card
|
|
611
|
+
* that was milliseconds from having a real row id. Bounded by the
|
|
368
612
|
* registration timeout (the registration itself can hang on a dead relay);
|
|
369
613
|
* on expiry — or on a registration that gave up (`rowReady` → false) — the
|
|
370
614
|
* caller proceeds and degrades exactly as before. Delivered turns are
|
|
371
615
|
* registered from birth and skip straight through.
|
|
372
616
|
*/
|
|
373
|
-
async function awaitInteractiveRow() {
|
|
374
|
-
const turn = activeTurn;
|
|
617
|
+
async function awaitInteractiveRow(turn) {
|
|
375
618
|
if (!turn || turn.registered || !turn.rowReady) return;
|
|
376
619
|
let timer = null;
|
|
377
620
|
await Promise.race([
|
|
@@ -537,19 +780,89 @@ export function createCopilotSdkSessionRunner({
|
|
|
537
780
|
}
|
|
538
781
|
|
|
539
782
|
/**
|
|
540
|
-
* Publish an action
|
|
783
|
+
* Publish an action onto the row that owns the runtime's current run, or
|
|
784
|
+
* hold it if the turn has no queue row yet.
|
|
541
785
|
*
|
|
542
786
|
* The single gate every producer goes through. A continuation's row is
|
|
543
787
|
* created asynchronously, and a POST carrying `messageId: null` is not merely
|
|
544
788
|
* useless — the relay's activity/stream routes key on it, so it would be
|
|
545
789
|
* attributed to nothing at all.
|
|
790
|
+
*
|
|
791
|
+
* One turn can answer several rows in sequence (the primary, then a message
|
|
792
|
+
* pushed mid-turn that the runtime ran as its own `delivery:"queued"` run).
|
|
793
|
+
* Root-level traffic — stream, thoughts, activity, subagent lanes — goes to
|
|
794
|
+
* whichever row owns the run in progress (`turn.currentOwner`); the stream
|
|
795
|
+
* text is re-cut to that row's own segments so a second run's prose never
|
|
796
|
+
* appears under the first row.
|
|
546
797
|
*/
|
|
547
798
|
async function emitAction(turn, action) {
|
|
548
799
|
if (!turn.registered) {
|
|
549
800
|
turn.bufferedActions.push(action);
|
|
550
801
|
return;
|
|
551
802
|
}
|
|
552
|
-
await
|
|
803
|
+
await dispatchOwned(turn, action);
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
async function dispatchOwned(turn, action) {
|
|
807
|
+
const owner = turn.currentOwner || turn.primaryEntry;
|
|
808
|
+
if (!owner || owner.settled) {
|
|
809
|
+
// Nothing left to publish into (every row this turn owned has settled);
|
|
810
|
+
// the text still lands in the runtime's own transcript.
|
|
811
|
+
return;
|
|
812
|
+
}
|
|
813
|
+
if (showsWork(action)) owner.showedWork = true;
|
|
814
|
+
if (action.channel === 'stream' && !action.payload?.subagentRunId) {
|
|
815
|
+
// Re-cut to the owner's own segments, and gated against what THAT row
|
|
816
|
+
// last received (the normalizer gated against the whole interaction).
|
|
817
|
+
const text = textFor(turn, owner);
|
|
818
|
+
if (!shouldEmitStreamUpdate(text, owner.state.lastStreamedText)) return;
|
|
819
|
+
await dispatchAction(owner.message, { channel: 'stream', payload: { ...action.payload, text } }, owner.state);
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
await dispatchAction(owner.message, action, owner.state);
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
/**
|
|
826
|
+
* Which row a normalizer segment's text belongs to. A segment nobody
|
|
827
|
+
* claimed (text produced before any prompt boundary — the answer to a
|
|
828
|
+
* prompt carried over from a previous interaction, or a turn the runtime
|
|
829
|
+
* answered without echoing a `user.message` at all) belongs to the next
|
|
830
|
+
* claimed segment's row, failing that to the turn's own row.
|
|
831
|
+
*/
|
|
832
|
+
function segmentOwner(turn, index) {
|
|
833
|
+
const owners = turn.segmentOwners;
|
|
834
|
+
if (owners[index]) return owners[index];
|
|
835
|
+
for (let j = index + 1; j < owners.length; j += 1) {
|
|
836
|
+
if (owners[j]) return owners[j];
|
|
837
|
+
}
|
|
838
|
+
return turn.primaryEntry;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
function textFor(turn, entry, segmentTexts = null) {
|
|
842
|
+
const count = Math.max(turn.normalizer.promptCount(), turn.segmentOwners.length);
|
|
843
|
+
const parts = [];
|
|
844
|
+
for (let index = 0; index < count; index += 1) {
|
|
845
|
+
if (segmentOwner(turn, index) !== entry) continue;
|
|
846
|
+
const text = String(segmentTexts?.[index] ?? turn.normalizer.segmentText(index) ?? '');
|
|
847
|
+
if (text.trim()) parts.push(text);
|
|
848
|
+
}
|
|
849
|
+
return parts.join('\n\n');
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
/**
|
|
853
|
+
* The final reply text for `entry`: its segments, with the task-completion
|
|
854
|
+
* summary taking precedence on the last one (the normalizer's
|
|
855
|
+
* `result.segmentTexts` already applies that rule; before this the runner
|
|
856
|
+
* read `segmentText` directly and lost the summary — the "Cha…" fragment
|
|
857
|
+
* case of 2026-09-12). A turn that opened no segment at all falls back to
|
|
858
|
+
* the whole composed text for its own row, which is strictly better than
|
|
859
|
+
* publishing nothing.
|
|
860
|
+
*/
|
|
861
|
+
function finalTextFor(turn, entry, result) {
|
|
862
|
+
const segmentTexts = Array.isArray(result?.segmentTexts) ? result.segmentTexts : null;
|
|
863
|
+
const text = textFor(turn, entry, segmentTexts).trim();
|
|
864
|
+
if (text || entry !== turn.primaryEntry) return text;
|
|
865
|
+
return String(result?.text || entry.state.lastStreamedText || '').trim();
|
|
553
866
|
}
|
|
554
867
|
|
|
555
868
|
async function publishFinalStream(message, text) {
|
|
@@ -563,26 +876,54 @@ export function createCopilotSdkSessionRunner({
|
|
|
563
876
|
}).catch(() => {});
|
|
564
877
|
}
|
|
565
878
|
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
879
|
+
/**
|
|
880
|
+
* Publish a row's outcome. Returns 'published' | 'stale' | 'failed' |
|
|
881
|
+
* 'requeued' so the settle paths can tell a landed marker from a lost one.
|
|
882
|
+
*
|
|
883
|
+
* `kind` ('folded' | 'stopped') stamps the settle stubs of a steered row the
|
|
884
|
+
* runtime consumed without answering on its own row; `consumedSteerIds`
|
|
885
|
+
* names the steers folded into the turn this response finishes, so the
|
|
886
|
+
* relay marks them consumed in the same fenced write. A response carrying a
|
|
887
|
+
* kind is never requeued on failure (the prompt was consumed — re-running it
|
|
888
|
+
* is the one thing that must not happen); `requeueOnFailure: false` opts an
|
|
889
|
+
* ordinary response out of the requeue too.
|
|
890
|
+
*/
|
|
891
|
+
async function publishResponse(message, {
|
|
892
|
+
text, model, terminalError = null, modelOrigin, kind = null, consumedSteerIds = null,
|
|
893
|
+
requeueOnFailure = true,
|
|
894
|
+
}) {
|
|
895
|
+
const consumed = Array.isArray(consumedSteerIds)
|
|
896
|
+
? consumedSteerIds
|
|
897
|
+
.map((entry) => ({ id: String(entry?.id || '').trim(), attemptId: entry?.attemptId || null }))
|
|
898
|
+
.filter((entry) => entry.id)
|
|
899
|
+
: [];
|
|
900
|
+
try {
|
|
901
|
+
await api('POST', '/api/response', {
|
|
902
|
+
messageId: message.id,
|
|
903
|
+
conversationId: message.conversationId,
|
|
904
|
+
text: String(text || ''),
|
|
905
|
+
model: model || null,
|
|
906
|
+
modelOrigin: modelOrigin
|
|
907
|
+
|| (String(message?.model || '').trim().toLowerCase() === 'auto' ? 'auto' : 'manual'),
|
|
908
|
+
...(terminalError ? { terminalError } : {}),
|
|
909
|
+
...(kind ? { kind } : {}),
|
|
910
|
+
...(consumed.length ? { consumedSteerIds: consumed } : {}),
|
|
911
|
+
...attemptFields(message),
|
|
912
|
+
});
|
|
913
|
+
return 'published';
|
|
914
|
+
} catch (error) {
|
|
577
915
|
// A stale_attempt 409 means the row already moved on to another attempt
|
|
578
916
|
// (requeued and re-delivered); the requeue would 409 the same way, and
|
|
579
917
|
// the newer attempt owes the row its answer — drop this one.
|
|
580
918
|
if (isStaleAttemptError(error)) {
|
|
581
919
|
dbg('response refused as stale_attempt; dropping', message.id);
|
|
582
|
-
return;
|
|
920
|
+
return 'stale';
|
|
583
921
|
}
|
|
584
|
-
|
|
585
|
-
|
|
922
|
+
if (kind || !requeueOnFailure) return 'failed';
|
|
923
|
+
const requeued = await api('POST', '/api/requeue', { messageId: message.id, ...attemptFields(message) })
|
|
924
|
+
.then(() => true, () => false);
|
|
925
|
+
return requeued ? 'requeued' : 'failed';
|
|
926
|
+
}
|
|
586
927
|
}
|
|
587
928
|
|
|
588
929
|
function terminalErrorRecord(message, classified) {
|
|
@@ -596,6 +937,117 @@ export function createCopilotSdkSessionRunner({
|
|
|
596
937
|
};
|
|
597
938
|
}
|
|
598
939
|
|
|
940
|
+
// ------------------------------------------------- at-most-once settling --
|
|
941
|
+
|
|
942
|
+
/**
|
|
943
|
+
* Claim a consumed row as settling — synchronously, BEFORE anything stops
|
|
944
|
+
* reporting it as live, so no heartbeat can see it unowned. `variant` picks
|
|
945
|
+
* the terminal wording should its settle never land.
|
|
946
|
+
*/
|
|
947
|
+
function claimSettling(message, variant = 'folded') {
|
|
948
|
+
const id = String(message?.id || '').trim();
|
|
949
|
+
if (id && !settlingMessages.has(id)) settlingMessages.set(id, { message, variant });
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
/** Release a settled row — unless it was handed to the relay to fail. */
|
|
953
|
+
function releaseSettling(id) {
|
|
954
|
+
const key = String(id || '').trim();
|
|
955
|
+
if (!settleFailedTerminals.has(key)) settlingMessages.delete(key);
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
/**
|
|
959
|
+
* Mark rows whose prompt the runtime has folded into a running turn as
|
|
960
|
+
* consumed on the relay (the moment `user.message{delivery:"steering"}`
|
|
961
|
+
* confirms it). Best-effort: the worker's own settle still owns these rows
|
|
962
|
+
* while it lives; the mark only protects them when it does not — a crash
|
|
963
|
+
* before the settle marker lands must fail the row, never re-run it.
|
|
964
|
+
*/
|
|
965
|
+
function markConsumed(conversationId, entries = [], { consumed = true } = {}) {
|
|
966
|
+
const rows = entries
|
|
967
|
+
.map((entry) => ({ id: String(entry?.id || '').trim(), attemptId: entry?.attemptId || null }))
|
|
968
|
+
.filter((entry) => entry.id);
|
|
969
|
+
if (!rows.length) return Promise.resolve();
|
|
970
|
+
const previous = Promise.allSettled(rows.map((row) => consumedMarkChains.get(row.id)).filter(Boolean));
|
|
971
|
+
const post = previous.then(() => api('POST', '/api/queue-consumed', {
|
|
972
|
+
conversationId: String(conversationId || sdkSessionId || ''),
|
|
973
|
+
entries: rows,
|
|
974
|
+
...(consumed ? {} : { consumed: false }),
|
|
975
|
+
})).catch((error) => {
|
|
976
|
+
dbg('consumed mark failed', rows.map((entry) => entry.id).join(','), error?.message || String(error));
|
|
977
|
+
});
|
|
978
|
+
for (const row of rows) consumedMarkChains.set(row.id, post);
|
|
979
|
+
post.then(() => {
|
|
980
|
+
for (const row of rows) if (consumedMarkChains.get(row.id) === post) consumedMarkChains.delete(row.id);
|
|
981
|
+
});
|
|
982
|
+
return post;
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
/** Heartbeat payload: rows whose terminal failure the relay must post. */
|
|
986
|
+
function getSettleFailed() {
|
|
987
|
+
return [...settleFailedTerminals.entries()].map(([id, terminalError]) => ({
|
|
988
|
+
id,
|
|
989
|
+
attemptId: settlingMessages.get(id)?.message?.attemptId || null,
|
|
990
|
+
terminalError,
|
|
991
|
+
}));
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
function acknowledgeSettleFailed(ids = []) {
|
|
995
|
+
for (const id of ids) {
|
|
996
|
+
const key = String(id || '').trim();
|
|
997
|
+
if (!settleFailedTerminals.delete(key)) continue;
|
|
998
|
+
settlingMessages.delete(key);
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
const sleep = (ms) => new Promise((resolve) => { const t = setTimeout(resolve, ms); t.unref?.(); });
|
|
1003
|
+
|
|
1004
|
+
/**
|
|
1005
|
+
* Save the settle marker of a steered message the runtime already consumed
|
|
1006
|
+
* — at most once. The row stays in `settlingMessages`, and so in every
|
|
1007
|
+
* heartbeat, until the relay commits the marker. A marker that cannot be
|
|
1008
|
+
* saved within the retry budget ends as a terminal failure — never a
|
|
1009
|
+
* requeue — and if even that cannot be posted, the heartbeat hands the row
|
|
1010
|
+
* to the relay to fail. Ported from the Claude worker; same guarantees.
|
|
1011
|
+
*/
|
|
1012
|
+
async function publishSettleMarker(message, { text, model, kind, variant = 'folded' }) {
|
|
1013
|
+
const id = String(message?.id || '').trim();
|
|
1014
|
+
if (!id) return;
|
|
1015
|
+
claimSettling(message, variant);
|
|
1016
|
+
try {
|
|
1017
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
1018
|
+
const outcome = await publishResponse(message, { text, model, kind });
|
|
1019
|
+
if (outcome !== 'failed') return;
|
|
1020
|
+
if (attempt >= settleRetryDelaysMs.length) break;
|
|
1021
|
+
await sleep(settleRetryDelaysMs[attempt]);
|
|
1022
|
+
}
|
|
1023
|
+
dbg('settle marker could not be saved; failing the row terminally', id);
|
|
1024
|
+
const terminalError = buildSteerSettleFailure(message, { variant, agentLabel: SETTLE_AGENT_LABEL });
|
|
1025
|
+
const retryMs = Math.max(100, Number(settleRetryDelaysMs.at(-1)) || 0);
|
|
1026
|
+
for (let attempt = 0; attempt < Math.max(1, maxTerminalSettleAttempts); attempt += 1) {
|
|
1027
|
+
if (attempt) await sleep(retryMs);
|
|
1028
|
+
const outcome = await publishResponse(message, {
|
|
1029
|
+
text: terminalError.message,
|
|
1030
|
+
model,
|
|
1031
|
+
terminalError,
|
|
1032
|
+
requeueOnFailure: false,
|
|
1033
|
+
});
|
|
1034
|
+
if (outcome !== 'failed') return;
|
|
1035
|
+
// Second channel: the requeue route fails a row outright when handed
|
|
1036
|
+
// a terminal error.
|
|
1037
|
+
const failedViaRequeue = await api('POST', '/api/requeue', {
|
|
1038
|
+
messageId: id,
|
|
1039
|
+
terminalError,
|
|
1040
|
+
...attemptFields(message),
|
|
1041
|
+
}).then(() => true, (error) => isStaleAttemptError(error));
|
|
1042
|
+
if (failedViaRequeue) return;
|
|
1043
|
+
}
|
|
1044
|
+
dbg('terminal settle could not be posted; handing it to the heartbeat', id);
|
|
1045
|
+
settleFailedTerminals.set(id, terminalError);
|
|
1046
|
+
} finally {
|
|
1047
|
+
releaseSettling(id);
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
|
|
599
1051
|
/**
|
|
600
1052
|
* Per-turn usage capture, recorded on the runner.
|
|
601
1053
|
*
|
|
@@ -731,16 +1183,34 @@ export function createCopilotSdkSessionRunner({
|
|
|
731
1183
|
return;
|
|
732
1184
|
}
|
|
733
1185
|
touch();
|
|
1186
|
+
observeRecalls(event);
|
|
734
1187
|
observeBackgroundShells(event);
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
//
|
|
739
|
-
|
|
740
|
-
//
|
|
741
|
-
//
|
|
742
|
-
|
|
1188
|
+
observeBackgroundAgents(event);
|
|
1189
|
+
observeCompaction(event);
|
|
1190
|
+
observeQueueLane(event);
|
|
1191
|
+
// A background subagent's spawn emits a SESSION-level
|
|
1192
|
+
// `session.model_change{source:"agent"}` for the subagent's own model
|
|
1193
|
+
// (live-probed on 1.0.88). It is not the main thread's model: feeding it
|
|
1194
|
+
// to the switcher would confirm a pending switch with the wrong model and
|
|
1195
|
+
// move the catalog's currentModel.
|
|
1196
|
+
const agentModelChange = event?.type === 'session.model_change'
|
|
1197
|
+
&& String(event?.data?.source || '').trim().toLowerCase() === 'agent';
|
|
1198
|
+
if (!agentModelChange) {
|
|
1199
|
+
// Keeps the confirmed-model tracking truthful for switches this worker
|
|
1200
|
+
// did not ask for, and resolves the bounded wait on a deferred switch's
|
|
1201
|
+
// drain. Behind the replay gate on purpose: a historical
|
|
1202
|
+
// `session.model_change` must not confirm a pending switch.
|
|
1203
|
+
modelSwitch.observeEvent(event);
|
|
1204
|
+
// A LIVE model change moves the catalog's currentModel; replays cannot
|
|
1205
|
+
// get here (the gate above), so this cannot re-publish days-old state.
|
|
1206
|
+
if (event?.type === 'session.model_change') publishModelSnapshot('model-change');
|
|
1207
|
+
}
|
|
743
1208
|
let turn = activeTurn;
|
|
1209
|
+
if (!turn) {
|
|
1210
|
+
// A prompt this worker pushed whose run the runtime opened after its
|
|
1211
|
+
// turn had already settled is that row's own turn, not a continuation.
|
|
1212
|
+
turn = adoptOrphanedRun(event);
|
|
1213
|
+
}
|
|
744
1214
|
if (!turn) {
|
|
745
1215
|
// The runtime started work with no row open. Anything that is not the
|
|
746
1216
|
// START of work (terminators, connection bookkeeping) stays a no-op:
|
|
@@ -777,15 +1247,396 @@ export function createCopilotSdkSessionRunner({
|
|
|
777
1247
|
* backgroundWorkHoldsRuntime. Advisory: a failed post never disturbs events.
|
|
778
1248
|
*/
|
|
779
1249
|
function publishBackgroundShellTasks() {
|
|
780
|
-
|
|
1250
|
+
publishBackgroundTasks({ immediate: true });
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
/** The panel rows for the shell tracker's set — the path without `rpc.tasks`. */
|
|
1254
|
+
function shellTrackerCards() {
|
|
1255
|
+
return backgroundShells.live().map((shell) => ({
|
|
781
1256
|
taskId: shell.shellId,
|
|
782
1257
|
taskType: 'local_bash',
|
|
783
1258
|
description: shell.description || 'Detached shell',
|
|
784
1259
|
startedAt: shell.startedAt || null,
|
|
785
1260
|
stoppable: false,
|
|
786
1261
|
}));
|
|
787
|
-
|
|
788
|
-
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
/**
|
|
1265
|
+
* The panel rows for the runtime's task registry: background agents and
|
|
1266
|
+
* shells still running, each with what the events added. `stoppable` is
|
|
1267
|
+
* true because `rpc.tasks.cancel` is present whenever this path is taken.
|
|
1268
|
+
*/
|
|
1269
|
+
function registryCards() {
|
|
1270
|
+
const cards = [];
|
|
1271
|
+
for (const task of backgroundTasks.values()) {
|
|
1272
|
+
if (task.status !== 'running') continue;
|
|
1273
|
+
const startedAt = Date.parse(String(task.startedAt || '')) || null;
|
|
1274
|
+
if (task.type === 'shell') {
|
|
1275
|
+
cards.push({
|
|
1276
|
+
taskId: task.id,
|
|
1277
|
+
taskType: 'local_bash',
|
|
1278
|
+
description: task.description || task.command || 'Background shell',
|
|
1279
|
+
startedAt,
|
|
1280
|
+
stoppable: true,
|
|
1281
|
+
...(task.progressSummary ? { summary: task.progressSummary } : {}),
|
|
1282
|
+
});
|
|
1283
|
+
continue;
|
|
1284
|
+
}
|
|
1285
|
+
if (task.type !== 'agent') continue;
|
|
1286
|
+
const detail = agentDetails.get(task.id) || {};
|
|
1287
|
+
const model = task.resolvedModel || task.model || detail.model || '';
|
|
1288
|
+
const inherited = !model;
|
|
1289
|
+
cards.push({
|
|
1290
|
+
taskId: task.id,
|
|
1291
|
+
taskType: 'local_agent',
|
|
1292
|
+
description: task.description || task.displayName || detail.displayName || 'Background agent',
|
|
1293
|
+
startedAt: startedAt || detail.startedAt || null,
|
|
1294
|
+
stoppable: true,
|
|
1295
|
+
subagentType: task.agentType || detail.agentType || null,
|
|
1296
|
+
model: model || appliedModel() || defaultModel || null,
|
|
1297
|
+
modelInherited: inherited,
|
|
1298
|
+
...(detail.lastToolCall ? { lastToolCall: detail.lastToolCall } : {}),
|
|
1299
|
+
...(Number.isFinite(detail.totalTokens) && detail.totalTokens > 0 ? { totalTokens: detail.totalTokens } : {}),
|
|
1300
|
+
...(task.progressSummary ? { summary: task.progressSummary } : {}),
|
|
1301
|
+
});
|
|
1302
|
+
}
|
|
1303
|
+
return cards;
|
|
1304
|
+
}
|
|
1305
|
+
|
|
1306
|
+
/**
|
|
1307
|
+
* Mirror the live background work into the relay's task panel (REPLACE
|
|
1308
|
+
* semantics, the same route the Claude worker publishes on). Set changes
|
|
1309
|
+
* publish at once; enrichment-only changes (a tool call, tokens) ride a
|
|
1310
|
+
* trailing throttle. Advisory: a failed post never disturbs events.
|
|
1311
|
+
*/
|
|
1312
|
+
function publishBackgroundTasks({ immediate = false } = {}) {
|
|
1313
|
+
const run = () => {
|
|
1314
|
+
if (taskPublishTimer) {
|
|
1315
|
+
clearTimeout(taskPublishTimer);
|
|
1316
|
+
taskPublishTimer = null;
|
|
1317
|
+
}
|
|
1318
|
+
const tasks = sessionRpc.tasks ? registryCards() : shellTrackerCards();
|
|
1319
|
+
const signature = JSON.stringify(tasks);
|
|
1320
|
+
if (signature === lastPublishedTasksSignature) return;
|
|
1321
|
+
lastPublishedTasksSignature = signature;
|
|
1322
|
+
api('POST', '/api/background-tasks', { conversationId: sdkSessionId, tasks })
|
|
1323
|
+
.catch((error) => dbg('background task publish failed', error?.message || String(error)));
|
|
1324
|
+
};
|
|
1325
|
+
if (immediate || !(taskPublishThrottleMs > 0)) {
|
|
1326
|
+
run();
|
|
1327
|
+
return;
|
|
1328
|
+
}
|
|
1329
|
+
if (taskPublishTimer) return;
|
|
1330
|
+
taskPublishTimer = setTimeout(run, taskPublishThrottleMs);
|
|
1331
|
+
taskPublishTimer.unref?.();
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
// ------------------------------------------------------- background tasks --
|
|
1335
|
+
|
|
1336
|
+
/**
|
|
1337
|
+
* What the event stream knows about background agents that the registry
|
|
1338
|
+
* does not: spawn details, the live tool call, a running token count, and
|
|
1339
|
+
* the completion heralds. `session.background_tasks_changed` itself is
|
|
1340
|
+
* empty and only says "re-read the registry".
|
|
1341
|
+
*/
|
|
1342
|
+
function observeBackgroundAgents(event) {
|
|
1343
|
+
const type = String(event?.type || '');
|
|
1344
|
+
const data = event?.data && typeof event.data === 'object' ? event.data : {};
|
|
1345
|
+
const agentId = String(event?.agentId || '').trim();
|
|
1346
|
+
if (type === 'session.background_tasks_changed') {
|
|
1347
|
+
scheduleTaskRefresh();
|
|
1348
|
+
return;
|
|
1349
|
+
}
|
|
1350
|
+
if (type === 'subagent.started' && String(data.executionMode || '').trim().toLowerCase() === 'background' && agentId) {
|
|
1351
|
+
// Kept for the life of the session (a finished agent's late events must
|
|
1352
|
+
// still be recognised as background), bounded so a long session cannot
|
|
1353
|
+
// grow it forever.
|
|
1354
|
+
while (agentDetails.size >= 500) agentDetails.delete(agentDetails.keys().next().value);
|
|
1355
|
+
agentDetails.set(agentId, {
|
|
1356
|
+
displayName: String(data.agentDisplayName || data.agentName || '').trim(),
|
|
1357
|
+
agentType: String(data.agentType || '').trim(),
|
|
1358
|
+
model: String(data.model || '').trim(),
|
|
1359
|
+
lastToolCall: '',
|
|
1360
|
+
totalTokens: 0,
|
|
1361
|
+
startedAt: Date.now(),
|
|
1362
|
+
});
|
|
1363
|
+
scheduleTaskRefresh();
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
const settledAgent = settledAgentFromNotification(event);
|
|
1367
|
+
if (settledAgent) {
|
|
1368
|
+
dbg('background agent settled', settledAgent.agentId, settledAgent.status);
|
|
1369
|
+
// The runtime is about to re-invoke the model off the back of this
|
|
1370
|
+
// (`read_agent` + a reply): pin the runtime for the continuation, and
|
|
1371
|
+
// carry a line into the turn it triggers when no turn is open to take
|
|
1372
|
+
// it. Same shape as a settled shell's herald.
|
|
1373
|
+
continuationDueSince = Date.now();
|
|
1374
|
+
const note = describeSettledAgent(settledAgent);
|
|
1375
|
+
if (note && !activeTurn && pendingActivities.length < MAX_PENDING_ACTIVITIES) {
|
|
1376
|
+
pendingActivities.push({ text: note, reportsFailure: settledAgent.status === 'failed' });
|
|
1377
|
+
}
|
|
1378
|
+
scheduleTaskRefresh();
|
|
1379
|
+
return;
|
|
1380
|
+
}
|
|
1381
|
+
if (agentId && agentDetails.has(agentId)) {
|
|
1382
|
+
const detail = agentDetails.get(agentId);
|
|
1383
|
+
if (type === 'tool.execution_start') {
|
|
1384
|
+
detail.lastToolCall = formatToolActivityText(data.toolName, data.arguments);
|
|
1385
|
+
publishBackgroundTasks();
|
|
1386
|
+
} else if (type === 'assistant.usage') {
|
|
1387
|
+
const used = (Number(data.inputTokens) || 0) + (Number(data.outputTokens) || 0);
|
|
1388
|
+
if (used > 0) {
|
|
1389
|
+
detail.totalTokens += used;
|
|
1390
|
+
publishBackgroundTasks();
|
|
1391
|
+
}
|
|
1392
|
+
} else if (type === 'subagent.completed' || type === 'subagent.failed') {
|
|
1393
|
+
const total = Number(data.totalTokens);
|
|
1394
|
+
if (Number.isFinite(total) && total > 0) detail.totalTokens = total;
|
|
1395
|
+
scheduleTaskRefresh();
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
function scheduleTaskRefresh() {
|
|
1401
|
+
if (!sessionRpc.tasks || taskRefreshTimer) return;
|
|
1402
|
+
taskRefreshTimer = setTimeout(() => {
|
|
1403
|
+
taskRefreshTimer = null;
|
|
1404
|
+
taskRefreshChain = taskRefreshChain.then(() => refreshBackgroundTasks()).catch(() => {});
|
|
1405
|
+
}, taskRefreshMs);
|
|
1406
|
+
taskRefreshTimer.unref?.();
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
/**
|
|
1410
|
+
* Re-read the runtime's task registry. Running agents and shells become
|
|
1411
|
+
* cards (with a progress line when the runtime has one); finished entries
|
|
1412
|
+
* are removed from the runtime's tracking so the registry cannot grow for
|
|
1413
|
+
* the life of the session. A refresh that fails leaves the last read in
|
|
1414
|
+
* place — a stale card beats a vanished one.
|
|
1415
|
+
*/
|
|
1416
|
+
async function refreshBackgroundTasks() {
|
|
1417
|
+
const target = session;
|
|
1418
|
+
if (!target || !sessionRpc.tasks) return;
|
|
1419
|
+
lastTaskRefreshAt = Date.now();
|
|
1420
|
+
let listed;
|
|
1421
|
+
try {
|
|
1422
|
+
listed = await target.rpc.tasks.list();
|
|
1423
|
+
} catch (error) {
|
|
1424
|
+
dbg('tasks.list failed', error?.message || String(error));
|
|
1425
|
+
return;
|
|
1426
|
+
}
|
|
1427
|
+
if (session !== target) return;
|
|
1428
|
+
const next = new Map();
|
|
1429
|
+
for (const task of Array.isArray(listed?.tasks) ? listed.tasks : []) {
|
|
1430
|
+
const id = String(task?.id || '').trim();
|
|
1431
|
+
if (!id || abandonedTasks.has(id)) continue;
|
|
1432
|
+
const status = String(task?.status || '').trim().toLowerCase();
|
|
1433
|
+
// Only running tasks are live. 'idle' (a multi-turn agent waiting for
|
|
1434
|
+
// messages) holds nothing and shows nowhere until it is messaged
|
|
1435
|
+
// (roadmap: tasks.sendMessage); finished entries are left to the
|
|
1436
|
+
// runtime's own tracking — removing them here could race the root
|
|
1437
|
+
// agent's `read_agent` follow-up that the completion notification
|
|
1438
|
+
// triggers.
|
|
1439
|
+
if (status === 'running') {
|
|
1440
|
+
next.set(id, { ...task, id, status, progressSummary: backgroundTasks.get(id)?.progressSummary || '' });
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
// Progress lines for the live set: the agent's latest intent / last
|
|
1444
|
+
// activity, a shell's last output line. One RPC per live task per refresh
|
|
1445
|
+
// (refreshes are debounced), best effort.
|
|
1446
|
+
if (typeof target.rpc.tasks.getProgress === 'function') {
|
|
1447
|
+
for (const task of next.values()) {
|
|
1448
|
+
try {
|
|
1449
|
+
const progress = (await target.rpc.tasks.getProgress({ id: task.id }))?.progress || null;
|
|
1450
|
+
task.progressSummary = summarizeTaskProgress(progress);
|
|
1451
|
+
} catch { /* a card without a progress line */ }
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
if (session !== target) return;
|
|
1455
|
+
const setChanged = next.size !== backgroundTasks.size
|
|
1456
|
+
|| [...next.keys()].some((id) => !backgroundTasks.has(id));
|
|
1457
|
+
backgroundTasks = next;
|
|
1458
|
+
publishBackgroundTasks({ immediate: setChanged });
|
|
1459
|
+
// The lifecycle poll (every `lifecyclePollMs`) picks the new set up; a
|
|
1460
|
+
// re-evaluation here would re-schedule a refresh and spin.
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1463
|
+
function summarizeTaskProgress(progress) {
|
|
1464
|
+
if (!progress || typeof progress !== 'object') return '';
|
|
1465
|
+
if (progress.type === 'agent') {
|
|
1466
|
+
const intent = String(progress.latestIntent || '').trim();
|
|
1467
|
+
if (intent) return intent.slice(0, 300);
|
|
1468
|
+
const lines = Array.isArray(progress.recentActivity) ? progress.recentActivity : [];
|
|
1469
|
+
const last = lines.length ? String(lines[lines.length - 1]?.message || '').trim() : '';
|
|
1470
|
+
return last.slice(0, 300);
|
|
1471
|
+
}
|
|
1472
|
+
if (progress.type === 'shell') {
|
|
1473
|
+
const output = String(progress.recentOutput || '').trim();
|
|
1474
|
+
if (!output || output === '(no output yet)') return '';
|
|
1475
|
+
const lines = output.split('\n').map((line) => line.trim()).filter(Boolean);
|
|
1476
|
+
return (lines[lines.length - 1] || '').slice(0, 300);
|
|
1477
|
+
}
|
|
1478
|
+
return '';
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
/** The live tasks from the registry (running agents + shells). */
|
|
1482
|
+
function runningBackgroundTasks() {
|
|
1483
|
+
return [...backgroundTasks.values()].filter((task) => task.status === 'running');
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
/**
|
|
1487
|
+
* Stop one background task (the panel's per-card Stop, or a subagent lane's
|
|
1488
|
+
* Stop for a background agent). Returns whether the runtime cancelled it.
|
|
1489
|
+
*/
|
|
1490
|
+
async function stopBackgroundTask(taskId) {
|
|
1491
|
+
const id = String(taskId || '').trim();
|
|
1492
|
+
if (!id || !session || !sessionRpc.tasks) return false;
|
|
1493
|
+
try {
|
|
1494
|
+
const outcome = await session.rpc.tasks.cancel({ id });
|
|
1495
|
+
const cancelled = outcome?.cancelled === true;
|
|
1496
|
+
dbg(cancelled ? 'background task cancelled' : 'background task cancel refused', id);
|
|
1497
|
+
if (cancelled) scheduleTaskRefresh();
|
|
1498
|
+
return cancelled;
|
|
1499
|
+
} catch (error) {
|
|
1500
|
+
dbg('tasks.cancel failed', id, error?.message || String(error));
|
|
1501
|
+
return false;
|
|
1502
|
+
}
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
|
|
1507
|
+
/**
|
|
1508
|
+
* The compaction hold. A message pushed while the runtime is compacting
|
|
1509
|
+
* would be injected into the compaction's replay, so steering is held from
|
|
1510
|
+
* `session.compaction_start` to `_complete` (the runtime queues messages
|
|
1511
|
+
* sent during a compaction itself, but holding them relay-side keeps the
|
|
1512
|
+
* bubble cancellable and the composer truthful — the Claude rule).
|
|
1513
|
+
*/
|
|
1514
|
+
function observeCompaction(event) {
|
|
1515
|
+
const type = String(event?.type || '');
|
|
1516
|
+
if (event?.agentId) return;
|
|
1517
|
+
if (type === 'session.compaction_start') {
|
|
1518
|
+
compactingSince = Date.now();
|
|
1519
|
+
syncDeliveryReadiness();
|
|
1520
|
+
} else if (type === 'session.compaction_complete') {
|
|
1521
|
+
compactingSince = 0;
|
|
1522
|
+
syncDeliveryReadiness();
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
function isCompacting() {
|
|
1527
|
+
if (!compactingSince) return false;
|
|
1528
|
+
if (Date.now() - compactingSince < compactionStaleMs) return true;
|
|
1529
|
+
dbg('compaction hold went stale; releasing it');
|
|
1530
|
+
compactingSince = 0;
|
|
1531
|
+
return false;
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
/**
|
|
1535
|
+
* Keep `queuedLane` current. `pending_messages.modified` carries no payload
|
|
1536
|
+
* (it only says "the queue changed"), so it schedules a debounced
|
|
1537
|
+
* `rpc.queue.pendingItems()` read. What matters here is which pushed rows
|
|
1538
|
+
* are still in the QUEUED lane: those the runtime has not consumed and the
|
|
1539
|
+
* client may still cancel (`cancellableIds` on the steering snapshot).
|
|
1540
|
+
*/
|
|
1541
|
+
function observeQueueLane(event) {
|
|
1542
|
+
if (String(event?.type || '') !== 'pending_messages.modified') return;
|
|
1543
|
+
scheduleQueueLaneRefresh();
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
/** Un-steer's synchronous view of the runtime taking a prompt (see `recallWatches`). */
|
|
1547
|
+
function observeRecalls(event) {
|
|
1548
|
+
if (String(event?.type || '') !== 'user.message' || event?.agentId) return;
|
|
1549
|
+
const runtimeId = String(event?.data?.messageId || '').trim();
|
|
1550
|
+
if (!runtimeId) return;
|
|
1551
|
+
const watch = recallWatches.get(runtimeId);
|
|
1552
|
+
if (watch) watch.taken = true;
|
|
1553
|
+
const recalledRow = recalledRuntimeIds.get(runtimeId);
|
|
1554
|
+
if (recalledRow) {
|
|
1555
|
+
dbg('UN-STEER MISMATCH: a prompt recalled and cancelled is running after all; the remove took another message',
|
|
1556
|
+
recalledRow, runtimeId);
|
|
1557
|
+
}
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
function scheduleQueueLaneRefresh() {
|
|
1561
|
+
if (!sessionRpc.queue || queueLaneTimer) return;
|
|
1562
|
+
queueLaneTimer = setTimeout(() => {
|
|
1563
|
+
queueLaneTimer = null;
|
|
1564
|
+
queueLaneRefreshChain = queueLaneRefreshChain.then(() => refreshQueueLane()).catch(() => {});
|
|
1565
|
+
}, queueLaneRefreshMs);
|
|
1566
|
+
queueLaneTimer.unref?.();
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
async function refreshQueueLane() {
|
|
1570
|
+
const target = session;
|
|
1571
|
+
if (!target || !sessionRpc.queue) return;
|
|
1572
|
+
try {
|
|
1573
|
+
const snapshot = await target.rpc.queue.pendingItems();
|
|
1574
|
+
const next = new Map();
|
|
1575
|
+
for (const item of Array.isArray(snapshot?.items) ? snapshot.items : []) {
|
|
1576
|
+
const runtimeId = String(item?.messageId || '').trim();
|
|
1577
|
+
const itemId = String(item?.id || '').trim();
|
|
1578
|
+
if (runtimeId && itemId) next.set(runtimeId, itemId);
|
|
1579
|
+
}
|
|
1580
|
+
if (session !== target) return;
|
|
1581
|
+
queuedLane = next;
|
|
1582
|
+
queuedItemCount = Array.isArray(snapshot?.items) ? snapshot.items.length : 0;
|
|
1583
|
+
const steering = Array.isArray(snapshot?.steeringMessages) ? snapshot.steeringMessages.length : 0;
|
|
1584
|
+
const inFlight = Number(snapshot?.inFlightSteeringCount) || 0;
|
|
1585
|
+
waitingSteers = Math.max(0, steering - inFlight);
|
|
1586
|
+
} catch (error) {
|
|
1587
|
+
dbg('queue.pendingItems failed', error?.message || String(error));
|
|
1588
|
+
}
|
|
1589
|
+
noteCancellableChange();
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
/** Tell the worker when the recallable set changed (see onCancellableChange). */
|
|
1593
|
+
function noteCancellableChange() {
|
|
1594
|
+
const signature = cancellableIds().join(',');
|
|
1595
|
+
if (signature === lastCancellableSignature) return;
|
|
1596
|
+
lastCancellableSignature = signature;
|
|
1597
|
+
try {
|
|
1598
|
+
onCancellableChange();
|
|
1599
|
+
} catch (error) {
|
|
1600
|
+
dbg('cancellable listener failed', error?.message || String(error));
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
|
|
1604
|
+
/** Our pushed steers the runtime has not started on yet, in send order. */
|
|
1605
|
+
function unconsumedSteers() {
|
|
1606
|
+
const list = [];
|
|
1607
|
+
for (const entry of ownedEntries()) {
|
|
1608
|
+
if (entry.role === 'steer' && !entry.consumed && !entry.settled && entry.runtimeId) list.push(entry);
|
|
1609
|
+
}
|
|
1610
|
+
return list.sort((a, b) => (a.sendSeq || 0) - (b.sendSeq || 0));
|
|
1611
|
+
}
|
|
1612
|
+
|
|
1613
|
+
/**
|
|
1614
|
+
* How `entry` can still be pulled back out of the runtime, or null: by its
|
|
1615
|
+
* stable id when it sits in the queued lane, or with removeMostRecent when
|
|
1616
|
+
* it is the runtime's newest waiting message (our newest unconsumed steer,
|
|
1617
|
+
* at least one steer waiting, nothing in the queued lane that a LIFO remove
|
|
1618
|
+
* would take instead).
|
|
1619
|
+
*/
|
|
1620
|
+
function recallMethodFor(entry) {
|
|
1621
|
+
if (!entry || entry.role !== 'steer' || entry.consumed || entry.settled || !entry.runtimeId) return null;
|
|
1622
|
+
const itemId = queuedLane.get(entry.runtimeId);
|
|
1623
|
+
if (itemId) return { kind: 'queued', itemId };
|
|
1624
|
+
if (typeof session?.rpc?.queue?.removeMostRecent !== 'function') return null;
|
|
1625
|
+
if (steerSendsInFlight > 0 || queuedItemCount > 0 || waitingSteers < 1) return null;
|
|
1626
|
+
const pending = unconsumedSteers();
|
|
1627
|
+
return pending.length && pending[pending.length - 1] === entry ? { kind: 'newest' } : null;
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1630
|
+
/** Probe the experimental RPC surfaces once per session. */
|
|
1631
|
+
function probeSessionRpc(target) {
|
|
1632
|
+
sessionRpc = {
|
|
1633
|
+
queue: typeof target?.rpc?.queue?.pendingItems === 'function'
|
|
1634
|
+
&& typeof target?.rpc?.queue?.removeAt === 'function',
|
|
1635
|
+
interruptMainTurn: typeof target?.rpc?.interruptMainTurn === 'function',
|
|
1636
|
+
tasks: typeof target?.rpc?.tasks?.list === 'function'
|
|
1637
|
+
&& typeof target?.rpc?.tasks?.cancel === 'function',
|
|
1638
|
+
};
|
|
1639
|
+
dbg(`copilot session rpc surfaces: queue=${sessionRpc.queue} interruptMainTurn=${sessionRpc.interruptMainTurn} tasks=${sessionRpc.tasks}`);
|
|
789
1640
|
}
|
|
790
1641
|
|
|
791
1642
|
function observeBackgroundShells(event) {
|
|
@@ -809,7 +1660,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
809
1660
|
// closed to new lines, so the note belongs to the continuation this
|
|
810
1661
|
// notification is about to trigger.
|
|
811
1662
|
if (note && !activeTurn && pendingActivities.length < MAX_PENDING_ACTIVITIES) {
|
|
812
|
-
pendingActivities.push(note);
|
|
1663
|
+
pendingActivities.push({ text: note, reportsFailure: false });
|
|
813
1664
|
}
|
|
814
1665
|
}
|
|
815
1666
|
// Only the runtime's own `system.notification` heralds a continuation. A
|
|
@@ -835,9 +1686,13 @@ export function createCopilotSdkSessionRunner({
|
|
|
835
1686
|
// — but it is guarded anyway: re-routing INTO a settled turn would recurse
|
|
836
1687
|
// through this function forever, so it is treated as "no owner".
|
|
837
1688
|
if (!turn || turn.settled) {
|
|
838
|
-
// Same
|
|
839
|
-
|
|
840
|
-
turn =
|
|
1689
|
+
// Same rules as `routeEvent`: a pushed prompt's late run is its own
|
|
1690
|
+
// turn; otherwise only the start of work opens a row.
|
|
1691
|
+
turn = adoptOrphanedRun(event);
|
|
1692
|
+
if (!turn) {
|
|
1693
|
+
if (!isContinuationOpeningEvent(event)) return;
|
|
1694
|
+
turn = openContinuationTurn();
|
|
1695
|
+
}
|
|
841
1696
|
}
|
|
842
1697
|
await handleTurnEvent(turn, event);
|
|
843
1698
|
}
|
|
@@ -848,6 +1703,9 @@ export function createCopilotSdkSessionRunner({
|
|
|
848
1703
|
return;
|
|
849
1704
|
}
|
|
850
1705
|
turn.armStall?.();
|
|
1706
|
+
// Root work ends on the runtime's own terminator, so a question-only
|
|
1707
|
+
// continuation it joins no longer closes itself (whileAwaitingHuman).
|
|
1708
|
+
if (isContinuationOpeningEvent(event)) turn.rootWork = true;
|
|
851
1709
|
let actions = [];
|
|
852
1710
|
try {
|
|
853
1711
|
actions = turn.normalizer.normalize(event);
|
|
@@ -855,6 +1713,12 @@ export function createCopilotSdkSessionRunner({
|
|
|
855
1713
|
dbg('normalize failed', String(event?.type || ''), error?.message || String(error));
|
|
856
1714
|
return;
|
|
857
1715
|
}
|
|
1716
|
+
// The prompt boundary: the runtime has started work on a prompt. Decides
|
|
1717
|
+
// which row owns the segment the normalizer just opened, and settles the
|
|
1718
|
+
// previous run's row when this one is a new run (see noteUserMessage).
|
|
1719
|
+
if (String(event?.type || '') === 'user.message' && !event?.agentId) {
|
|
1720
|
+
await noteUserMessage(turn, event);
|
|
1721
|
+
}
|
|
858
1722
|
for (const action of actions) {
|
|
859
1723
|
if (action.channel === 'result') {
|
|
860
1724
|
turn.result = action.payload;
|
|
@@ -874,27 +1738,109 @@ export function createCopilotSdkSessionRunner({
|
|
|
874
1738
|
while (turn.bufferedActions.length) {
|
|
875
1739
|
const batch = turn.bufferedActions.splice(0);
|
|
876
1740
|
for (const action of batch) {
|
|
877
|
-
await
|
|
1741
|
+
await dispatchOwned(turn, action).catch(() => {});
|
|
878
1742
|
}
|
|
879
1743
|
}
|
|
880
1744
|
}
|
|
881
1745
|
|
|
882
|
-
|
|
1746
|
+
/**
|
|
1747
|
+
* The relay's `ask_user` tool: same name and fields as the runtime's
|
|
1748
|
+
* built-in, plus `multi_select`. The handler runs under exactly the same
|
|
1749
|
+
* human-gating as the built-in path (`onUserInputRequest`): the stall
|
|
1750
|
+
* watchdog is held, steering is held (a question card is open), and a card
|
|
1751
|
+
* raised the moment a continuation opens waits for its row. It never
|
|
1752
|
+
* throws: a failed card answers in-band so the model can carry on.
|
|
1753
|
+
*/
|
|
1754
|
+
function buildRelayAskUserTool() {
|
|
1755
|
+
return {
|
|
1756
|
+
name: RELAY_ASK_USER_TOOL_NAME,
|
|
1757
|
+
description: RELAY_ASK_USER_TOOL_DESCRIPTION,
|
|
1758
|
+
parameters: RELAY_ASK_USER_TOOL_PARAMETERS,
|
|
1759
|
+
overridesBuiltInTool: true,
|
|
1760
|
+
// The built-in asks no permission either: asking a question is not an action.
|
|
1761
|
+
skipPermission: true,
|
|
1762
|
+
defer: 'never',
|
|
1763
|
+
handler: async (args, invocation) => {
|
|
1764
|
+
const { question, choices, multiSelect } = parseAskUserToolArguments(args);
|
|
1765
|
+
try {
|
|
1766
|
+
const result = await whileAwaitingHuman(async (turn) => {
|
|
1767
|
+
await awaitInteractiveRow(turn);
|
|
1768
|
+
// Cancelled by either side: the turn (Stop, teardown) or the
|
|
1769
|
+
// runtime itself (a subagent cancelled via tasks.cancel, a
|
|
1770
|
+
// disconnect) — a card nobody will read must not hold steering.
|
|
1771
|
+
const signal = combineSignals(turn?.abortController?.signal, invocation?.signal);
|
|
1772
|
+
return questionBridge.askUserInput(
|
|
1773
|
+
{ question, choices, allowFreeform: true, multiSelect, source: 'relay-ask-user-tool' },
|
|
1774
|
+
{ signal, message: turn?.message },
|
|
1775
|
+
);
|
|
1776
|
+
});
|
|
1777
|
+
return formatAskUserToolResult({
|
|
1778
|
+
answer: result?.answer,
|
|
1779
|
+
wasFreeform: result?.wasFreeform !== false,
|
|
1780
|
+
timedOut: result?.timedOut === true,
|
|
1781
|
+
});
|
|
1782
|
+
} catch (error) {
|
|
1783
|
+
dbg('relay ask_user failed', error?.message || String(error));
|
|
1784
|
+
return USER_INPUT_UNSUPPORTED_ANSWER;
|
|
1785
|
+
}
|
|
1786
|
+
},
|
|
1787
|
+
};
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1790
|
+
/** One signal that aborts when any of the given ones does (nulls ignored). */
|
|
1791
|
+
function combineSignals(...signals) {
|
|
1792
|
+
const live = signals.filter((signal) => signal && typeof signal.aborted === 'boolean');
|
|
1793
|
+
if (live.length <= 1) return live[0] || null;
|
|
1794
|
+
if (typeof AbortSignal.any === 'function') return AbortSignal.any(live);
|
|
1795
|
+
const controller = new AbortController();
|
|
1796
|
+
for (const signal of live) {
|
|
1797
|
+
if (signal.aborted) { controller.abort(); break; }
|
|
1798
|
+
signal.addEventListener('abort', () => controller.abort(), { once: true });
|
|
1799
|
+
}
|
|
1800
|
+
return controller.signal;
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
/**
|
|
1804
|
+
* Open a session, retrying once without the relay's `ask_user` when the
|
|
1805
|
+
* runtime refuses the override (an older runtime). The refusal is
|
|
1806
|
+
* remembered until the runtime is stopped — the next runtime (possibly
|
|
1807
|
+
* auto-updated under the relay) gets another try.
|
|
1808
|
+
*/
|
|
1809
|
+
async function openSession(open, config) {
|
|
1810
|
+
try {
|
|
1811
|
+
return await open(config);
|
|
1812
|
+
} catch (error) {
|
|
1813
|
+
if (!config?.tools?.length || !isToolOverrideRejection(error) || isSessionNotFoundError(error)) throw error;
|
|
1814
|
+
dbg('the runtime refused the relay ask_user override; using its built-in ask_user', error?.message || String(error));
|
|
1815
|
+
askUserOverrideRejected = true;
|
|
1816
|
+
const { tools: _tools, ...withoutTools } = config;
|
|
1817
|
+
return open(withoutTools);
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
function buildSessionConfig(model, reasoningEffort = null) {
|
|
883
1822
|
// Built once per session, not per request. Reads the mode off the LIVE turn
|
|
884
1823
|
// rather than closing over the one the session was built with, because one
|
|
885
1824
|
// session serves every turn and the user can switch modes between them.
|
|
1825
|
+
// With no turn live (a background agent's request) it is the mode of the
|
|
1826
|
+
// last delivered turn, as for a continuation.
|
|
886
1827
|
const decidePermission = createCopilotPermissionHandler({
|
|
887
1828
|
// Only the ask-mode branch actually blocks on a human; wrapping it keeps
|
|
888
|
-
// the stall watchdog off a turn where someone is deciding
|
|
1829
|
+
// the stall watchdog off a turn where someone is deciding, and only
|
|
1830
|
+
// there can an approval open a row of its own.
|
|
889
1831
|
bridge: {
|
|
890
|
-
askToolApproval: (permissionRequest, options) => whileAwaitingHuman(async () => {
|
|
1832
|
+
askToolApproval: (permissionRequest, options) => whileAwaitingHuman(async (turn) => {
|
|
891
1833
|
// An ask-mode approval raised the moment a continuation opens must
|
|
892
1834
|
// wait for the row its card will hang off — see `awaitInteractiveRow`.
|
|
893
|
-
await awaitInteractiveRow();
|
|
894
|
-
return questionBridge.askToolApproval(permissionRequest,
|
|
1835
|
+
await awaitInteractiveRow(turn);
|
|
1836
|
+
return questionBridge.askToolApproval(permissionRequest, {
|
|
1837
|
+
...options,
|
|
1838
|
+
signal: turn?.abortController?.signal || options?.signal || null,
|
|
1839
|
+
message: turn?.message,
|
|
1840
|
+
});
|
|
895
1841
|
}),
|
|
896
1842
|
},
|
|
897
|
-
getRelayMode: () => activeTurn?.message?.relayMode ||
|
|
1843
|
+
getRelayMode: () => activeTurn?.message?.relayMode || lastRelayMode,
|
|
898
1844
|
// An aborted turn must not leave the human staring at a card whose answer
|
|
899
1845
|
// nothing will read; the bridge times the card out instead.
|
|
900
1846
|
getSignal: () => activeTurn?.abortController?.signal || null,
|
|
@@ -933,6 +1879,10 @@ export function createCopilotSdkSessionRunner({
|
|
|
933
1879
|
// The runtime's own defaults, pinned. See DEFAULT_INFINITE_SESSION_CONFIG.
|
|
934
1880
|
...(infiniteSessionConfig ? { infiniteSessions: { ...infiniteSessionConfig } } : {}),
|
|
935
1881
|
onEvent: routeEvent,
|
|
1882
|
+
// The relay's `ask_user` (see copilot-ask-user-tool.mjs): the built-in's
|
|
1883
|
+
// fields plus `multi_select`, so a model can declare a multi-select
|
|
1884
|
+
// question instead of the relay guessing from its wording.
|
|
1885
|
+
...(relayAskUserTool && !askUserOverrideRejected ? { tools: [buildRelayAskUserTool()] } : {}),
|
|
936
1886
|
// Permission policy follows the conversation's relay mode: agent and
|
|
937
1887
|
// autopilot auto-approve, plan denies non-read tools with feedback, and
|
|
938
1888
|
// ask asks the human through a relay question card. The handler reads the
|
|
@@ -956,12 +1906,13 @@ export function createCopilotSdkSessionRunner({
|
|
|
956
1906
|
// the deserializer is strict, so it is always a real boolean.
|
|
957
1907
|
onUserInputRequest: async (request) => {
|
|
958
1908
|
try {
|
|
959
|
-
const { answer, wasFreeform } = await whileAwaitingHuman(async () => {
|
|
1909
|
+
const { answer, wasFreeform } = await whileAwaitingHuman(async (turn) => {
|
|
960
1910
|
// Same gate as the approval path: an `ask_user` fired straight
|
|
961
1911
|
// after a continuation opened must get the row, not a null id.
|
|
962
|
-
await awaitInteractiveRow();
|
|
1912
|
+
await awaitInteractiveRow(turn);
|
|
963
1913
|
return questionBridge.askUserInput(request, {
|
|
964
|
-
signal:
|
|
1914
|
+
signal: turn?.abortController?.signal || null,
|
|
1915
|
+
message: turn?.message,
|
|
965
1916
|
});
|
|
966
1917
|
});
|
|
967
1918
|
return { answer, wasFreeform };
|
|
@@ -1015,12 +1966,12 @@ export function createCopilotSdkSessionRunner({
|
|
|
1015
1966
|
const source = String(request?.elicitationSource || '').trim();
|
|
1016
1967
|
const message = String(request?.message || '').trim()
|
|
1017
1968
|
|| 'Copilot needs structured input to continue this turn.';
|
|
1018
|
-
const result = await whileAwaitingHuman(async () => {
|
|
1019
|
-
await awaitInteractiveRow();
|
|
1969
|
+
const result = await whileAwaitingHuman(async (turn) => {
|
|
1970
|
+
await awaitInteractiveRow(turn);
|
|
1020
1971
|
return questionBridge.askStructured({
|
|
1021
1972
|
prompt: source ? `${message}\n\n(Requested by ${source}.)` : message,
|
|
1022
1973
|
requestedSchema: schema,
|
|
1023
|
-
}, { signal:
|
|
1974
|
+
}, { signal: turn?.abortController?.signal || null, message: turn?.message });
|
|
1024
1975
|
});
|
|
1025
1976
|
const content = result?.structuredAnswer;
|
|
1026
1977
|
if (result?.timedOut || !content || typeof content !== 'object' || Array.isArray(content)) {
|
|
@@ -1070,8 +2021,8 @@ export function createCopilotSdkSessionRunner({
|
|
|
1070
2021
|
dbg('copilot runtime exited', detail);
|
|
1071
2022
|
const turn = activeTurn;
|
|
1072
2023
|
if (turn && !turn.settled) {
|
|
1073
|
-
//
|
|
1074
|
-
//
|
|
2024
|
+
// driveTurn tears the runtime down and publishes the failure record
|
|
2025
|
+
// onto every row the turn owns.
|
|
1075
2026
|
turn.fail(new Error(`the Copilot runtime exited before the turn completed (${detail})`));
|
|
1076
2027
|
return;
|
|
1077
2028
|
}
|
|
@@ -1126,7 +2077,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
1126
2077
|
* rebuilt config. Nothing is lost either way: the relay session id IS the
|
|
1127
2078
|
* SDK session id, so the rebuild takes the ordinary resume path.
|
|
1128
2079
|
*/
|
|
1129
|
-
async function applySelection(model, effort
|
|
2080
|
+
async function applySelection(model, effort) {
|
|
1130
2081
|
if (!byokProvider) {
|
|
1131
2082
|
await modelSwitch.apply(session, { model, effort, byok: false });
|
|
1132
2083
|
return session;
|
|
@@ -1140,7 +2091,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
1140
2091
|
} else {
|
|
1141
2092
|
dbg('BYOK ceilings differ for the target model; rebuilding the session', model);
|
|
1142
2093
|
}
|
|
1143
|
-
return rebuildByokSession(model, effort
|
|
2094
|
+
return rebuildByokSession(model, effort);
|
|
1144
2095
|
}
|
|
1145
2096
|
|
|
1146
2097
|
/**
|
|
@@ -1152,7 +2103,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
1152
2103
|
* `SessionConfig.model`), so this path never re-enters the RPC attempt —
|
|
1153
2104
|
* which is also what makes it terminate: RPC refusal → rebuild → done.
|
|
1154
2105
|
*/
|
|
1155
|
-
async function rebuildByokSession(model, effort
|
|
2106
|
+
async function rebuildByokSession(model, effort) {
|
|
1156
2107
|
dbg('rebuilding the copilot session for a BYOK model/effort switch', model);
|
|
1157
2108
|
const closing = session;
|
|
1158
2109
|
session = null;
|
|
@@ -1162,13 +2113,13 @@ export function createCopilotSdkSessionRunner({
|
|
|
1162
2113
|
try { await closing?.disconnect?.(); } catch (error) {
|
|
1163
2114
|
dbg('session disconnect before model switch failed', error?.message || String(error));
|
|
1164
2115
|
}
|
|
1165
|
-
return ensureSession(model, effort
|
|
2116
|
+
return ensureSession(model, effort);
|
|
1166
2117
|
}
|
|
1167
2118
|
|
|
1168
|
-
async function ensureSession(model, effort
|
|
2119
|
+
async function ensureSession(model, effort) {
|
|
1169
2120
|
await ensureClient();
|
|
1170
2121
|
if (!session) {
|
|
1171
|
-
const config = buildSessionConfig(model,
|
|
2122
|
+
const config = buildSessionConfig(model, effort);
|
|
1172
2123
|
// Resume first, always. On a brand-new conversation this costs one
|
|
1173
2124
|
// failed RPC; on every other path (worker restart, idle shutdown,
|
|
1174
2125
|
// relay restart) it is the difference between continuing the
|
|
@@ -1177,7 +2128,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
1177
2128
|
// exists and the NEXT reconnect must resume it.
|
|
1178
2129
|
let resumed = false;
|
|
1179
2130
|
try {
|
|
1180
|
-
session = await client.resumeSession(sdkSessionId, config);
|
|
2131
|
+
session = await openSession((cfg) => client.resumeSession(sdkSessionId, cfg), config);
|
|
1181
2132
|
resumed = true;
|
|
1182
2133
|
dbg(`resumed copilot session ${sdkSessionId.slice(0, 8)}`);
|
|
1183
2134
|
} catch (error) {
|
|
@@ -1191,9 +2142,13 @@ export function createCopilotSdkSessionRunner({
|
|
|
1191
2142
|
throw error;
|
|
1192
2143
|
}
|
|
1193
2144
|
dbg('copilot session state not found, creating', error?.message || String(error));
|
|
1194
|
-
session = await client.createSession(config);
|
|
2145
|
+
session = await openSession((cfg) => client.createSession(cfg), config);
|
|
1195
2146
|
dbg(`created copilot session ${sdkSessionId.slice(0, 8)}`);
|
|
1196
2147
|
}
|
|
2148
|
+
// Which experimental surfaces THIS runtime has (it auto-updates under
|
|
2149
|
+
// the relay): decides un-steer and the targeted Stop for the session.
|
|
2150
|
+
probeSessionRpc(session);
|
|
2151
|
+
scheduleQueueLaneRefresh();
|
|
1197
2152
|
if (resumed && !byokProvider) {
|
|
1198
2153
|
// A resumed HOSTED session keeps whatever model it was created with —
|
|
1199
2154
|
// `config.model` is not guaranteed to be honoured on resume — so the
|
|
@@ -1225,7 +2180,7 @@ export function createCopilotSdkSessionRunner({
|
|
|
1225
2180
|
publishModelSnapshot(resumed ? 'session-resume' : 'session-start');
|
|
1226
2181
|
return session;
|
|
1227
2182
|
}
|
|
1228
|
-
return applySelection(model, effort
|
|
2183
|
+
return applySelection(model, effort);
|
|
1229
2184
|
}
|
|
1230
2185
|
|
|
1231
2186
|
async function stopRuntime(reason) {
|
|
@@ -1241,14 +2196,33 @@ export function createCopilotSdkSessionRunner({
|
|
|
1241
2196
|
// them: the tracked set is state about a process that no longer exists and
|
|
1242
2197
|
// must not pin the next one. The replay gate is reset for the same reason —
|
|
1243
2198
|
// the next connection resumes and arms its own window.
|
|
1244
|
-
if (backgroundShells.size()) {
|
|
2199
|
+
if (backgroundShells.size() || backgroundTasks.size) {
|
|
1245
2200
|
backgroundShells.reset();
|
|
1246
|
-
//
|
|
1247
|
-
|
|
2201
|
+
// Background agents and shells die with the runtime (live-probed: a
|
|
2202
|
+
// disconnect cancels the agents and drops the shells from the registry);
|
|
2203
|
+
// clear their panel cards too.
|
|
2204
|
+
backgroundTasks = new Map();
|
|
2205
|
+
agentDetails.clear();
|
|
2206
|
+
publishBackgroundTasks({ immediate: true });
|
|
2207
|
+
}
|
|
2208
|
+
if (taskRefreshTimer) {
|
|
2209
|
+
clearTimeout(taskRefreshTimer);
|
|
2210
|
+
taskRefreshTimer = null;
|
|
1248
2211
|
}
|
|
1249
2212
|
replayGate.reset();
|
|
2213
|
+
// A settled turn waiting on a card (driveTurn) would wait for an answer
|
|
2214
|
+
// nothing can take any more: the agent that asked died with the runtime.
|
|
2215
|
+
for (const turn of publishingTurns) turn.abortController.abort();
|
|
1250
2216
|
continuationDueSince = 0;
|
|
1251
2217
|
pendingActivities = [];
|
|
2218
|
+
compactingSince = 0;
|
|
2219
|
+
queuedLane = new Map();
|
|
2220
|
+
waitingSteers = 0;
|
|
2221
|
+
queuedItemCount = 0;
|
|
2222
|
+
steerSendsInFlight = 0;
|
|
2223
|
+
sessionRpc = { queue: false, interruptMainTurn: false, tasks: false };
|
|
2224
|
+
askUserOverrideRejected = false;
|
|
2225
|
+
syncDeliveryReadiness();
|
|
1252
2226
|
if (!closingSession && !closingClient) return;
|
|
1253
2227
|
dbg(`stopping the copilot runtime (${reason})`);
|
|
1254
2228
|
try { await closingSession?.disconnect?.(); } catch (error) {
|
|
@@ -1291,8 +2265,44 @@ export function createCopilotSdkSessionRunner({
|
|
|
1291
2265
|
dbg('continuation grace expired with no continuation turn');
|
|
1292
2266
|
continuationDueSince = 0;
|
|
1293
2267
|
}
|
|
2268
|
+
if (sessionRpc.tasks) {
|
|
2269
|
+
// The registry is the truth: live agents and shells hold the runtime up
|
|
2270
|
+
// to the cap (the relay's background-task slider, 0 = unlimited), and on
|
|
2271
|
+
// expiry they are CANCELLED — the runtime can stop them now — rather
|
|
2272
|
+
// than forgotten. Bounded: a task whose cancel the runtime refuses a
|
|
2273
|
+
// few times is abandoned instead of pinning the runtime forever. A
|
|
2274
|
+
// re-read once per lifecycle poll catches state the events miss (a
|
|
2275
|
+
// shell that exited after a cancel emits no notification).
|
|
2276
|
+
const capMs = Number(getBackgroundTaskTimeoutMs()) || 0;
|
|
2277
|
+
const live = runningBackgroundTasks();
|
|
2278
|
+
if (!live.length) return false;
|
|
2279
|
+
if (capMs > 0) {
|
|
2280
|
+
for (const task of live) {
|
|
2281
|
+
const startedAt = Date.parse(String(task.startedAt || '')) || agentDetails.get(task.id)?.startedAt || 0;
|
|
2282
|
+
if (!startedAt || now - startedAt < capMs) continue;
|
|
2283
|
+
const refusals = taskCancelRefusals.get(task.id) || 0;
|
|
2284
|
+
if (refusals >= MAX_TASK_CANCEL_REFUSALS) {
|
|
2285
|
+
dbg('background task cap reached and the runtime would not cancel it; abandoning it', task.id);
|
|
2286
|
+
abandonedTasks.add(task.id);
|
|
2287
|
+
backgroundTasks.delete(task.id);
|
|
2288
|
+
publishBackgroundTasks({ immediate: true });
|
|
2289
|
+
continue;
|
|
2290
|
+
}
|
|
2291
|
+
dbg('background task cap reached; cancelling it', task.id, task.description || task.command || '');
|
|
2292
|
+
void stopBackgroundTask(task.id).then((cancelled) => {
|
|
2293
|
+
if (!cancelled) taskCancelRefusals.set(task.id, (taskCancelRefusals.get(task.id) || 0) + 1);
|
|
2294
|
+
});
|
|
2295
|
+
}
|
|
2296
|
+
if (!runningBackgroundTasks().length) return false;
|
|
2297
|
+
}
|
|
2298
|
+
if (now - lastTaskRefreshAt >= lifecyclePollMs) scheduleTaskRefresh();
|
|
2299
|
+
return true;
|
|
2300
|
+
}
|
|
1294
2301
|
if (!backgroundShells.size()) return false;
|
|
1295
|
-
|
|
2302
|
+
// Without the registry there is no Stop and no card worth the name, so an
|
|
2303
|
+
// unlimited slider (0) must not mean "a forgotten `sleep 99999` pins the
|
|
2304
|
+
// runtime forever": the old 30-minute default applies on this path.
|
|
2305
|
+
const capMs = Number(getBackgroundTaskTimeoutMs()) || DEFAULT_BACKGROUND_TASK_TIMEOUT_MS;
|
|
1296
2306
|
let expiredAny = false;
|
|
1297
2307
|
for (const shell of backgroundShells.expireOlderThan(capMs)) {
|
|
1298
2308
|
expiredAny = true;
|
|
@@ -1339,18 +2349,81 @@ export function createCopilotSdkSessionRunner({
|
|
|
1339
2349
|
return String(message?.providerModel || '').trim() || defaultModel;
|
|
1340
2350
|
}
|
|
1341
2351
|
|
|
1342
|
-
|
|
2352
|
+
/**
|
|
2353
|
+
* A queue row this turn owes an answer.
|
|
2354
|
+
*
|
|
2355
|
+
* `role`: 'primary' (the delivered row that opened the turn), 'steer' (a row
|
|
2356
|
+
* delivered mid-turn and pushed into the same runtime interaction) or
|
|
2357
|
+
* 'self' (the synthetic row of a self-initiated turn). An entry is owned —
|
|
2358
|
+
* heartbeat-claimed, crash-guard-requeued — from creation until `released`.
|
|
2359
|
+
*/
|
|
2360
|
+
function createEntry(turn, message, role) {
|
|
2361
|
+
const entry = {
|
|
2362
|
+
turn,
|
|
2363
|
+
message,
|
|
2364
|
+
role,
|
|
2365
|
+
// The id `send()` resolved; the key `user.message.messageId` matches.
|
|
2366
|
+
runtimeId: null,
|
|
2367
|
+
// A root `user.message` named this entry: the runtime has taken the
|
|
2368
|
+
// prompt. `delivery` is the runtime's own word for how.
|
|
2369
|
+
consumed: false,
|
|
2370
|
+
delivery: null,
|
|
2371
|
+
interactionId: null,
|
|
2372
|
+
// The entry whose run absorbed this one (`delivery:"steering"`).
|
|
2373
|
+
foldedInto: null,
|
|
2374
|
+
consumedMarked: false,
|
|
2375
|
+
// Its outcome was decided and its publishes started / it landed.
|
|
2376
|
+
settled: false,
|
|
2377
|
+
released: false,
|
|
2378
|
+
cancelled: false,
|
|
2379
|
+
// Something worth keeping was published on this row: the run's own
|
|
2380
|
+
// work (a tool line, a thought, a lane), or a carried-in line that
|
|
2381
|
+
// reports a failure — not bookkeeping, and not the other lines carried
|
|
2382
|
+
// in from between turns (`showsWork`). Decides whether an empty
|
|
2383
|
+
// continuation may vanish.
|
|
2384
|
+
showedWork: false,
|
|
2385
|
+
// `lastStreamedText` is what this row last received on /api/stream —
|
|
2386
|
+
// the stream gate and the abort/error fallback text both read it.
|
|
2387
|
+
state: { lastStreamedText: '' },
|
|
2388
|
+
};
|
|
2389
|
+
entry.done = new Promise((resolve) => { entry.resolveDone = resolve; });
|
|
2390
|
+
turn.entries.push(entry);
|
|
2391
|
+
return entry;
|
|
2392
|
+
}
|
|
2393
|
+
|
|
2394
|
+
function assignRuntimeId(turn, entry, runtimeId) {
|
|
2395
|
+
const id = String(runtimeId || '').trim();
|
|
2396
|
+
entry.runtimeId = id || null;
|
|
2397
|
+
if (!id) return;
|
|
2398
|
+
turn.pushed.set(id, entry);
|
|
2399
|
+
// The runtime can start work on a prompt before the send() that pushed it
|
|
2400
|
+
// has resolved with its id. The boundary was remembered; reconcile it now,
|
|
2401
|
+
// on the dispatch chain so it orders after the events already captured.
|
|
2402
|
+
const unclaimed = turn.unclaimedUserMessages.get(id);
|
|
2403
|
+
if (!unclaimed) return;
|
|
2404
|
+
turn.unclaimedUserMessages.delete(id);
|
|
2405
|
+
dispatchChain = dispatchChain
|
|
2406
|
+
.then(() => adoptSegment(turn, entry, unclaimed))
|
|
2407
|
+
.catch((error) => { dbg('late prompt-boundary reconcile failed', error?.message || String(error)); });
|
|
2408
|
+
}
|
|
2409
|
+
|
|
2410
|
+
function createTurn(message, { kind = 'delivered', adoptEntry = null } = {}) {
|
|
1343
2411
|
const continuation = kind === 'continuation';
|
|
1344
2412
|
const turn = {
|
|
1345
2413
|
// 'delivered' (a queue row arrived) | 'continuation' (the runtime started
|
|
1346
2414
|
// work by itself and this worker minted a synthetic row for it).
|
|
1347
2415
|
kind,
|
|
1348
2416
|
message,
|
|
1349
|
-
|
|
1350
|
-
|
|
2417
|
+
model: '',
|
|
2418
|
+
// The runner's cross-turn knowledge of background agents rides in, so a
|
|
2419
|
+
// later turn's normalizer opens no lane for an agent spawned earlier.
|
|
2420
|
+
normalizer: createNormalizerImpl({ isBackgroundAgent: (agentId) => agentDetails.has(agentId) }),
|
|
1351
2421
|
result: null,
|
|
2422
|
+
failure: null,
|
|
1352
2423
|
settled: false,
|
|
1353
2424
|
aborted: false,
|
|
2425
|
+
// The row the user pressed Stop on (from the abort control), when known.
|
|
2426
|
+
abortedRowId: null,
|
|
1354
2427
|
stallTimer: null,
|
|
1355
2428
|
planBoardPosted: false,
|
|
1356
2429
|
// Set when a mutating tool was actually approved and run this turn.
|
|
@@ -1365,45 +2438,57 @@ export function createCopilotSdkSessionRunner({
|
|
|
1365
2438
|
// rather than failing the worker.
|
|
1366
2439
|
discarded: false,
|
|
1367
2440
|
controlState: null,
|
|
1368
|
-
// How many prompts have been SENT into this interaction. The runtime
|
|
1369
|
-
// consumes queued prompts in order, and opens one `user.message` segment
|
|
1370
|
-
// per prompt as it picks each up, so send order IS segment order.
|
|
1371
|
-
//
|
|
1372
|
-
// Deliberately not derived from the normalizer's live segment count: at
|
|
1373
|
-
// the moment a steering send happens the runtime may not have opened the
|
|
1374
|
-
// previous prompt's segment yet, and the steered row would then be
|
|
1375
|
-
// attributed the PREVIOUS prompt's answer.
|
|
1376
|
-
//
|
|
1377
|
-
// Starts at 1, not 0: this turn's own prompt is always its first, and
|
|
1378
|
-
// counting it here rather than after `send()` resolves closes the race
|
|
1379
|
-
// where a steering delivery lands between the two. A continuation sent no
|
|
1380
|
-
// prompt at all, so it starts at 0.
|
|
1381
|
-
promptsSent: continuation ? 0 : 1,
|
|
1382
|
-
// The segment this turn's OWN reply lands in. For a delivered turn that
|
|
1383
|
-
// is where its prompt is picked up — non-zero when a previous interaction
|
|
1384
|
-
// left prompts queued inside the runtime, which are consumed first. A
|
|
1385
|
-
// continuation has no `user.message` of its own, so the normalizer's
|
|
1386
|
-
// implicit first segment is its own.
|
|
1387
|
-
baseSegment: continuation ? 0 : carriedPrompts,
|
|
1388
|
-
// The segment the NEXT prompt steered into this interaction will be
|
|
1389
|
-
// answered in. Tracked explicitly rather than derived, because the two
|
|
1390
|
-
// kinds differ: a delivered turn's own prompt occupies `baseSegment`, so
|
|
1391
|
-
// the next is `baseSegment + 1`; a continuation occupies segment 0
|
|
1392
|
-
// without having sent anything, so the first steered prompt opens
|
|
1393
|
-
// segment 1. Getting this wrong cross-publishes the continuation's reply
|
|
1394
|
-
// into the user's row.
|
|
1395
|
-
nextSegmentIndex: continuation ? 1 : carriedPrompts + 1,
|
|
1396
|
-
// The lowest segment index any prompt SENT by this turn can occupy — the
|
|
1397
|
-
// floor for the carried-prompt arithmetic when the interaction ends.
|
|
1398
|
-
firstSentSegment: continuation ? 1 : carriedPrompts,
|
|
1399
2441
|
// Cancels any relay question card this turn is blocked on, so an aborted
|
|
1400
2442
|
// turn does not leave a human answering into the void.
|
|
1401
2443
|
abortController: new AbortController(),
|
|
1402
|
-
//
|
|
1403
|
-
//
|
|
1404
|
-
//
|
|
1405
|
-
|
|
2444
|
+
// Cards (questions, ask-mode approvals, elicitations) open on this
|
|
2445
|
+
// turn's row; `humansIdle` resolves when the last one is done. A turn
|
|
2446
|
+
// that settles with one still open publishes only after it (driveTurn).
|
|
2447
|
+
humanRequests: 0,
|
|
2448
|
+
humansIdle: null,
|
|
2449
|
+
// Whether any card was ever raised on this turn's row.
|
|
2450
|
+
askedHuman: false,
|
|
2451
|
+
// Opened by `ensureInteractiveTurn` for a question rather than by the
|
|
2452
|
+
// runtime, and whether root work has joined it since.
|
|
2453
|
+
humanOrigin: false,
|
|
2454
|
+
rootWork: false,
|
|
2455
|
+
// Every row this turn owes an answer, in push order.
|
|
2456
|
+
entries: [],
|
|
2457
|
+
// runtime message id → entry, for the `user.message` acknowledgement.
|
|
2458
|
+
pushed: new Map(),
|
|
2459
|
+
// Normalizer segment index → the entry whose row the segment's text
|
|
2460
|
+
// belongs to. A `delivery:"steering"` prompt's segment belongs to the
|
|
2461
|
+
// run it was folded into; a `queued`/`idle` one starts its own run.
|
|
2462
|
+
segmentOwners: [],
|
|
2463
|
+
// The entry whose run the runtime is working on right now.
|
|
2464
|
+
currentOwner: null,
|
|
2465
|
+
primaryEntry: null,
|
|
2466
|
+
// Early settles (a previous run's row, published when the next run
|
|
2467
|
+
// opened) run off the event dispatch chain; driveTurn awaits them
|
|
2468
|
+
// before releasing ownership.
|
|
2469
|
+
settlePromises: [],
|
|
2470
|
+
// False until the turn-opening send() has resolved: the delivery hold.
|
|
2471
|
+
primarySent: continuation,
|
|
2472
|
+
// Root `user.message`s whose id no pushed entry knew yet (the event beat
|
|
2473
|
+
// the send() resolution): runtime id → boundary, reconciled by
|
|
2474
|
+
// assignRuntimeId.
|
|
2475
|
+
unclaimedUserMessages: new Map(),
|
|
1406
2476
|
};
|
|
2477
|
+
if (adoptEntry) {
|
|
2478
|
+
// A pushed row whose run the runtime opened after the turn it was pushed
|
|
2479
|
+
// into had already settled: it becomes the primary of a turn of its own.
|
|
2480
|
+
adoptEntry.turn = turn;
|
|
2481
|
+
turn.entries.push(adoptEntry);
|
|
2482
|
+
turn.primaryEntry = adoptEntry;
|
|
2483
|
+
if (adoptEntry.runtimeId) turn.pushed.set(adoptEntry.runtimeId, adoptEntry);
|
|
2484
|
+
turn.primarySent = true;
|
|
2485
|
+
} else {
|
|
2486
|
+
turn.primaryEntry = createEntry(turn, message, continuation ? 'self' : 'primary');
|
|
2487
|
+
}
|
|
2488
|
+
turn.currentOwner = turn.primaryEntry;
|
|
2489
|
+
if (continuation) turn.segmentOwners[0] = turn.primaryEntry;
|
|
2490
|
+
turn.primaryReady = new Promise((resolve) => { turn.resolvePrimaryReady = resolve; });
|
|
2491
|
+
if (turn.primarySent) turn.resolvePrimaryReady();
|
|
1407
2492
|
if (continuation) {
|
|
1408
2493
|
// The single in-flight registration for this turn's synthetic row, and
|
|
1409
2494
|
// the signal that abandons it. One promise, stored ON the turn, so the
|
|
@@ -1425,18 +2510,19 @@ export function createCopilotSdkSessionRunner({
|
|
|
1425
2510
|
turn.rejectDone = reject;
|
|
1426
2511
|
});
|
|
1427
2512
|
// The stall watchdog and the runtime-exit observer can reject `turn.done`
|
|
1428
|
-
// before
|
|
1429
|
-
//
|
|
1430
|
-
//
|
|
1431
|
-
// still sees the rejection.
|
|
2513
|
+
// before anything awaits it — an unhandled rejection that the worker crash
|
|
2514
|
+
// guard would escalate into a whole-worker failure. This handler exists
|
|
2515
|
+
// only to mark the promise as handled; driveTurn still sees the rejection.
|
|
1432
2516
|
turn.done.catch(() => {});
|
|
1433
2517
|
turn.settle = () => {
|
|
1434
2518
|
if (turn.settled) return;
|
|
1435
2519
|
turn.settled = true;
|
|
1436
2520
|
turn.disarmStall();
|
|
1437
2521
|
// The terminal event releases the event stream at once — see
|
|
1438
|
-
// `beginPublishing`. The queue
|
|
2522
|
+
// `beginPublishing`. The queue rows stay owned until the publishes land.
|
|
1439
2523
|
beginPublishing(turn);
|
|
2524
|
+
turn.resolvePrimaryReady();
|
|
2525
|
+
syncDeliveryReadiness();
|
|
1440
2526
|
turn.resolveDone();
|
|
1441
2527
|
};
|
|
1442
2528
|
turn.fail = (error) => {
|
|
@@ -1444,6 +2530,8 @@ export function createCopilotSdkSessionRunner({
|
|
|
1444
2530
|
turn.settled = true;
|
|
1445
2531
|
turn.disarmStall();
|
|
1446
2532
|
beginPublishing(turn);
|
|
2533
|
+
turn.resolvePrimaryReady();
|
|
2534
|
+
syncDeliveryReadiness();
|
|
1447
2535
|
turn.rejectDone(error);
|
|
1448
2536
|
};
|
|
1449
2537
|
turn.disarmStall = () => {
|
|
@@ -1472,44 +2560,513 @@ export function createCopilotSdkSessionRunner({
|
|
|
1472
2560
|
return turn;
|
|
1473
2561
|
}
|
|
1474
2562
|
|
|
1475
|
-
|
|
1476
|
-
|
|
2563
|
+
// ----------------------------------------------------- prompt boundaries --
|
|
2564
|
+
|
|
2565
|
+
function claimSegment(turn, index, owner) {
|
|
2566
|
+
// Leading unowned segments (text the runtime produced before any prompt
|
|
2567
|
+
// boundary, e.g. the answer to a prompt carried over from a previous
|
|
2568
|
+
// interaction) go to the first owner: that is where the merged note sent
|
|
2569
|
+
// the reader.
|
|
2570
|
+
for (let j = 0; j <= index; j += 1) {
|
|
2571
|
+
if (!turn.segmentOwners[j]) turn.segmentOwners[j] = owner;
|
|
2572
|
+
}
|
|
2573
|
+
}
|
|
2574
|
+
|
|
2575
|
+
/**
|
|
2576
|
+
* A root `user.message` arrived: the runtime started work on a prompt. Called
|
|
2577
|
+
* on the dispatch chain right after the normalizer opened the prompt's
|
|
2578
|
+
* segment, so `promptCount() - 1` is that segment.
|
|
2579
|
+
*/
|
|
2580
|
+
async function noteUserMessage(turn, event) {
|
|
2581
|
+
const data = event?.data && typeof event.data === 'object' ? event.data : {};
|
|
2582
|
+
const index = Math.max(0, turn.normalizer.promptCount() - 1);
|
|
2583
|
+
const runtimeId = String(data.messageId || '').trim();
|
|
2584
|
+
const boundary = {
|
|
2585
|
+
index,
|
|
2586
|
+
delivery: String(data.delivery || '').trim().toLowerCase(),
|
|
2587
|
+
interactionId: String(data.interactionId || '').trim(),
|
|
2588
|
+
};
|
|
2589
|
+
const entry = runtimeId ? turn.pushed.get(runtimeId) : null;
|
|
2590
|
+
if (entry) {
|
|
2591
|
+
await adoptSegment(turn, entry, boundary);
|
|
2592
|
+
return;
|
|
2593
|
+
}
|
|
2594
|
+
// Not one of ours (yet): a prompt the runtime injected itself (an autopilot
|
|
2595
|
+
// nudge, a carried-over prompt from a previous interaction), or a pushed
|
|
2596
|
+
// prompt whose send() has not resolved with its id. Its text belongs to
|
|
2597
|
+
// the run in progress until proven otherwise.
|
|
2598
|
+
if (runtimeId && !turn.unclaimedUserMessages.has(runtimeId)) turn.unclaimedUserMessages.set(runtimeId, boundary);
|
|
2599
|
+
claimSegment(turn, index, turn.currentOwner || turn.primaryEntry);
|
|
2600
|
+
}
|
|
2601
|
+
|
|
2602
|
+
/**
|
|
2603
|
+
* Bind a prompt boundary to the entry it names.
|
|
2604
|
+
*
|
|
2605
|
+
* `delivery:"steering"`: the runtime folded the prompt into the run already
|
|
2606
|
+
* in progress — the reply that follows belongs to that run's row, and this
|
|
2607
|
+
* row will settle with the fold marker (Claude's `folded`). The relay is
|
|
2608
|
+
* told the prompt is consumed at once, so a crash between now and the
|
|
2609
|
+
* marker fails the row instead of re-running it.
|
|
2610
|
+
*
|
|
2611
|
+
* Anything else (`queued`, `idle`, unknown): the prompt starts its own run
|
|
2612
|
+
* on the same interaction. The previous run is over — nothing more will be
|
|
2613
|
+
* produced for its row, so it is settled right away rather than at the
|
|
2614
|
+
* drain's end, exactly as a Claude context finalizes when the CLI opens the
|
|
2615
|
+
* next one.
|
|
2616
|
+
*/
|
|
2617
|
+
async function adoptSegment(turn, entry, { index, delivery, interactionId }) {
|
|
2618
|
+
entry.consumed = true;
|
|
2619
|
+
noteCancellableChange();
|
|
2620
|
+
entry.delivery = delivery || null;
|
|
2621
|
+
entry.interactionId = interactionId || null;
|
|
2622
|
+
const running = turn.currentOwner || turn.primaryEntry;
|
|
2623
|
+
if (delivery === 'steering' && running && running !== entry && !running.settled) {
|
|
2624
|
+
entry.foldedInto = running;
|
|
2625
|
+
turn.segmentOwners[index] = running;
|
|
2626
|
+
claimSegment(turn, index, running);
|
|
2627
|
+
if (entry.role === 'steer' && !entry.consumedMarked) {
|
|
2628
|
+
entry.consumedMarked = true;
|
|
2629
|
+
void markConsumed(entry.message.conversationId, [entry.message]);
|
|
2630
|
+
}
|
|
2631
|
+
dbg('steered prompt folded into the running turn', entry.message.id, `→ ${running.message.id || '(continuation)'}`);
|
|
2632
|
+
return;
|
|
2633
|
+
}
|
|
2634
|
+
turn.segmentOwners[index] = entry;
|
|
2635
|
+
claimSegment(turn, index, entry);
|
|
2636
|
+
if (running === entry) return;
|
|
2637
|
+
turn.currentOwner = entry;
|
|
2638
|
+
if (running && !running.settled) {
|
|
2639
|
+
dbg('pushed prompt runs as its own turn; settling the previous run', entry.message.id, `after ${running.message.id || '(continuation)'}`);
|
|
2640
|
+
// Off the dispatch chain: a slow relay must not stall the events of
|
|
2641
|
+
// the run that just opened (or its terminator) behind this publish.
|
|
2642
|
+
// A continuation whose row is not minted yet (a steer pushed past the
|
|
2643
|
+
// hold after its wait ran out) settles once the row exists — publishing
|
|
2644
|
+
// now would name no row — or is discarded with it.
|
|
2645
|
+
const settled = running.message?.id || !turn.rowReady
|
|
2646
|
+
? settleEntry(turn, running, { type: 'completed' })
|
|
2647
|
+
: turn.rowReady.then((ready) => settleEntry(turn, running, { type: ready ? 'completed' : 'discarded' }));
|
|
2648
|
+
turn.settlePromises.push(settled);
|
|
2649
|
+
}
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
// -------------------------------------------------------------- settling --
|
|
2653
|
+
|
|
2654
|
+
/**
|
|
2655
|
+
* Whether a completed row with no text has nothing to show at all: the
|
|
2656
|
+
* runtime opened it by itself (typically right after a background agent
|
|
2657
|
+
* finished), its run published no tool line, thought or lane, no card was
|
|
2658
|
+
* raised on it, no user message was folded into it, and nothing carried in
|
|
2659
|
+
* reported a failure. Such a row would only ever read "the turn completed
|
|
2660
|
+
* without a text reply", and the task card already showed the agent
|
|
2661
|
+
* finishing (live, gpt-5.6-luna, 2026-09-26). A failed agent's card is gone
|
|
2662
|
+
* by then, so the row carrying its failure line is kept (`showsWork`).
|
|
2663
|
+
* Every delivered row keeps the note — somebody asked something there.
|
|
2664
|
+
*/
|
|
2665
|
+
function settlesSilently(turn, entry, folded) {
|
|
2666
|
+
return turn.kind === 'continuation'
|
|
2667
|
+
&& entry.role === 'self'
|
|
2668
|
+
&& !turn.humanOrigin
|
|
2669
|
+
&& !turn.askedHuman
|
|
2670
|
+
&& !turn.planBoardPosted
|
|
2671
|
+
&& !entry.showedWork
|
|
2672
|
+
&& !folded.length
|
|
2673
|
+
&& Boolean(entry.message?.id);
|
|
2674
|
+
}
|
|
2675
|
+
|
|
2676
|
+
/**
|
|
2677
|
+
* Settle a silent continuation by tearing its row down: the requeue route's
|
|
2678
|
+
* continuation branch drops a processing continuation without an answer
|
|
2679
|
+
* (`dropped: 'continuation'`), the same teardown a dead worker's rows get.
|
|
2680
|
+
* A lost request leaves the row to that same stale-continuation recovery.
|
|
2681
|
+
*/
|
|
2682
|
+
async function dropSilentContinuation(entry) {
|
|
2683
|
+
dbg('continuation ended with nothing to show; dropping its row', entry.message.id);
|
|
2684
|
+
await api('POST', '/api/requeue', { messageId: entry.message.id, ...attemptFields(entry.message) })
|
|
2685
|
+
.catch((error) => { dbg('silent continuation drop failed', entry.message.id, error?.message || String(error)); });
|
|
2686
|
+
}
|
|
2687
|
+
|
|
2688
|
+
/**
|
|
2689
|
+
* Decide and publish one row's outcome. Idempotent per entry. A run owner's
|
|
2690
|
+
* settle also settles the steers folded into its run (they were answered by
|
|
2691
|
+
* its reply), in push order, after the reply itself.
|
|
2692
|
+
*/
|
|
2693
|
+
async function settleEntry(turn, entry, outcome) {
|
|
2694
|
+
if (entry.settled) return;
|
|
2695
|
+
entry.settled = true;
|
|
2696
|
+
const model = turn.result?.model || turn.normalizer.model || turn.model || null;
|
|
2697
|
+
const folded = turn.entries.filter((other) => other !== entry && !other.settled && other.foldedInto === entry);
|
|
2698
|
+
try {
|
|
2699
|
+
switch (outcome.type) {
|
|
2700
|
+
case 'completed': {
|
|
2701
|
+
const text = finalTextFor(turn, entry, turn.result);
|
|
2702
|
+
if (!text && settlesSilently(turn, entry, folded)) {
|
|
2703
|
+
await dropSilentContinuation(entry);
|
|
2704
|
+
break;
|
|
2705
|
+
}
|
|
2706
|
+
const published = text || EMPTY_TURN_COMPLETION_NOTE;
|
|
2707
|
+
if (entry.role !== 'steer' && shouldPostPlanBoard({
|
|
2708
|
+
relayMode: entry.message.relayMode,
|
|
2709
|
+
finalText: text,
|
|
2710
|
+
alreadyPosted: turn.planBoardPosted,
|
|
2711
|
+
acted: turn.acted === true,
|
|
2712
|
+
})) {
|
|
2713
|
+
await publishPlanBoard(turn, text, 'plan-mode-fallback');
|
|
2714
|
+
}
|
|
2715
|
+
await publishFinalStream(entry.message, published);
|
|
2716
|
+
await publishResponse(entry.message, {
|
|
2717
|
+
text: published,
|
|
2718
|
+
model,
|
|
2719
|
+
consumedSteerIds: folded.map((other) => other.message),
|
|
2720
|
+
});
|
|
2721
|
+
break;
|
|
2722
|
+
}
|
|
2723
|
+
case 'aborted-by-user': {
|
|
2724
|
+
// The queue row's fate belongs to the server-side abort control,
|
|
2725
|
+
// exactly as in the Claude and Cursor workers. Publishing a response
|
|
2726
|
+
// here would double-settle the row.
|
|
2727
|
+
await publishFinalStream(entry.message, finalTextFor(turn, entry, turn.result));
|
|
2728
|
+
break;
|
|
2729
|
+
}
|
|
2730
|
+
case 'user-stopped': {
|
|
2731
|
+
// The run a Stop cut short while naming another row. The abort ack
|
|
2732
|
+
// fails only the row it names, so this one is failed here — as
|
|
2733
|
+
// stopped, never requeued: a requeue would re-run a prompt the user
|
|
2734
|
+
// stopped (the Claude worker's lingering-stopped rule). Claimed
|
|
2735
|
+
// first, so a crash meanwhile fails it rather than re-running it.
|
|
2736
|
+
const id = String(entry.message.id || '');
|
|
2737
|
+
const terminalError = buildRelayStopFailure();
|
|
2738
|
+
claimSettling(entry.message, 'stopped');
|
|
2739
|
+
try {
|
|
2740
|
+
await publishFinalStream(entry.message, finalTextFor(turn, entry, turn.result));
|
|
2741
|
+
const outcome = await publishResponse(entry.message, {
|
|
2742
|
+
text: terminalError.message,
|
|
2743
|
+
model,
|
|
2744
|
+
terminalError,
|
|
2745
|
+
requeueOnFailure: false,
|
|
2746
|
+
consumedSteerIds: folded.map((other) => other.message),
|
|
2747
|
+
});
|
|
2748
|
+
// Not even the stop landed: the heartbeat hands it to the relay to fail.
|
|
2749
|
+
if (outcome === 'failed' && id) settleFailedTerminals.set(id, terminalError);
|
|
2750
|
+
} finally {
|
|
2751
|
+
releaseSettling(id);
|
|
2752
|
+
}
|
|
2753
|
+
break;
|
|
2754
|
+
}
|
|
2755
|
+
case 'runtime-interrupted': {
|
|
2756
|
+
const own = finalTextFor(turn, entry, turn.result);
|
|
2757
|
+
const text = own ? `${own}\n\n${RUNTIME_INTERRUPTED_NOTE}` : RUNTIME_INTERRUPTED_NOTE;
|
|
2758
|
+
await publishFinalStream(entry.message, text);
|
|
2759
|
+
await publishResponse(entry.message, { text, model, consumedSteerIds: folded.map((other) => other.message) });
|
|
2760
|
+
break;
|
|
2761
|
+
}
|
|
2762
|
+
case 'error': {
|
|
2763
|
+
const { classified } = outcome;
|
|
2764
|
+
await publishFinalStream(entry.message, entry.state.lastStreamedText);
|
|
2765
|
+
await publishResponse(entry.message, {
|
|
2766
|
+
text: classified.text,
|
|
2767
|
+
model,
|
|
2768
|
+
terminalError: terminalErrorRecord(entry.message, classified),
|
|
2769
|
+
consumedSteerIds: folded.map((other) => other.message),
|
|
2770
|
+
});
|
|
2771
|
+
break;
|
|
2772
|
+
}
|
|
2773
|
+
case 'folded':
|
|
2774
|
+
await publishSettleMarker(entry.message, { text: STEER_FOLDED_TEXT, model, kind: 'folded', variant: 'folded' });
|
|
2775
|
+
break;
|
|
2776
|
+
case 'stopped':
|
|
2777
|
+
await publishSettleMarker(entry.message, { text: STEER_STOPPED_TEXT, model, kind: 'stopped', variant: 'stopped' });
|
|
2778
|
+
break;
|
|
2779
|
+
case 'merged-note':
|
|
2780
|
+
// Accepted by the runtime, never started: still queued inside it, so
|
|
2781
|
+
// it will be answered at the start of the next interaction — a
|
|
2782
|
+
// requeue would run it twice.
|
|
2783
|
+
await publishFinalStream(entry.message, STEERED_ROW_MERGED_NOTE);
|
|
2784
|
+
await publishResponse(entry.message, { text: STEERED_ROW_MERGED_NOTE, model });
|
|
2785
|
+
break;
|
|
2786
|
+
case 'cancelled':
|
|
2787
|
+
case 'discarded':
|
|
2788
|
+
default:
|
|
2789
|
+
break;
|
|
2790
|
+
}
|
|
2791
|
+
} catch (error) {
|
|
2792
|
+
dbg('settle publish failed', entry.message?.id || '(no id)', outcome.type, error?.message || String(error));
|
|
2793
|
+
} finally {
|
|
2794
|
+
entry.released = true;
|
|
2795
|
+
entry.resolveDone(true);
|
|
2796
|
+
}
|
|
2797
|
+
for (const other of folded) {
|
|
2798
|
+
const foldOutcome = outcome.type === 'completed' || outcome.type === 'merged-note'
|
|
2799
|
+
? { type: 'folded' }
|
|
2800
|
+
: outcome.type === 'error' ? outcome : { type: 'stopped' };
|
|
2801
|
+
await settleEntry(turn, other, foldOutcome);
|
|
2802
|
+
}
|
|
2803
|
+
}
|
|
2804
|
+
|
|
2805
|
+
/** Remove our unconsumed prompts from the runtime's queued lane (best effort). */
|
|
2806
|
+
async function clearOwnedQueuedItems(entries) {
|
|
2807
|
+
if (!entries.length || !session || !sessionRpc.queue) return;
|
|
2808
|
+
await refreshQueueLane();
|
|
2809
|
+
for (const entry of entries) {
|
|
2810
|
+
const itemId = queuedLane.get(entry.runtimeId);
|
|
2811
|
+
if (!itemId) continue;
|
|
2812
|
+
try {
|
|
2813
|
+
await session.rpc.queue.removeAt({ id: itemId });
|
|
2814
|
+
dbg('removed a stopped prompt from the runtime queue', entry.message.id);
|
|
2815
|
+
} catch (error) {
|
|
2816
|
+
dbg('queue.removeAt after Stop failed', entry.message.id, error?.message || String(error));
|
|
2817
|
+
}
|
|
2818
|
+
}
|
|
2819
|
+
}
|
|
2820
|
+
|
|
2821
|
+
/**
|
|
2822
|
+
* Hold pushed-but-unstarted prompts for the orphan grace, then return the
|
|
2823
|
+
* ones the runtime still did not open (an opened one was adopted meanwhile
|
|
2824
|
+
* and belongs to a turn of its own now).
|
|
2825
|
+
*/
|
|
2826
|
+
async function settleAfterOrphanGrace(turn, orphans) {
|
|
2827
|
+
if (!orphans.length) return [];
|
|
2828
|
+
if (orphanGraceMs > 0) {
|
|
2829
|
+
dbg('pushed prompts not started at idle; waiting for the runtime to open them', orphans.map((entry) => entry.message.id).join(','));
|
|
2830
|
+
await sleep(orphanGraceMs);
|
|
2831
|
+
}
|
|
2832
|
+
return orphans.filter((entry) => entry.turn === turn && !entry.settled);
|
|
2833
|
+
}
|
|
2834
|
+
|
|
2835
|
+
/**
|
|
2836
|
+
* Publish everything a finished turn still owes, per row.
|
|
2837
|
+
*
|
|
2838
|
+
* Shared by both turn kinds: the runtime does not distinguish a turn it
|
|
2839
|
+
* started from one it was asked for, so neither does the reporting.
|
|
2840
|
+
*/
|
|
2841
|
+
async function finishTurn(turn) {
|
|
2842
|
+
const result = turn.result;
|
|
2843
|
+
const failure = turn.failure;
|
|
2844
|
+
// Capture only. The POST fires from driveTurn's `finally`, after the rows
|
|
2845
|
+
// have been published.
|
|
2846
|
+
captureTurnUsage(turn.primaryEntry.message, result);
|
|
2847
|
+
|
|
2848
|
+
const pending = turn.entries.filter((entry) => !entry.settled && entry.turn === turn);
|
|
2849
|
+
if (!pending.length) return;
|
|
2850
|
+
|
|
2851
|
+
if (failure) {
|
|
2852
|
+
const classified = classifyCopilotTurnException(failure);
|
|
2853
|
+
dbg('copilot turn failed', turn.message?.id || '(continuation)', classified.detail);
|
|
2854
|
+
for (const entry of pending) {
|
|
2855
|
+
if (entry.settled) continue;
|
|
2856
|
+
await settleEntry(turn, entry, { type: 'error', classified });
|
|
2857
|
+
}
|
|
2858
|
+
return;
|
|
2859
|
+
}
|
|
2860
|
+
if (result?.isError) {
|
|
2861
|
+
const classified = classifyCopilotSessionError(result.errorData || { message: result.errorMessage });
|
|
2862
|
+
dbg('turn failed', turn.message?.id || '(continuation)', classified.stableCode);
|
|
2863
|
+
for (const entry of pending) {
|
|
2864
|
+
if (entry.settled) continue;
|
|
2865
|
+
await settleEntry(turn, entry, { type: 'error', classified });
|
|
2866
|
+
}
|
|
2867
|
+
return;
|
|
2868
|
+
}
|
|
2869
|
+
|
|
2870
|
+
const live = turn.currentOwner || turn.primaryEntry;
|
|
2871
|
+
// The row the user pressed Stop on. Normally the live one; the relay's
|
|
2872
|
+
// live-turn picker can also land on another row this turn owns, in which
|
|
2873
|
+
// case THAT row is left to the relay's abort control and the live row
|
|
2874
|
+
// keeps its partial reply. A Stop naming a row that is no longer pending
|
|
2875
|
+
// (its early settle already published, or it is no row of this turn's)
|
|
2876
|
+
// still cut the live run short — and nobody else will settle that row.
|
|
2877
|
+
const namedPending = turn.aborted && turn.abortedRowId
|
|
2878
|
+
? pending.find((entry) => String(entry.message?.id || '') === turn.abortedRowId) || null
|
|
2879
|
+
: null;
|
|
2880
|
+
const stopped = turn.aborted ? namedPending || (turn.abortedRowId ? null : live) : null;
|
|
2881
|
+
const cutRun = turn.aborted && !stopped && !live.settled ? live : null;
|
|
2882
|
+
const settleOne = async (entry) => {
|
|
2883
|
+
// A row that moved to a turn of its own (adoptOrphanedRun) is not ours.
|
|
2884
|
+
if (entry.turn !== turn || entry.settled) return;
|
|
2885
|
+
const ownsRun = entry === live || (!entry.foldedInto && entry.consumed);
|
|
2886
|
+
if (turn.aborted) {
|
|
2887
|
+
// A user Stop. The stopped row is settled server-side; every other
|
|
2888
|
+
// row pushed into the interaction went unanswered — the runtime
|
|
2889
|
+
// clears both its lanes on an interrupt — and says so, with Resend.
|
|
2890
|
+
if (entry === stopped) {
|
|
2891
|
+
dbg('turn aborted', entry.message.id);
|
|
2892
|
+
await settleEntry(turn, entry, { type: 'aborted-by-user' });
|
|
2893
|
+
} else if (entry === cutRun) {
|
|
2894
|
+
dbg('turn aborted by a Stop that named another row', entry.message.id, `named ${turn.abortedRowId}`);
|
|
2895
|
+
await settleEntry(turn, entry, { type: 'user-stopped' });
|
|
2896
|
+
} else if (ownsRun) {
|
|
2897
|
+
await settleEntry(turn, entry, { type: 'completed' });
|
|
2898
|
+
} else {
|
|
2899
|
+
await settleEntry(turn, entry, { type: 'stopped' });
|
|
2900
|
+
}
|
|
2901
|
+
return;
|
|
2902
|
+
}
|
|
2903
|
+
if (result?.aborted) {
|
|
2904
|
+
// The runtime interrupted the turn on its own. Nothing server-side is
|
|
2905
|
+
// waiting to settle these rows.
|
|
2906
|
+
if (entry === live) {
|
|
2907
|
+
dbg('turn interrupted by the runtime', entry.message.id);
|
|
2908
|
+
await settleEntry(turn, entry, { type: 'runtime-interrupted' });
|
|
2909
|
+
} else if (ownsRun) {
|
|
2910
|
+
await settleEntry(turn, entry, { type: 'completed' });
|
|
2911
|
+
} else {
|
|
2912
|
+
await settleEntry(turn, entry, { type: 'stopped' });
|
|
2913
|
+
}
|
|
2914
|
+
return;
|
|
2915
|
+
}
|
|
2916
|
+
if (ownsRun) {
|
|
2917
|
+
await settleEntry(turn, entry, { type: 'completed' });
|
|
2918
|
+
} else if (entry.foldedInto) {
|
|
2919
|
+
await settleEntry(turn, entry, { type: 'folded' });
|
|
2920
|
+
} else {
|
|
2921
|
+
await settleEntry(turn, entry, { type: 'merged-note' });
|
|
2922
|
+
}
|
|
2923
|
+
};
|
|
2924
|
+
// A normal end with pushed prompts the runtime never started: the runtime
|
|
2925
|
+
// gets a moment to open their runs (adoptOrphanedRun takes them) before
|
|
2926
|
+
// they are noted as carried over — AFTER the rows whose replies are
|
|
2927
|
+
// complete have been published, so a finished answer never waits on the
|
|
2928
|
+
// grace. Not on a Stop (the runtime cleared them) and not on an error.
|
|
2929
|
+
const orphans = (!turn.aborted && !result?.aborted)
|
|
2930
|
+
? pending.filter((entry) => entry.role === 'steer' && !entry.consumed && !entry.foldedInto)
|
|
2931
|
+
: [];
|
|
2932
|
+
if (turn.aborted || result?.aborted) {
|
|
2933
|
+
// Belt and braces: `interruptMainTurn({flushQueued:false})` and
|
|
2934
|
+
// `abort()` both empty the runtime's steering AND queued lanes
|
|
2935
|
+
// (probe-verified 2026-09-25), so nothing pushed runs after a Stop. Any
|
|
2936
|
+
// prompt of ours still listed in the queued lane is removed by id anyway
|
|
2937
|
+
// before its row is marked stopped. (The id-less steering lane cannot be
|
|
2938
|
+
// targeted, and needs no help.)
|
|
2939
|
+
await clearOwnedQueuedItems(pending.filter((entry) => !entry.consumed && entry.runtimeId));
|
|
2940
|
+
}
|
|
2941
|
+
// The stopped row first: settling it before the live row's cascade means
|
|
2942
|
+
// the cascade (which settles the steers folded into a run) cannot touch
|
|
2943
|
+
// the row the relay's abort control owns. A cut run goes first too, so
|
|
2944
|
+
// the steers folded into it settle through its cascade, after its stop.
|
|
2945
|
+
const first = stopped || cutRun;
|
|
2946
|
+
const ordered = first ? [first, ...pending.filter((entry) => entry !== first)] : pending;
|
|
2947
|
+
for (const entry of ordered) {
|
|
2948
|
+
if (!orphans.includes(entry)) await settleOne(entry);
|
|
2949
|
+
}
|
|
2950
|
+
for (const entry of await settleAfterOrphanGrace(turn, orphans)) {
|
|
2951
|
+
await settleOne(entry);
|
|
2952
|
+
}
|
|
2953
|
+
}
|
|
2954
|
+
|
|
2955
|
+
// ------------------------------------------------------------ lifecycle --
|
|
2956
|
+
|
|
2957
|
+
/**
|
|
2958
|
+
* The turn a Stop is for. Every turn's control poller runs until its
|
|
2959
|
+
* `driveTurn` finishes, so a turn that is still publishing can claim a Stop
|
|
2960
|
+
* meant for another: the Stop goes to the turn owning the row it names, and
|
|
2961
|
+
* one naming no owned row goes to the turn that is running.
|
|
2962
|
+
*/
|
|
2963
|
+
function stopTarget(turn, control) {
|
|
2964
|
+
const rowId = String(control?.queueMessageId || '').trim();
|
|
2965
|
+
if (rowId) {
|
|
2966
|
+
for (const candidate of [activeTurn, ...publishingTurns]) {
|
|
2967
|
+
if (candidate?.entries.some((entry) => !entry.released && String(entry.message?.id || '') === rowId)) {
|
|
2968
|
+
return candidate;
|
|
2969
|
+
}
|
|
2970
|
+
}
|
|
2971
|
+
}
|
|
2972
|
+
return turn.settled && activeTurn ? activeTurn : turn;
|
|
2973
|
+
}
|
|
2974
|
+
|
|
2975
|
+
/**
|
|
2976
|
+
* Handle a Stop for `turn`. When the row it names settled early and its
|
|
2977
|
+
* reply is still publishing, the Stop resolves only once that reply has
|
|
2978
|
+
* landed: the relay's ack then fails the named row, which is a no-op on a
|
|
2979
|
+
* completed one but would overwrite a reply still in flight.
|
|
2980
|
+
*/
|
|
2981
|
+
async function interruptTurn(turn, control) {
|
|
2982
|
+
const rowId = String(control?.queueMessageId || '').trim() || null;
|
|
2983
|
+
const publishing = rowId
|
|
2984
|
+
? turn.entries.find((entry) => String(entry.message?.id || '') === rowId && entry.settled && !entry.released)
|
|
2985
|
+
: null;
|
|
2986
|
+
await interruptRun(turn, rowId);
|
|
2987
|
+
if (publishing) await withTimeout(publishing.done, STOPPED_ROW_PUBLISH_WAIT_MS, false);
|
|
2988
|
+
}
|
|
2989
|
+
|
|
2990
|
+
/**
|
|
2991
|
+
* Interrupt the turn the user stopped. `interruptMainTurn` (when the
|
|
2992
|
+
* runtime has it) stops the main agent loop and nothing else: background
|
|
2993
|
+
* agents and promoted shells keep running under their own Stop, exactly
|
|
2994
|
+
* like the Claude worker's targeted interrupt. `session.abort()` — the
|
|
2995
|
+
* fallback — also cancels every background agent and clears both message
|
|
2996
|
+
* lanes. Either way the runtime answers with an aborted idle event, which
|
|
2997
|
+
* settles the turn through the normal terminator.
|
|
2998
|
+
*/
|
|
2999
|
+
async function interruptRun(turn, rowId) {
|
|
3000
|
+
if (turn.settled) {
|
|
3001
|
+
// Its run is over and the main loop may already belong to the next
|
|
3002
|
+
// turn, so nothing is interrupted. While it waits on a card (driveTurn)
|
|
3003
|
+
// the Stop ends the cards, and the row settles as stopped.
|
|
3004
|
+
if (turn.humanRequests > 0) {
|
|
3005
|
+
turn.aborted = true;
|
|
3006
|
+
turn.abortedRowId = rowId;
|
|
3007
|
+
turn.abortController.abort();
|
|
3008
|
+
}
|
|
3009
|
+
return;
|
|
3010
|
+
}
|
|
3011
|
+
turn.aborted = true;
|
|
3012
|
+
turn.abortedRowId = rowId;
|
|
3013
|
+
// While `ensureSession` is still connecting there is no session to abort
|
|
3014
|
+
// and this would be a silent no-op; `runTurn` re-checks `turn.aborted`
|
|
3015
|
+
// once the session exists and settles there instead.
|
|
3016
|
+
if (!session) return;
|
|
3017
|
+
if (sessionRpc.interruptMainTurn) {
|
|
3018
|
+
try {
|
|
3019
|
+
const outcome = await session.rpc.interruptMainTurn({ flushQueued: false });
|
|
3020
|
+
if (outcome?.interrupted === false) {
|
|
3021
|
+
// Nothing was processing: no aborted idle will come. The turn is
|
|
3022
|
+
// over as far as the runtime is concerned.
|
|
3023
|
+
dbg('interruptMainTurn found no main turn in flight; settling locally', turn.message?.id || '');
|
|
3024
|
+
turn.settle();
|
|
3025
|
+
}
|
|
3026
|
+
return;
|
|
3027
|
+
} catch (error) {
|
|
3028
|
+
dbg('interruptMainTurn failed; falling back to abort()', error?.message || String(error));
|
|
3029
|
+
}
|
|
3030
|
+
}
|
|
3031
|
+
await session.abort?.();
|
|
3032
|
+
}
|
|
3033
|
+
|
|
3034
|
+
/**
|
|
3035
|
+
* Start a delivered turn: bring the session up, send the prompt, and hand
|
|
3036
|
+
* the turn's lifecycle to `driveTurn`. Resolves when THIS row has settled —
|
|
3037
|
+
* which can be before the turn ends, when a message pushed later ran as its
|
|
3038
|
+
* own run and this one's reply was already complete.
|
|
3039
|
+
*/
|
|
3040
|
+
async function runTurn(turn) {
|
|
3041
|
+
const { message } = turn;
|
|
3042
|
+
const entry = turn.primaryEntry;
|
|
1477
3043
|
const model = resolvePerTurnModel(message);
|
|
1478
3044
|
// The dequeued row's reasoning effort (audit #9). Normalisation happens in
|
|
1479
3045
|
// the switcher: `none`/absent mean "the model's default".
|
|
1480
3046
|
const effort = message?.reasoningEffort ?? null;
|
|
1481
3047
|
const relayMode = message?.relayMode || 'agent';
|
|
1482
3048
|
lastRelayMode = relayMode;
|
|
3049
|
+
turn.model = model;
|
|
1483
3050
|
// Set before the session is touched: the heartbeat's owner-recovery guard
|
|
1484
3051
|
// reads the active ids, so a cold-start delivery must already own its row.
|
|
1485
3052
|
activeTurn = turn;
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
// `assistant.turn_end` → `session.idle{aborted:true}`, which settles
|
|
1497
|
-
// the turn through the normal terminator; this only asks for it.
|
|
1498
|
-
await session.abort?.();
|
|
1499
|
-
},
|
|
1500
|
-
});
|
|
1501
|
-
|
|
3053
|
+
// The delivery hold begins: nothing steers in until the prompt is sent.
|
|
3054
|
+
syncDeliveryReadiness();
|
|
3055
|
+
// Session-wide, not per row: the client's Stop targets whichever of this
|
|
3056
|
+
// turn's rows is live, and the relay only hands out controls for rows
|
|
3057
|
+
// this worker owns.
|
|
3058
|
+
turn.controlState = controlPoller?.start?.({
|
|
3059
|
+
queueMessageId: '',
|
|
3060
|
+
onAbortTurn: (control) => interruptTurn(stopTarget(turn, control), control),
|
|
3061
|
+
}) || null;
|
|
3062
|
+
turn.drive = driveTurn(turn);
|
|
1502
3063
|
try {
|
|
1503
|
-
await ensureSession(model, effort
|
|
3064
|
+
await ensureSession(model, effort);
|
|
1504
3065
|
if (turn.aborted) {
|
|
1505
3066
|
// The abort landed while the session was still being built. Nothing
|
|
1506
|
-
// was sent, so there is no runtime turn to interrupt —
|
|
1507
|
-
//
|
|
1508
|
-
// settle locally rather than sending a prompt the user just cancelled.
|
|
3067
|
+
// was sent, so there is no runtime turn to interrupt — settle locally
|
|
3068
|
+
// rather than sending a prompt the user just cancelled.
|
|
1509
3069
|
dbg('turn aborted before send', message.id);
|
|
1510
|
-
try { await session?.abort?.(); } catch (error) {
|
|
1511
|
-
dbg('abort during session setup failed', error?.message || String(error));
|
|
1512
|
-
}
|
|
1513
3070
|
turn.settle();
|
|
1514
3071
|
} else {
|
|
1515
3072
|
turn.armStall();
|
|
@@ -1522,26 +3079,147 @@ export function createCopilotSdkSessionRunner({
|
|
|
1522
3079
|
// `mode` and `attachments` are FIELDS of the single MessageOptions
|
|
1523
3080
|
// argument — `send()` takes no second parameter, so passing options
|
|
1524
3081
|
// positionally drops them silently.
|
|
1525
|
-
|
|
3082
|
+
const sending = session.send({
|
|
1526
3083
|
prompt,
|
|
1527
3084
|
...(attachments?.length ? { attachments } : {}),
|
|
1528
3085
|
...(sendMode ? { mode: sendMode } : {}),
|
|
1529
3086
|
agentMode: copilotAgentModeForRelayMode(relayMode),
|
|
1530
3087
|
});
|
|
1531
|
-
//
|
|
1532
|
-
//
|
|
1533
|
-
//
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
3088
|
+
// A send that never resolves (a wedged runtime) must not hold this
|
|
3089
|
+
// delivery hostage once the watchdog has failed the turn: the id is
|
|
3090
|
+
// still recorded if it ever arrives, but the wait ends with the turn.
|
|
3091
|
+
const settledFirst = Symbol('turn-settled');
|
|
3092
|
+
const runtimeId = await Promise.race([
|
|
3093
|
+
sending,
|
|
3094
|
+
turn.done.then(() => settledFirst, () => settledFirst),
|
|
3095
|
+
]);
|
|
3096
|
+
if (runtimeId === settledFirst) {
|
|
3097
|
+
sending.then((id) => assignRuntimeId(turn, entry, id)).catch(() => {});
|
|
3098
|
+
} else {
|
|
3099
|
+
assignRuntimeId(turn, entry, runtimeId);
|
|
3100
|
+
// Committed only now that `send()` accepted the prompt (audit #32): a
|
|
3101
|
+
// failed send means the runtime never READ the mode guidance, and
|
|
3102
|
+
// committing before it would make the same-mode retry omit it.
|
|
3103
|
+
context?.commit();
|
|
3104
|
+
// The runtime has the prompt: the delivery hold ends and later
|
|
3105
|
+
// deliveries steer into this turn.
|
|
3106
|
+
turn.primarySent = true;
|
|
3107
|
+
turn.resolvePrimaryReady();
|
|
3108
|
+
syncDeliveryReadiness();
|
|
3109
|
+
}
|
|
1538
3110
|
}
|
|
1539
|
-
}
|
|
1540
|
-
|
|
1541
|
-
|
|
3111
|
+
} catch (error) {
|
|
3112
|
+
// A failure that killed the session (or came from starting it) leaves a
|
|
3113
|
+
// handle nothing else can use; drop it so the next delivery rebuilds and
|
|
3114
|
+
// resumes rather than sending into a dead runtime. An UNCONFIRMED MODEL
|
|
3115
|
+
// SWITCH is the exception: the session is healthy and merely still on
|
|
3116
|
+
// its previous model, and tearing the runtime down over it would kill
|
|
3117
|
+
// any live background work for a selection problem the user fixes by
|
|
3118
|
+
// picking another model.
|
|
3119
|
+
if (!isModelSwitchUnconfirmedError(error)) {
|
|
3120
|
+
await stopRuntime('turn-failure').catch(() => {});
|
|
3121
|
+
}
|
|
3122
|
+
turn.fail(error);
|
|
1542
3123
|
}
|
|
3124
|
+
// `send()` resolving is NOT the turn's completion; the event stream is.
|
|
3125
|
+
await awaitEntry(turn, entry);
|
|
3126
|
+
return true;
|
|
3127
|
+
}
|
|
1543
3128
|
|
|
1544
|
-
|
|
3129
|
+
/**
|
|
3130
|
+
* Wait for a row's settle. A row's settle is not necessarily the turn's
|
|
3131
|
+
* end: a prompt pushed later may still be running on the same interaction,
|
|
3132
|
+
* and this delivery must not wait for it. But when the turn IS over, wait
|
|
3133
|
+
* for its wrap-up too (ownership release, usage ingest), so a caller that
|
|
3134
|
+
* awaited the delivery sees a quiescent runner — the contract every test
|
|
3135
|
+
* and the crash guard rely on. `driveTurn` never rejects.
|
|
3136
|
+
*/
|
|
3137
|
+
async function awaitEntry(turn, entry) {
|
|
3138
|
+
await entry.done;
|
|
3139
|
+
if (turn.settled && turn.drive) await turn.drive.catch(() => {});
|
|
3140
|
+
}
|
|
3141
|
+
|
|
3142
|
+
/**
|
|
3143
|
+
* Run a turn from its first event to its last publish.
|
|
3144
|
+
*
|
|
3145
|
+
* Row ownership is released only after every publish has landed: a
|
|
3146
|
+
* heartbeat firing inside the publish window with no active ids would tell
|
|
3147
|
+
* the relay this worker owns nothing, and the still-`processing` rows would
|
|
3148
|
+
* be recovered underneath it. (The EVENT stream was already released at the
|
|
3149
|
+
* terminator — see `beginPublishing` — so the runtime's next self-initiated
|
|
3150
|
+
* turn can open while this one is still writing.)
|
|
3151
|
+
*/
|
|
3152
|
+
async function driveTurn(turn) {
|
|
3153
|
+
try {
|
|
3154
|
+
await turn.done;
|
|
3155
|
+
} catch (error) {
|
|
3156
|
+
turn.failure = turn.failure || error;
|
|
3157
|
+
}
|
|
3158
|
+
// A card still open on this row — a background agent asking while the
|
|
3159
|
+
// main loop went idle (a root question cannot get here: the loop is
|
|
3160
|
+
// blocked on its handler) — keeps the row processing until it is done:
|
|
3161
|
+
// the relay opens cards only on a processing row, and `/api/response`
|
|
3162
|
+
// cancels the ones still open. The event stream is already released
|
|
3163
|
+
// (`beginPublishing`), so new root work opens a turn of its own meanwhile.
|
|
3164
|
+
if (!turn.aborted && !turn.failure && turn.humanRequests > 0) await turn.humansIdle;
|
|
3165
|
+
try {
|
|
3166
|
+
await quiesceTurn(turn);
|
|
3167
|
+
if (turn.failure && turn.kind === 'delivered' && !isModelSwitchUnconfirmedError(turn.failure) && session) {
|
|
3168
|
+
// A stall or a runtime death mid-turn (a setup failure already tore
|
|
3169
|
+
// the runtime down in runTurn).
|
|
3170
|
+
await stopRuntime('turn-failure').catch(() => {});
|
|
3171
|
+
}
|
|
3172
|
+
if (turn.kind === 'continuation') {
|
|
3173
|
+
// Nothing can be published before the row exists. The buffer is
|
|
3174
|
+
// drained by the registration itself; this only covers actions that
|
|
3175
|
+
// arrived during the drain.
|
|
3176
|
+
await awaitContinuationRow(turn);
|
|
3177
|
+
// The invariant is the id, not the flag: registration can also give up
|
|
3178
|
+
// by throwing, or by taking longer than `awaitContinuationRow` waits,
|
|
3179
|
+
// and publishing against `messageId: null` would attribute the whole
|
|
3180
|
+
// turn to nothing at all.
|
|
3181
|
+
if (!turn.message.id) {
|
|
3182
|
+
if (!turn.discarded) abandonContinuationRegistration(turn, 'no relay row at publish time');
|
|
3183
|
+
dbg('continuation output dropped (no relay row)');
|
|
3184
|
+
const classified = classifyCopilotTurnException(
|
|
3185
|
+
turn.failure || new Error('the relay refused a continuation row for this turn'),
|
|
3186
|
+
);
|
|
3187
|
+
for (const entry of turn.entries) {
|
|
3188
|
+
if (entry.settled || entry.turn !== turn) continue;
|
|
3189
|
+
// The steering path is still waiting on any row it handed us, and
|
|
3190
|
+
// it has no row of its own to fall back to.
|
|
3191
|
+
await settleEntry(turn, entry, entry.role === 'self' ? { type: 'discarded' } : { type: 'error', classified });
|
|
3192
|
+
}
|
|
3193
|
+
return;
|
|
3194
|
+
}
|
|
3195
|
+
await flushBufferedActions(turn);
|
|
3196
|
+
}
|
|
3197
|
+
await finishTurn(turn);
|
|
3198
|
+
} catch (error) {
|
|
3199
|
+
dbg('turn publish failed', error?.message || String(error));
|
|
3200
|
+
} finally {
|
|
3201
|
+
// Early settles still in flight must land before the rows stop being
|
|
3202
|
+
// reported (settleEntry never rejects, but this is the ordering guard).
|
|
3203
|
+
await Promise.allSettled(turn.settlePromises);
|
|
3204
|
+
controlPoller?.stop?.(turn.controlState);
|
|
3205
|
+
turn.controlState = null;
|
|
3206
|
+
// Every path through the turn — published, failed, aborted, threw — has
|
|
3207
|
+
// finished by here. Anything still unresolved is released so no caller
|
|
3208
|
+
// waits forever; its row is left to the relay's recovery.
|
|
3209
|
+
for (const entry of turn.entries) {
|
|
3210
|
+
if (entry.turn !== turn || entry.released) continue;
|
|
3211
|
+
entry.settled = true;
|
|
3212
|
+
entry.released = true;
|
|
3213
|
+
entry.resolveDone(true);
|
|
3214
|
+
}
|
|
3215
|
+
// Released only once every publish for this turn's rows has landed.
|
|
3216
|
+
releaseTurnOwnership(turn);
|
|
3217
|
+
touch();
|
|
3218
|
+
syncDeliveryReadiness();
|
|
3219
|
+
// The ingest is advisory, never awaited, and must never be able to
|
|
3220
|
+
// delay a reply that is already written.
|
|
3221
|
+
postTurnUsage();
|
|
3222
|
+
}
|
|
1545
3223
|
}
|
|
1546
3224
|
|
|
1547
3225
|
/**
|
|
@@ -1563,111 +3241,6 @@ export function createCopilotSdkSessionRunner({
|
|
|
1563
3241
|
await closeStraySubagentRuns(turn).catch(() => {});
|
|
1564
3242
|
}
|
|
1565
3243
|
|
|
1566
|
-
/**
|
|
1567
|
-
* Publish a finished turn's outcome onto its queue row (and every row steered
|
|
1568
|
-
* into it).
|
|
1569
|
-
*
|
|
1570
|
-
* Shared by both turn kinds. A continuation reaches it through
|
|
1571
|
-
* `driveContinuation` instead of `runTurn`, but the outcomes are identical:
|
|
1572
|
-
* the runtime does not distinguish a turn it started from one it was asked
|
|
1573
|
-
* for, so neither does the reporting.
|
|
1574
|
-
*/
|
|
1575
|
-
async function finishTurn(turn, model) {
|
|
1576
|
-
const { message } = turn;
|
|
1577
|
-
const result = turn.result;
|
|
1578
|
-
const responseModel = result?.model || turn.normalizer.model || model || null;
|
|
1579
|
-
// Capture only. The POST fires from `handlePendingPayload`'s `finally`,
|
|
1580
|
-
// after this row has been published.
|
|
1581
|
-
captureTurnUsage(message, result);
|
|
1582
|
-
|
|
1583
|
-
// Whatever this interaction did not get to stays queued in the runtime and
|
|
1584
|
-
// shifts the NEXT interaction's segments: the prompts this turn sent occupy
|
|
1585
|
-
// `[firstSentSegment, nextSegmentIndex)`, and the ones the runtime never
|
|
1586
|
-
// opened a segment for are still in its pending queue.
|
|
1587
|
-
//
|
|
1588
|
-
// The `max` against `firstSentSegment` is what makes this correct for a
|
|
1589
|
-
// continuation, which occupies segment 0 without having sent anything: a
|
|
1590
|
-
// continuation that opened no segment at all (an empty self-initiated turn)
|
|
1591
|
-
// would otherwise report one carried prompt and shift the next real turn's
|
|
1592
|
-
// answer by one.
|
|
1593
|
-
const segmentsOpened = turn.normalizer.promptCount();
|
|
1594
|
-
carriedPrompts = Math.max(0, turn.nextSegmentIndex - Math.max(turn.firstSentSegment, segmentsOpened));
|
|
1595
|
-
|
|
1596
|
-
// This row's own reply. Falls back to the whole composed text when the
|
|
1597
|
-
// runtime opened no segment at all (an empty or immediately-failed turn),
|
|
1598
|
-
// which is strictly better than publishing nothing.
|
|
1599
|
-
const ownText = String(
|
|
1600
|
-
turn.normalizer.segmentText(turn.baseSegment) || result?.text || turn.state.lastStreamedText || '',
|
|
1601
|
-
).trim();
|
|
1602
|
-
|
|
1603
|
-
// A user-initiated abort publishes the partial text and nothing else: the
|
|
1604
|
-
// queue row's fate belongs to the server-side abort control, exactly as in
|
|
1605
|
-
// the Claude and Cursor workers. Publishing a response here would
|
|
1606
|
-
// double-settle the row.
|
|
1607
|
-
if (turn.aborted) {
|
|
1608
|
-
dbg('turn aborted', message.id);
|
|
1609
|
-
// This row's segment only: the steered rows publish their own, and
|
|
1610
|
-
// publishing the composed text here would show their replies twice.
|
|
1611
|
-
await publishFinalStream(message, ownText);
|
|
1612
|
-
// A steered row is NOT covered by the abort control (which knows only
|
|
1613
|
-
// about the row the user aborted), so it still has to be settled here or
|
|
1614
|
-
// it holds `processing` forever behind a renewing lease.
|
|
1615
|
-
await settleSteeredRows(turn, result, responseModel);
|
|
1616
|
-
return true;
|
|
1617
|
-
}
|
|
1618
|
-
|
|
1619
|
-
// The runtime aborted on its own (`result.aborted` with no relay-side
|
|
1620
|
-
// abort control in flight). Nothing server-side is waiting to settle this
|
|
1621
|
-
// row, so returning here would leave it pending until the delivery
|
|
1622
|
-
// watchdog fails it with a misleading "Relay timeout" — the row has to be
|
|
1623
|
-
// settled with a record that says what actually happened.
|
|
1624
|
-
if (result?.aborted) {
|
|
1625
|
-
dbg('turn interrupted by the runtime', message.id);
|
|
1626
|
-
const text = ownText ? `${ownText}\n\n${RUNTIME_INTERRUPTED_NOTE}` : RUNTIME_INTERRUPTED_NOTE;
|
|
1627
|
-
await publishFinalStream(message, text);
|
|
1628
|
-
await publishResponse(message, { text, model: responseModel });
|
|
1629
|
-
await settleSteeredRows(turn, result, responseModel);
|
|
1630
|
-
return true;
|
|
1631
|
-
}
|
|
1632
|
-
|
|
1633
|
-
if (result?.isError) {
|
|
1634
|
-
const classified = classifyCopilotSessionError(result.errorData || { message: result.errorMessage });
|
|
1635
|
-
dbg('turn failed', message.id, classified.stableCode);
|
|
1636
|
-
await publishFinalStream(message, turn.state.lastStreamedText);
|
|
1637
|
-
await publishResponse(message, {
|
|
1638
|
-
text: classified.text,
|
|
1639
|
-
model: responseModel,
|
|
1640
|
-
terminalError: terminalErrorRecord(message, classified),
|
|
1641
|
-
});
|
|
1642
|
-
// The steered rows failed with it — they were being answered by the same
|
|
1643
|
-
// interaction. They get the same terminal record so each row says why.
|
|
1644
|
-
await settleSteeredRows(turn, result, responseModel, classified);
|
|
1645
|
-
return true;
|
|
1646
|
-
}
|
|
1647
|
-
|
|
1648
|
-
// The turn produced a plan but never called exit-plan-mode (or the runtime
|
|
1649
|
-
// build has no such hook). Same text-shape fallback the siblings use, and
|
|
1650
|
-
// it must go out BEFORE the response: `/api/relay-board` 409s once the
|
|
1651
|
-
// queue row leaves `processing`.
|
|
1652
|
-
if (shouldPostPlanBoard({
|
|
1653
|
-
relayMode: message.relayMode,
|
|
1654
|
-
finalText: ownText,
|
|
1655
|
-
alreadyPosted: turn.planBoardPosted,
|
|
1656
|
-
acted: turn.acted === true,
|
|
1657
|
-
})) {
|
|
1658
|
-
await publishPlanBoard(turn, ownText, 'plan-mode-fallback');
|
|
1659
|
-
}
|
|
1660
|
-
|
|
1661
|
-
// A terminal, non-error turn with no prose is COMPLETE, not a failed
|
|
1662
|
-
// delivery — requeuing re-runs deterministically empty work until the
|
|
1663
|
-
// retry cap fails the row with a misleading "Relay timeout".
|
|
1664
|
-
const publishedText = ownText || EMPTY_TURN_COMPLETION_NOTE;
|
|
1665
|
-
await publishFinalStream(message, publishedText);
|
|
1666
|
-
await publishResponse(message, { text: publishedText, model: responseModel });
|
|
1667
|
-
await settleSteeredRows(turn, result, responseModel);
|
|
1668
|
-
return true;
|
|
1669
|
-
}
|
|
1670
|
-
|
|
1671
3244
|
// ----------------------------------------------------------- continuation --
|
|
1672
3245
|
|
|
1673
3246
|
/**
|
|
@@ -1696,14 +3269,15 @@ export function createCopilotSdkSessionRunner({
|
|
|
1696
3269
|
// The gap this pin covers has closed.
|
|
1697
3270
|
continuationDueSince = 0;
|
|
1698
3271
|
turn.armStall();
|
|
1699
|
-
// Lines produced between turns (a settled shell's
|
|
1700
|
-
// the turn they triggered.
|
|
3272
|
+
// Lines produced between turns (a settled shell's or background agent's
|
|
3273
|
+
// notification) belong to the turn they triggered.
|
|
1701
3274
|
if (pendingActivities.length) {
|
|
1702
|
-
for (const text of pendingActivities.splice(0)) {
|
|
1703
|
-
turn.bufferedActions.push({ channel: 'activity', payload: { text, subagentRunId: null } });
|
|
3275
|
+
for (const { text, reportsFailure } of pendingActivities.splice(0)) {
|
|
3276
|
+
turn.bufferedActions.push({ channel: 'activity', payload: { text, subagentRunId: null }, carried: true, reportsFailure });
|
|
1704
3277
|
}
|
|
1705
3278
|
}
|
|
1706
3279
|
dbg('opening a continuation turn for runtime-initiated work');
|
|
3280
|
+
syncDeliveryReadiness();
|
|
1707
3281
|
// Both are fire-and-forget by design (the SDK's event callback is
|
|
1708
3282
|
// synchronous and cannot await a turn), so both must swallow: an unhandled
|
|
1709
3283
|
// rejection here would reach the worker crash guard and take the whole
|
|
@@ -1717,16 +3291,54 @@ export function createCopilotSdkSessionRunner({
|
|
|
1717
3291
|
// has somewhere to go.
|
|
1718
3292
|
turn.registered = true;
|
|
1719
3293
|
turn.resolveRowReady?.(true);
|
|
3294
|
+
syncDeliveryReadiness();
|
|
1720
3295
|
return;
|
|
1721
3296
|
}
|
|
1722
3297
|
abandonContinuationRegistration(turn, 'registration threw');
|
|
1723
3298
|
});
|
|
1724
|
-
|
|
3299
|
+
turn.drive = driveTurn(turn);
|
|
3300
|
+
turn.drive.catch((error) => {
|
|
1725
3301
|
dbg('continuation driver threw', error?.message || String(error));
|
|
1726
3302
|
});
|
|
1727
3303
|
return turn;
|
|
1728
3304
|
}
|
|
1729
3305
|
|
|
3306
|
+
/**
|
|
3307
|
+
* A pushed row whose run the runtime opened AFTER the turn it was pushed
|
|
3308
|
+
* into had settled (the drain's idle raced the send). It is not a
|
|
3309
|
+
* self-initiated turn — it is that row's own turn, and its reply belongs on
|
|
3310
|
+
* that row. Returns the new turn, or null when the id is not one of ours.
|
|
3311
|
+
*/
|
|
3312
|
+
function adoptOrphanedRun(event) {
|
|
3313
|
+
if (String(event?.type || '') !== 'user.message' || event?.agentId) return null;
|
|
3314
|
+
const runtimeId = String(event?.data?.messageId || '').trim();
|
|
3315
|
+
if (!runtimeId) return null;
|
|
3316
|
+
for (const previous of publishingTurns) {
|
|
3317
|
+
const entry = previous.pushed.get(runtimeId);
|
|
3318
|
+
if (!entry || entry.consumed || entry.settled) continue;
|
|
3319
|
+
previous.entries = previous.entries.filter((other) => other !== entry);
|
|
3320
|
+
previous.pushed.delete(runtimeId);
|
|
3321
|
+
// From here the entry belongs to the new turn (`entry.turn`); the old
|
|
3322
|
+
// turn's settle passes skip rows that moved away.
|
|
3323
|
+
const turn = createTurn(entry.message, { kind: 'delivered', adoptEntry: entry });
|
|
3324
|
+
turn.model = resolvePerTurnModel(entry.message);
|
|
3325
|
+
activeTurn = turn;
|
|
3326
|
+
turn.armStall();
|
|
3327
|
+
turn.controlState = controlPoller?.start?.({
|
|
3328
|
+
queueMessageId: '',
|
|
3329
|
+
onAbortTurn: (control) => interruptTurn(stopTarget(turn, control), control),
|
|
3330
|
+
}) || null;
|
|
3331
|
+
dbg('a pushed prompt opened its own run after its turn settled; adopting it', entry.message.id);
|
|
3332
|
+
syncDeliveryReadiness();
|
|
3333
|
+
turn.drive = driveTurn(turn);
|
|
3334
|
+
turn.drive.catch((error) => {
|
|
3335
|
+
dbg('adopted turn driver threw', error?.message || String(error));
|
|
3336
|
+
});
|
|
3337
|
+
return turn;
|
|
3338
|
+
}
|
|
3339
|
+
return null;
|
|
3340
|
+
}
|
|
3341
|
+
|
|
1730
3342
|
/**
|
|
1731
3343
|
* Give up on a continuation's synthetic row: nothing buffered will ever
|
|
1732
3344
|
* publish, and a registration still in flight must not adopt a row into this
|
|
@@ -1738,8 +3350,10 @@ export function createCopilotSdkSessionRunner({
|
|
|
1738
3350
|
turn.discarded = true;
|
|
1739
3351
|
turn.bufferedActions = [];
|
|
1740
3352
|
turn.registrationAbort?.abort?.();
|
|
1741
|
-
// Interactive handlers stop waiting and take their degraded path
|
|
3353
|
+
// Interactive handlers stop waiting and take their degraded path, and the
|
|
3354
|
+
// delivery hold that waited for the row ends.
|
|
1742
3355
|
turn.resolveRowReady?.(false);
|
|
3356
|
+
syncDeliveryReadiness();
|
|
1743
3357
|
dbg('continuation registration abandoned:', reason);
|
|
1744
3358
|
}
|
|
1745
3359
|
|
|
@@ -1813,16 +3427,14 @@ export function createCopilotSdkSessionRunner({
|
|
|
1813
3427
|
// Only now is there a row to abort, so this is where the control poller can
|
|
1814
3428
|
// start.
|
|
1815
3429
|
turn.controlState = controlPoller?.start?.({
|
|
1816
|
-
queueMessageId:
|
|
1817
|
-
onAbortTurn:
|
|
1818
|
-
turn.aborted = true;
|
|
1819
|
-
if (!session) return;
|
|
1820
|
-
await session.abort?.();
|
|
1821
|
-
},
|
|
3430
|
+
queueMessageId: '',
|
|
3431
|
+
onAbortTurn: (control) => interruptTurn(stopTarget(turn, control), control),
|
|
1822
3432
|
}) || null;
|
|
1823
3433
|
await flushBufferedActions(turn);
|
|
1824
3434
|
turn.registered = true;
|
|
1825
3435
|
turn.resolveRowReady?.(true);
|
|
3436
|
+
// The row exists: the delivery hold it imposed ends (canAcceptSteering).
|
|
3437
|
+
syncDeliveryReadiness();
|
|
1826
3438
|
}
|
|
1827
3439
|
|
|
1828
3440
|
/**
|
|
@@ -1846,184 +3458,347 @@ export function createCopilotSdkSessionRunner({
|
|
|
1846
3458
|
}
|
|
1847
3459
|
}
|
|
1848
3460
|
|
|
1849
|
-
|
|
1850
|
-
* Run a continuation to its terminator and publish it, mirroring
|
|
1851
|
-
* `handlePendingPayload`'s outer shape.
|
|
1852
|
-
*
|
|
1853
|
-
* Row ownership is released only after every publish has landed, exactly as
|
|
1854
|
-
* for a delivered turn: a heartbeat firing inside the publish window with no
|
|
1855
|
-
* active ids would tell the relay this worker owns nothing, and the
|
|
1856
|
-
* still-`processing` synthetic row would be recovered underneath it. (The
|
|
1857
|
-
* EVENT stream was already released at the terminator — see
|
|
1858
|
-
* `beginPublishing` — so the runtime's next self-initiated turn can open
|
|
1859
|
-
* while this one is still writing.)
|
|
1860
|
-
*/
|
|
1861
|
-
async function driveContinuation(turn) {
|
|
1862
|
-
let failure = null;
|
|
1863
|
-
try {
|
|
1864
|
-
await turn.done;
|
|
1865
|
-
} catch (error) {
|
|
1866
|
-
failure = error;
|
|
1867
|
-
}
|
|
1868
|
-
try {
|
|
1869
|
-
await quiesceTurn(turn);
|
|
1870
|
-
// Nothing can be published before the row exists. The buffer is drained
|
|
1871
|
-
// by the registration itself; this only covers actions that arrived
|
|
1872
|
-
// during the drain.
|
|
1873
|
-
await awaitContinuationRow(turn);
|
|
1874
|
-
// The invariant is the id, not the flag: registration can also give up by
|
|
1875
|
-
// throwing, or by taking longer than `awaitContinuationRow` waits, and
|
|
1876
|
-
// publishing against `messageId: null` would attribute the whole turn to
|
|
1877
|
-
// nothing at all.
|
|
1878
|
-
if (!turn.message.id) {
|
|
1879
|
-
if (!turn.discarded) abandonContinuationRegistration(turn, 'no relay row at publish time');
|
|
1880
|
-
dbg('continuation output dropped (no relay row)');
|
|
1881
|
-
// The steering path is still waiting on any row it handed us, and it
|
|
1882
|
-
// has no row of its own to fall back to.
|
|
1883
|
-
await settleSteeredRows(turn, null, null, classifyCopilotTurnException(
|
|
1884
|
-
failure || new Error('the relay refused a continuation row for this turn'),
|
|
1885
|
-
)).catch(() => {});
|
|
1886
|
-
return;
|
|
1887
|
-
}
|
|
1888
|
-
await flushBufferedActions(turn);
|
|
1889
|
-
if (failure) {
|
|
1890
|
-
const classified = classifyCopilotTurnException(failure);
|
|
1891
|
-
dbg('continuation turn failed', turn.message.id, classified.detail);
|
|
1892
|
-
await publishResponse(turn.message, {
|
|
1893
|
-
text: classified.text,
|
|
1894
|
-
model: null,
|
|
1895
|
-
terminalError: terminalErrorRecord(turn.message, classified),
|
|
1896
|
-
});
|
|
1897
|
-
await settleSteeredRows(turn, null, null, classified).catch(() => {});
|
|
1898
|
-
return;
|
|
1899
|
-
}
|
|
1900
|
-
await finishTurn(turn, '');
|
|
1901
|
-
} catch (error) {
|
|
1902
|
-
dbg('continuation publish failed', error?.message || String(error));
|
|
1903
|
-
} finally {
|
|
1904
|
-
controlPoller?.stop?.(turn.controlState);
|
|
1905
|
-
turn.controlState = null;
|
|
1906
|
-
releaseTurnOwnership(turn);
|
|
1907
|
-
touch();
|
|
1908
|
-
// A continuation spends real quota (the live capture burned a premium
|
|
1909
|
-
// request on `read_bash` + the reply), so its numbers ride the same
|
|
1910
|
-
// fire-and-forget ingest as a delivered turn's.
|
|
1911
|
-
postTurnUsage();
|
|
1912
|
-
}
|
|
1913
|
-
}
|
|
3461
|
+
// --------------------------------------------------------------- steering --
|
|
1914
3462
|
|
|
1915
3463
|
/**
|
|
1916
|
-
*
|
|
3464
|
+
* A delivery arrived while a turn was already running: push it into the
|
|
3465
|
+
* same runtime interaction as a steer.
|
|
1917
3466
|
*
|
|
1918
|
-
*
|
|
1919
|
-
*
|
|
1920
|
-
*
|
|
1921
|
-
*
|
|
1922
|
-
*
|
|
3467
|
+
* `mode: "immediate"` makes it a real steer — injected at the next tool
|
|
3468
|
+
* boundary (`delivery:"steering"`, folded into the running reply), or run
|
|
3469
|
+
* as its own turn right after when the model is mid-stream
|
|
3470
|
+
* (`delivery:"queued"`). Either way the row is owned by this turn and
|
|
3471
|
+
* settled by it (see adoptSegment / finishTurn); nothing else will.
|
|
1923
3472
|
*
|
|
1924
|
-
*
|
|
1925
|
-
*
|
|
1926
|
-
* pending queue and will be answered at the start of the next turn, so
|
|
1927
|
-
* redelivering it would run the same prompt twice. It gets a note instead —
|
|
1928
|
-
* the same trade the Claude worker makes for a handed-off context.
|
|
1929
|
-
*/
|
|
1930
|
-
async function settleSteeredRows(turn, result, responseModel, classified = null) {
|
|
1931
|
-
if (!turn.steeredRows.length) return;
|
|
1932
|
-
for (const steered of turn.steeredRows) {
|
|
1933
|
-
const { message: steeredMessage, segmentIndex } = steered;
|
|
1934
|
-
let text = String(turn.normalizer.segmentText(segmentIndex) || '').trim();
|
|
1935
|
-
if (classified) {
|
|
1936
|
-
text = classified.text;
|
|
1937
|
-
} else if (!text) {
|
|
1938
|
-
text = STEERED_ROW_MERGED_NOTE;
|
|
1939
|
-
}
|
|
1940
|
-
await publishFinalStream(steeredMessage, text);
|
|
1941
|
-
await publishResponse(steeredMessage, {
|
|
1942
|
-
text,
|
|
1943
|
-
model: responseModel,
|
|
1944
|
-
...(classified ? { terminalError: terminalErrorRecord(steeredMessage, classified) } : {}),
|
|
1945
|
-
});
|
|
1946
|
-
steered.settle?.();
|
|
1947
|
-
}
|
|
1948
|
-
turn.steeredRows.length = 0;
|
|
1949
|
-
}
|
|
1950
|
-
|
|
1951
|
-
/**
|
|
1952
|
-
* A delivery arrived while a turn was already running.
|
|
1953
|
-
*
|
|
1954
|
-
* The relay's worker socket is single-flight — it will not deliver a second
|
|
1955
|
-
* row until `onDeliver` resolves — so this is not the normal path. It is
|
|
1956
|
-
* reachable on a socket reconnect that redelivers, and it is the path a
|
|
1957
|
-
* future relay change would take, so it steers rather than corrupting the
|
|
1958
|
-
* turn: the prompt is queued into the SAME interaction (which is all
|
|
1959
|
-
* `mode: "enqueue"` can do — it cannot interrupt an in-flight model call),
|
|
1960
|
-
* and the row is adopted so the interaction's terminator settles it too.
|
|
1961
|
-
*
|
|
1962
|
-
* The row is registered BEFORE the send so a failure between the two cannot
|
|
1963
|
-
* leave a row nobody owns.
|
|
3473
|
+
* The entry is registered BEFORE the send so a failure between the two
|
|
3474
|
+
* cannot leave a row nobody owns.
|
|
1964
3475
|
*/
|
|
1965
3476
|
async function steerIntoActiveTurn(turn, message) {
|
|
1966
3477
|
const { prompt: body, attachments } = buildMessageOptionsImpl(message);
|
|
1967
3478
|
// A real relay round trip on the first turn of a mode — long enough for the
|
|
1968
3479
|
// turn to finish underneath us.
|
|
1969
3480
|
const context = await buildRelayContextPrefix(message).catch(() => null);
|
|
1970
|
-
|
|
3481
|
+
// The hidden steer note (shared/steer-note.mjs): a steer that lands before
|
|
3482
|
+
// the model's first output otherwise tempts it to answer only the newest
|
|
3483
|
+
// message and drop the request it was working on (burn-in 2026-09-25).
|
|
3484
|
+
// Not in a continuation: there the running work is the runtime's own
|
|
3485
|
+
// follow-up, not a request of the user's to keep going with.
|
|
3486
|
+
const prompt = withRelayContext(context?.prefix, turn.kind === 'continuation' ? body : withSteerNote(body));
|
|
1971
3487
|
// Re-checked AFTER the awaits and before anything is sent or registered.
|
|
1972
|
-
//
|
|
1973
|
-
//
|
|
1974
|
-
// wedging the single-flight delivery socket. Nothing has been sent yet, so
|
|
1975
|
-
// handing the row back to the normal path is free.
|
|
3488
|
+
// Nothing has been sent yet, so handing the row back to the normal path is
|
|
3489
|
+
// free.
|
|
1976
3490
|
if (turn.settled) {
|
|
1977
3491
|
dbg('turn settled while steering was preparing; running it as a fresh turn', message.id);
|
|
1978
3492
|
return NOT_STEERED;
|
|
1979
3493
|
}
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
//
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
turn.promptsSent += 1;
|
|
1987
|
-
const steered = { message, segmentIndex };
|
|
1988
|
-
steered.done = new Promise((resolve) => { steered.settle = resolve; });
|
|
1989
|
-
turn.steeredRows.push(steered);
|
|
1990
|
-
dbg('steering a mid-turn delivery into the running turn', message.id, `segment=${segmentIndex}`);
|
|
3494
|
+
const entry = createEntry(turn, message, 'steer');
|
|
3495
|
+
dbg('steering a mid-turn delivery into the running turn', message.id);
|
|
3496
|
+
let runtimeId;
|
|
3497
|
+
// Counted until send() names the message (see `steerSendsInFlight`).
|
|
3498
|
+
steerSendsInFlight += 1;
|
|
3499
|
+
entry.sendSeq = ++steerSendSeq;
|
|
1991
3500
|
try {
|
|
1992
|
-
await session.send({
|
|
3501
|
+
runtimeId = await session.send({
|
|
1993
3502
|
prompt,
|
|
1994
3503
|
...(attachments?.length ? { attachments } : {}),
|
|
1995
|
-
mode: '
|
|
3504
|
+
mode: 'immediate',
|
|
1996
3505
|
agentMode: copilotAgentModeForRelayMode(message?.relayMode || 'agent'),
|
|
1997
3506
|
});
|
|
1998
3507
|
} catch (error) {
|
|
1999
3508
|
// The prompt never reached the runtime, so nothing will answer it and the
|
|
2000
3509
|
// row is safe to requeue — unlike an accepted one.
|
|
2001
|
-
turn.
|
|
3510
|
+
turn.entries = turn.entries.filter((other) => other !== entry);
|
|
3511
|
+
entry.settled = true;
|
|
3512
|
+
entry.released = true;
|
|
3513
|
+
entry.resolveDone(true);
|
|
3514
|
+
steerSendsInFlight = Math.max(0, steerSendsInFlight - 1);
|
|
2002
3515
|
dbg('steering send failed, requeuing the row', message.id, error?.message || String(error));
|
|
2003
3516
|
await api('POST', '/api/requeue', { messageId: message.id, ...attemptFields(message) }).catch(() => {});
|
|
2004
3517
|
return true;
|
|
2005
3518
|
}
|
|
3519
|
+
assignRuntimeId(turn, entry, runtimeId);
|
|
3520
|
+
steerSendsInFlight = Math.max(0, steerSendsInFlight - 1);
|
|
2006
3521
|
// Same commit-after-send rule as `runTurn` (audit #32): a steered prompt
|
|
2007
3522
|
// whose send failed never delivered its guidance, so the retry must
|
|
2008
3523
|
// include it again.
|
|
2009
3524
|
context?.commit();
|
|
2010
|
-
|
|
2011
|
-
|
|
3525
|
+
scheduleQueueLaneRefresh();
|
|
3526
|
+
// Resolves when the interaction settles this row.
|
|
3527
|
+
await awaitEntry(turn, entry);
|
|
3528
|
+
return true;
|
|
3529
|
+
}
|
|
3530
|
+
|
|
3531
|
+
// ------------------------------------------------------------------ gate --
|
|
3532
|
+
|
|
3533
|
+
/**
|
|
3534
|
+
* Whether a message delivered now would steer into the live turn. False
|
|
3535
|
+
* while: no session / no live turn; the turn-opening prompt has not been
|
|
3536
|
+
* sent yet, or a continuation's row does not exist yet (the delivery hold —
|
|
3537
|
+
* a second send before the first would invert the runs, and a steer's run
|
|
3538
|
+
* would settle the continuation's reply against no row while its buffered
|
|
3539
|
+
* output flushed into the steered one); a question card, ask-mode approval
|
|
3540
|
+
* or elicitation is open (the relay must not push past a card); a
|
|
3541
|
+
* compaction is running.
|
|
3542
|
+
*/
|
|
3543
|
+
function canAcceptSteering() {
|
|
3544
|
+
if (disposed || !session) return false;
|
|
3545
|
+
const turn = activeTurn;
|
|
3546
|
+
if (!turn || turn.settled || turn.aborted) return false;
|
|
3547
|
+
if (!turn.primarySent || awaitingRow(turn)) return false;
|
|
3548
|
+
if (pendingHumanRequests > 0) return false;
|
|
3549
|
+
if (isCompacting()) return false;
|
|
3550
|
+
return true;
|
|
3551
|
+
}
|
|
3552
|
+
|
|
3553
|
+
/** A continuation whose synthetic row is still being minted (and not given up on). */
|
|
3554
|
+
function awaitingRow(turn) {
|
|
3555
|
+
return turn.kind === 'continuation' && !turn.registered && !turn.discarded;
|
|
3556
|
+
}
|
|
3557
|
+
|
|
3558
|
+
/**
|
|
3559
|
+
* The one steering gate. A delivery arriving now is held — handed back to
|
|
3560
|
+
* the queue instead of pushed — when a turn is live (or a human request /
|
|
3561
|
+
* compaction is open) and it cannot accept a steer. The socket link consults
|
|
3562
|
+
* this for its readiness too, idle or mid-delivery.
|
|
3563
|
+
*/
|
|
3564
|
+
function isDeliveryHeld() {
|
|
3565
|
+
if (disposed) return false;
|
|
3566
|
+
const turn = activeTurn;
|
|
3567
|
+
const live = Boolean(turn && !turn.settled);
|
|
3568
|
+
if (!live && pendingHumanRequests === 0 && !isCompacting()) return false;
|
|
3569
|
+
return !canAcceptSteering();
|
|
3570
|
+
}
|
|
3571
|
+
|
|
3572
|
+
function syncDeliveryReadiness() {
|
|
3573
|
+
const ready = !isDeliveryHeld();
|
|
3574
|
+
if (ready === lastReportedDeliveryReady) return;
|
|
3575
|
+
lastReportedDeliveryReady = ready;
|
|
3576
|
+
try {
|
|
3577
|
+
onDeliveryReadinessChange(ready);
|
|
3578
|
+
} catch (error) {
|
|
3579
|
+
dbg('delivery readiness listener failed', error?.message || String(error));
|
|
3580
|
+
}
|
|
3581
|
+
}
|
|
3582
|
+
|
|
3583
|
+
/**
|
|
3584
|
+
* Give a held delivery back to the queue with no retry penalty. The relay
|
|
3585
|
+
* re-delivers it once this worker signals ready again, i.e. when the hold
|
|
3586
|
+
* ends — the message then steers into the resumed turn. Pushing it into the
|
|
3587
|
+
* hold instead would bypass the question card or land inside a compaction.
|
|
3588
|
+
*/
|
|
3589
|
+
async function handBackHeldDelivery(message) {
|
|
3590
|
+
dbg('steering held; handing the delivery back to the queue', message.id || '(no id)');
|
|
3591
|
+
if (!message.id) return false;
|
|
3592
|
+
await api('POST', '/api/requeue', {
|
|
3593
|
+
messageId: message.id,
|
|
3594
|
+
class: 'steering-held',
|
|
3595
|
+
...attemptFields(message),
|
|
3596
|
+
}).catch((error) => {
|
|
3597
|
+
if (isStaleAttemptError(error)) {
|
|
3598
|
+
dbg('steering-held hand-back refused as stale_attempt', message.id);
|
|
3599
|
+
return;
|
|
3600
|
+
}
|
|
3601
|
+
dbg('steering-held hand-back failed', message.id, error?.message || String(error));
|
|
3602
|
+
});
|
|
3603
|
+
return false;
|
|
3604
|
+
}
|
|
3605
|
+
|
|
3606
|
+
function* ownedEntries() {
|
|
3607
|
+
if (activeTurn) yield* activeTurn.entries;
|
|
3608
|
+
for (const turn of publishingTurns) yield* turn.entries;
|
|
3609
|
+
}
|
|
3610
|
+
|
|
3611
|
+
/**
|
|
3612
|
+
* Pushed rows the client may still cancel: unconsumed and recallable — in
|
|
3613
|
+
* practice the newest waiting steer (immediate sends wait in the runtime's
|
|
3614
|
+
* id-less steering lane), or any row that sits in the queued lane.
|
|
3615
|
+
*/
|
|
3616
|
+
function cancellableIds() {
|
|
3617
|
+
const ids = [];
|
|
3618
|
+
for (const entry of unconsumedSteers()) {
|
|
3619
|
+
if (!recallMethodFor(entry)) continue;
|
|
3620
|
+
const id = String(entry.message?.id || '').trim();
|
|
3621
|
+
if (id && !ids.includes(id)) ids.push(id);
|
|
3622
|
+
}
|
|
3623
|
+
return ids;
|
|
3624
|
+
}
|
|
3625
|
+
|
|
3626
|
+
/**
|
|
3627
|
+
* Composer-facing steering snapshot, published on the heartbeat: whether a
|
|
3628
|
+
* turn is live, whether a message typed now would steer into it, and — when
|
|
3629
|
+
* it would not — why. `supported` is how the client learns this Copilot
|
|
3630
|
+
* conversation steers at all (its provider type alone cannot tell an SDK
|
|
3631
|
+
* worker from the extension engine); `cancellableIds` are the pushed rows
|
|
3632
|
+
* that may still be pulled back out of the runtime (decision 8).
|
|
3633
|
+
*/
|
|
3634
|
+
function steeringState() {
|
|
3635
|
+
const turn = activeTurn;
|
|
3636
|
+
const turnActive = Boolean(turn && !turn.settled);
|
|
3637
|
+
const canSteer = canAcceptSteering();
|
|
3638
|
+
let holdReason = null;
|
|
3639
|
+
if (turnActive && !canSteer) {
|
|
3640
|
+
if (pendingHumanRequests > 0) holdReason = 'question';
|
|
3641
|
+
else if (isCompacting()) holdReason = 'compaction';
|
|
3642
|
+
else if (!turn.primarySent || awaitingRow(turn)) holdReason = 'delivery';
|
|
3643
|
+
else holdReason = 'other';
|
|
3644
|
+
}
|
|
3645
|
+
const messageId = String(turn?.currentOwner?.message?.id || turn?.message?.id || '').trim() || null;
|
|
3646
|
+
return { turnActive, canSteer, holdReason, messageId, supported: true, cancellableIds: cancellableIds() };
|
|
3647
|
+
}
|
|
3648
|
+
|
|
3649
|
+
/**
|
|
3650
|
+
* Re-read the queue, and — only if the prompt is still recallable at that
|
|
3651
|
+
* instant — issue its remove request in the same tick as the check, so no
|
|
3652
|
+
* steer send can be issued in between (JSON-RPC keeps request order).
|
|
3653
|
+
*/
|
|
3654
|
+
async function recallPushedPrompt(entry, id) {
|
|
3655
|
+
// Watched from before the read, so a fold that lands during the read or
|
|
3656
|
+
// the remove is seen however far behind the dispatch chain runs.
|
|
3657
|
+
const watch = { taken: false };
|
|
3658
|
+
recallWatches.set(entry.runtimeId, watch);
|
|
3659
|
+
try {
|
|
3660
|
+
// Only the READ is bounded: a runtime that will not answer it gets no
|
|
3661
|
+
// remove at all. Once a remove is sent it is seen through to its answer —
|
|
3662
|
+
// giving up on it could leave the prompt removed and the row uncancelled.
|
|
3663
|
+
const fresh = await withTimeout(refreshQueueLane().then(() => true), UNSTEER_TIMEOUT_MS, false);
|
|
3664
|
+
if (!fresh) {
|
|
3665
|
+
dbg('un-steer refused: the runtime did not answer the queue read in time', id);
|
|
3666
|
+
return false;
|
|
3667
|
+
}
|
|
3668
|
+
const target = session;
|
|
3669
|
+
const method = watch.taken ? null : recallMethodFor(entry);
|
|
3670
|
+
if (!target || !method) {
|
|
3671
|
+
dbg('un-steer refused: the runtime no longer holds this prompt as recallable', id);
|
|
3672
|
+
return false;
|
|
3673
|
+
}
|
|
3674
|
+
const request = method.kind === 'queued'
|
|
3675
|
+
? target.rpc.queue.removeAt({ id: method.itemId })
|
|
3676
|
+
: target.rpc.queue.removeMostRecent();
|
|
3677
|
+
const removed = (await request)?.removed === true;
|
|
3678
|
+
if (removed && (watch.taken || entry.consumed || entry.settled)) {
|
|
3679
|
+
// The runtime took this prompt before it processed the remove, so
|
|
3680
|
+
// what was removed is some other waiting message — another client's,
|
|
3681
|
+
// never a runtime notification (see `recallWatches`). It cannot be put
|
|
3682
|
+
// back; this row settles through its normal path.
|
|
3683
|
+
dbg('UN-STEER MISMATCH: the runtime took the prompt before the remove; the remove took another message', id, method.kind);
|
|
3684
|
+
return false;
|
|
3685
|
+
}
|
|
3686
|
+
return removed;
|
|
3687
|
+
} finally {
|
|
3688
|
+
if (recallWatches.get(entry.runtimeId) === watch) recallWatches.delete(entry.runtimeId);
|
|
3689
|
+
}
|
|
3690
|
+
}
|
|
3691
|
+
|
|
3692
|
+
function withTimeout(promise, ms, fallback) {
|
|
3693
|
+
let timer = null;
|
|
3694
|
+
return Promise.race([
|
|
3695
|
+
promise,
|
|
3696
|
+
new Promise((resolve) => { timer = setTimeout(() => resolve(fallback), ms); timer.unref?.(); }),
|
|
3697
|
+
]).finally(() => clearTimeout(timer));
|
|
3698
|
+
}
|
|
3699
|
+
|
|
3700
|
+
/**
|
|
3701
|
+
* Un-steer (decision 8, narrowed 2026-09-25 after the live check): pull a
|
|
3702
|
+
* pushed-but-unconsumed row back out of the runtime and cancel it. Immediate
|
|
3703
|
+
* sends wait in the runtime's id-less steering lane, so only the NEWEST
|
|
3704
|
+
* waiting one is recallable (`queue.removeMostRecent`); a row in the queued
|
|
3705
|
+
* lane goes by its stable id. A prompt the runtime already took cannot be
|
|
3706
|
+
* recalled; the control is then a no-op and the row settles normally.
|
|
3707
|
+
*/
|
|
3708
|
+
async function cancelPushedMessage(messageId) {
|
|
3709
|
+
const id = String(messageId || '').trim();
|
|
3710
|
+
if (!id) return false;
|
|
3711
|
+
let entry = null;
|
|
3712
|
+
for (const candidate of ownedEntries()) {
|
|
3713
|
+
if (String(candidate.message?.id || '') === id) { entry = candidate; break; }
|
|
3714
|
+
}
|
|
3715
|
+
if (!entry || entry.consumed || entry.settled || !entry.runtimeId) {
|
|
3716
|
+
dbg('un-steer refused: row not cancellable', id, entry ? `consumed=${entry.consumed} settled=${entry.settled}` : 'unknown row');
|
|
3717
|
+
return false;
|
|
3718
|
+
}
|
|
3719
|
+
if (!session || !sessionRpc.queue) return false;
|
|
3720
|
+
// One un-steer at a time; each re-reads the queue (that read is bounded —
|
|
3721
|
+
// see recallPushedPrompt). Steer sends never wait on this chain.
|
|
3722
|
+
const run = unsteerChain.then(() => recallPushedPrompt(entry, id));
|
|
3723
|
+
unsteerChain = run.catch(() => false);
|
|
3724
|
+
const removed = await run.catch((error) => {
|
|
3725
|
+
dbg('un-steer failed', id, error?.message || String(error));
|
|
3726
|
+
return false;
|
|
3727
|
+
});
|
|
3728
|
+
if (!removed) return false;
|
|
3729
|
+
// Settled meanwhile by its turn's end, or taken since the answer (the
|
|
3730
|
+
// remove then took another message): it keeps its normal outcome.
|
|
3731
|
+
if (entry.consumed || entry.settled) {
|
|
3732
|
+
dbg('un-steer raced the runtime taking the prompt or the row settling; leaving it to settle normally', id);
|
|
3733
|
+
return false;
|
|
3734
|
+
}
|
|
3735
|
+
entry.cancelled = true;
|
|
3736
|
+
entry.settled = true;
|
|
3737
|
+
while (recalledRuntimeIds.size >= MAX_RECALLED_RUNTIME_IDS) recalledRuntimeIds.delete(recalledRuntimeIds.keys().next().value);
|
|
3738
|
+
recalledRuntimeIds.set(entry.runtimeId, id);
|
|
3739
|
+
dbg('un-steered a pushed prompt', id);
|
|
3740
|
+
// The prompt is gone from the runtime; the row must end cancelled and
|
|
3741
|
+
// never be re-delivered. It stays owned (settling, with the terminal
|
|
3742
|
+
// wording the crash guard would send) until the relay acknowledges the
|
|
3743
|
+
// cancel; if the relay never does, the heartbeat hands it over to fail.
|
|
3744
|
+
claimSettling(entry.message, 'cancelled');
|
|
3745
|
+
const cancelBody = {
|
|
3746
|
+
conversationId: entry.message.conversationId,
|
|
3747
|
+
messageId: id,
|
|
3748
|
+
...attemptFields(entry.message),
|
|
3749
|
+
};
|
|
3750
|
+
let acknowledged = false;
|
|
3751
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
3752
|
+
acknowledged = await api('POST', '/api/queue-cancelled', cancelBody).then(() => true, (error) => {
|
|
3753
|
+
dbg('queue-cancelled publish failed', id, error?.message || String(error));
|
|
3754
|
+
// The row already left `processing` (answered, recovered, or already
|
|
3755
|
+
// cancelled): nothing left for this worker to do.
|
|
3756
|
+
return Number(error?.status) === 404 || Number(error?.status) === 409;
|
|
3757
|
+
});
|
|
3758
|
+
if (acknowledged || attempt >= settleRetryDelaysMs.length) break;
|
|
3759
|
+
await sleep(settleRetryDelaysMs[attempt]);
|
|
3760
|
+
}
|
|
3761
|
+
if (!acknowledged) {
|
|
3762
|
+
dbg('queue-cancelled could not be posted; handing the row to the heartbeat', id);
|
|
3763
|
+
settleFailedTerminals.set(id, buildSteerSettleFailure(entry.message, { variant: 'cancelled', agentLabel: SETTLE_AGENT_LABEL }));
|
|
3764
|
+
}
|
|
3765
|
+
releaseSettling(id);
|
|
3766
|
+
entry.released = true;
|
|
3767
|
+
entry.resolveDone(true);
|
|
3768
|
+
const turn = entry.turn;
|
|
3769
|
+
if (turn) {
|
|
3770
|
+
turn.entries = turn.entries.filter((other) => other !== entry);
|
|
3771
|
+
turn.pushed.delete(entry.runtimeId);
|
|
3772
|
+
}
|
|
3773
|
+
scheduleQueueLaneRefresh();
|
|
2012
3774
|
return true;
|
|
2013
3775
|
}
|
|
2014
3776
|
|
|
2015
3777
|
async function handlePendingPayload(pending) {
|
|
2016
3778
|
const message = pending?.message || null;
|
|
2017
3779
|
if (!message) return false;
|
|
3780
|
+
// Held (a card is open, a compaction is running, the turn-opening prompt
|
|
3781
|
+
// is not sent yet): hand the row back with no retry penalty; the relay
|
|
3782
|
+
// re-delivers it when the hold ends. An older relay (the worker updated on
|
|
3783
|
+
// disk before the relay restarted) treats the hand-back as a failed
|
|
3784
|
+
// delivery — retry, backoff, worker marked errored — so against one the
|
|
3785
|
+
// message is pushed the legacy way.
|
|
3786
|
+
if (isDeliveryHeld() && canHandBackHeldDelivery() !== false) return handBackHeldDelivery(message);
|
|
3787
|
+
// Legacy path only: never send a second prompt before the turn-opening one
|
|
3788
|
+
// (it would invert the runs), and never start a second turn beside one
|
|
3789
|
+
// that is still connecting.
|
|
3790
|
+
if (activeTurn && !activeTurn.settled && !activeTurn.primarySent) {
|
|
3791
|
+
await activeTurn.primaryReady;
|
|
3792
|
+
}
|
|
3793
|
+
// Nor steer into a continuation before its row exists (canAcceptSteering);
|
|
3794
|
+
// bounded like the registration itself, which can hang on a dead relay.
|
|
3795
|
+
if (activeTurn && !activeTurn.settled && awaitingRow(activeTurn)) {
|
|
3796
|
+
await withTimeout(activeTurn.rowReady, continuationRegistrationTimeoutMs, false);
|
|
3797
|
+
}
|
|
2018
3798
|
// A delivery that lands while a turn is running is steered into it rather
|
|
2019
3799
|
// than starting a second one: the runtime has a single conversation and a
|
|
2020
3800
|
// concurrent `send` would interleave into the same interaction anyway —
|
|
2021
3801
|
// this way the row is owned and settled instead of orphaned.
|
|
2022
|
-
//
|
|
2023
|
-
// Deliberately OUTSIDE the try/finally below: that `finally` clears
|
|
2024
|
-
// `activeTurn`, and a steered call returns while the turn it was steered
|
|
2025
|
-
// into is still publishing. Clearing there would tell the heartbeat this
|
|
2026
|
-
// worker owns nothing and the relay would recover the live row.
|
|
2027
3802
|
if (activeTurn && !activeTurn.settled && session) {
|
|
2028
3803
|
const turn = activeTurn;
|
|
2029
3804
|
try {
|
|
@@ -2040,75 +3815,30 @@ export function createCopilotSdkSessionRunner({
|
|
|
2040
3815
|
return true;
|
|
2041
3816
|
}
|
|
2042
3817
|
}
|
|
2043
|
-
// Created here rather than inside `runTurn` so the catch and finally below
|
|
2044
|
-
// act on THIS turn by identity: under split ownership, `activeTurn` may
|
|
2045
|
-
// already belong to a newer turn (a continuation the runtime opened while
|
|
2046
|
-
// this one was publishing) by the time they run.
|
|
2047
3818
|
const turn = createTurn(message);
|
|
2048
|
-
|
|
2049
|
-
return await runTurn(turn);
|
|
2050
|
-
} catch (error) {
|
|
2051
|
-
const classified = classifyCopilotTurnException(error);
|
|
2052
|
-
dbg('copilot turn failed', message.id, classified.detail);
|
|
2053
|
-
// A failure that killed the session (or came from starting it) leaves a
|
|
2054
|
-
// handle nothing else can use; drop it so the next delivery rebuilds and
|
|
2055
|
-
// resumes rather than sending into a dead runtime. An UNCONFIRMED MODEL
|
|
2056
|
-
// SWITCH is the exception: the session is healthy and merely still on
|
|
2057
|
-
// its previous model, and tearing the runtime down over it would kill
|
|
2058
|
-
// any live detached shells for a selection problem the user fixes by
|
|
2059
|
-
// picking another model.
|
|
2060
|
-
if (!isModelSwitchUnconfirmedError(error)) {
|
|
2061
|
-
await stopRuntime('turn-failure').catch(() => {});
|
|
2062
|
-
}
|
|
2063
|
-
await publishResponse(message, {
|
|
2064
|
-
text: classified.text,
|
|
2065
|
-
model: null,
|
|
2066
|
-
terminalError: terminalErrorRecord(message, classified),
|
|
2067
|
-
});
|
|
2068
|
-
// Rows steered into the turn that just threw are owed a response too —
|
|
2069
|
-
// and, more urgently, `steerIntoActiveTurn` is still awaiting their
|
|
2070
|
-
// settle. Skipping this would wedge that caller (and its queue row)
|
|
2071
|
-
// forever behind a heartbeat that keeps renewing the lease.
|
|
2072
|
-
await settleSteeredRows(turn, null, null, classified).catch(() => {});
|
|
2073
|
-
await closeStraySubagentRuns(turn).catch(() => {});
|
|
2074
|
-
return true;
|
|
2075
|
-
} finally {
|
|
2076
|
-
// Released only once every publish for this row has landed. A heartbeat
|
|
2077
|
-
// that fired during the publish window with the row unowned would tell
|
|
2078
|
-
// the relay this worker owns nothing, and the still-processing row would
|
|
2079
|
-
// be recovered (`owner-heartbeat-idle`) and re-delivered — a duplicate
|
|
2080
|
-
// execution racing the response that was already on its way.
|
|
2081
|
-
releaseTurnOwnership(turn);
|
|
2082
|
-
touch();
|
|
2083
|
-
// Every path through the turn — published, failed, aborted, threw — has
|
|
2084
|
-
// finished by here, which is the only safe place for the ingest: it is
|
|
2085
|
-
// advisory, it is not awaited, and it must never be able to delay a reply
|
|
2086
|
-
// that is already written.
|
|
2087
|
-
postTurnUsage();
|
|
2088
|
-
}
|
|
3819
|
+
return runTurn(turn);
|
|
2089
3820
|
}
|
|
2090
3821
|
|
|
2091
3822
|
// --------------------------------------------------------------- teardown --
|
|
2092
3823
|
|
|
2093
3824
|
function getActiveQueueMessageId() {
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
const id = String(turn.message?.id || '');
|
|
2100
|
-
if (id) return id;
|
|
3825
|
+
const live = activeTurn?.currentOwner;
|
|
3826
|
+
if (live && !live.released && live.message?.id) return String(live.message.id);
|
|
3827
|
+
for (const entry of ownedEntries()) {
|
|
3828
|
+
const id = String(entry.message?.id || '');
|
|
3829
|
+
if (id && !entry.released) return id;
|
|
2101
3830
|
}
|
|
2102
3831
|
return '';
|
|
2103
3832
|
}
|
|
2104
3833
|
|
|
2105
3834
|
/**
|
|
2106
|
-
* Every row this worker owns — the running turn's
|
|
2107
|
-
*
|
|
2108
|
-
*
|
|
2109
|
-
*
|
|
2110
|
-
*
|
|
2111
|
-
*
|
|
3835
|
+
* Every row this worker owns — the running turn's rows and the same for
|
|
3836
|
+
* every settled turn still publishing — as `{ id, attemptId }` entries; a
|
|
3837
|
+
* consumed row whose settle marker is still being saved carries the
|
|
3838
|
+
* terminal failure the crash guard must send instead of a requeue, because
|
|
3839
|
+
* its prompt must never run twice. A row missing from this list would be
|
|
3840
|
+
* recovered mid-flight and re-delivered while the runtime was still
|
|
3841
|
+
* answering it.
|
|
2112
3842
|
*
|
|
2113
3843
|
* Both the heartbeat (lease renewal) and the crash guard (requeue-on-exit)
|
|
2114
3844
|
* read this: the crash guard takes the entries whole so its requeues stay
|
|
@@ -2117,24 +3847,36 @@ export function createCopilotSdkSessionRunner({
|
|
|
2117
3847
|
*/
|
|
2118
3848
|
function getActiveQueueMessageIds() {
|
|
2119
3849
|
const entries = [];
|
|
2120
|
-
const
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
3850
|
+
const byId = new Map();
|
|
3851
|
+
const push = (message, terminalError = null) => {
|
|
3852
|
+
const id = String(message?.id || '').trim();
|
|
3853
|
+
if (!id) return;
|
|
3854
|
+
const existing = byId.get(id);
|
|
3855
|
+
if (existing) {
|
|
3856
|
+
if (terminalError && !existing.terminalError) existing.terminalError = terminalError;
|
|
3857
|
+
return;
|
|
3858
|
+
}
|
|
3859
|
+
const entry = { id, attemptId: message?.attemptId || null, ...(terminalError ? { terminalError } : {}) };
|
|
3860
|
+
byId.set(id, entry);
|
|
3861
|
+
entries.push(entry);
|
|
2129
3862
|
};
|
|
2130
|
-
|
|
2131
|
-
|
|
3863
|
+
for (const entry of ownedEntries()) {
|
|
3864
|
+
if (entry.released) continue;
|
|
3865
|
+
push(entry.message);
|
|
3866
|
+
}
|
|
3867
|
+
for (const { message, variant } of settlingMessages.values()) {
|
|
3868
|
+
push(message, buildSteerSettleFailure(message, { variant, agentLabel: SETTLE_AGENT_LABEL }));
|
|
3869
|
+
}
|
|
2132
3870
|
return entries;
|
|
2133
3871
|
}
|
|
2134
3872
|
|
|
2135
3873
|
async function dispose() {
|
|
2136
3874
|
disposed = true;
|
|
2137
3875
|
stopLifecycleTimer();
|
|
3876
|
+
if (queueLaneTimer) {
|
|
3877
|
+
clearTimeout(queueLaneTimer);
|
|
3878
|
+
queueLaneTimer = null;
|
|
3879
|
+
}
|
|
2138
3880
|
// A question card left `pending` would sit in the UI inviting an answer
|
|
2139
3881
|
// that nothing is left to read. Time them out before the socket goes.
|
|
2140
3882
|
await questionBridge.cancelPendingQuestions?.().catch?.(() => {});
|
|
@@ -2145,6 +3887,18 @@ export function createCopilotSdkSessionRunner({
|
|
|
2145
3887
|
handlePendingPayload,
|
|
2146
3888
|
getActiveQueueMessageId,
|
|
2147
3889
|
getActiveQueueMessageIds,
|
|
3890
|
+
// The delivery gate (the socket link's probes) and the composer snapshot
|
|
3891
|
+
// (the heartbeat's).
|
|
3892
|
+
canAcceptSteering,
|
|
3893
|
+
isDeliveryHeld,
|
|
3894
|
+
steeringState,
|
|
3895
|
+
getSettleFailed,
|
|
3896
|
+
acknowledgeSettleFailed,
|
|
3897
|
+
cancelPushedMessage,
|
|
3898
|
+
// The task panel's per-card Stop (`rpc.tasks.cancel`); also what a
|
|
3899
|
+
// subagent lane's Stop tries — the runtime tracks sync agents in the same
|
|
3900
|
+
// registry and refuses ids it does not know.
|
|
3901
|
+
stopBackgroundTask,
|
|
2148
3902
|
// "Active" spans both ownerships: a settled turn still publishing must
|
|
2149
3903
|
// keep the worker's idle/shutdown gates closed just like a running one.
|
|
2150
3904
|
isTurnActive: () => !!activeTurn || publishingTurns.size > 0,
|
|
@@ -2157,6 +3911,14 @@ export function createCopilotSdkSessionRunner({
|
|
|
2157
3911
|
whenUsagePosted: () => usagePostChain,
|
|
2158
3912
|
// The in-flight model-catalog snapshot POST — the same kind of seam.
|
|
2159
3913
|
whenModelSnapshotPosted: () => modelSnapshotChain,
|
|
3914
|
+
// The in-flight queued-lane read — same kind of seam.
|
|
3915
|
+
whenQueueLaneRefreshed: () => queueLaneRefreshChain,
|
|
3916
|
+
// The in-flight `rpc.tasks.list()` re-read — same kind of seam. A refresh
|
|
3917
|
+
// that is still debounced counts too, so a caller sees the registry read
|
|
3918
|
+
// that the last event scheduled.
|
|
3919
|
+
whenTasksRefreshed: () => (taskRefreshTimer
|
|
3920
|
+
? new Promise((resolve) => { setTimeout(resolve, taskRefreshMs + 5); }).then(() => taskRefreshChain)
|
|
3921
|
+
: taskRefreshChain),
|
|
2160
3922
|
// Test seams / observability.
|
|
2161
3923
|
_getState: () => ({
|
|
2162
3924
|
hasClient: !!client,
|
|
@@ -2171,7 +3933,22 @@ export function createCopilotSdkSessionRunner({
|
|
|
2171
3933
|
// Settled turns whose relay publishes have not all landed yet.
|
|
2172
3934
|
publishingTurnCount: publishingTurns.size,
|
|
2173
3935
|
backgroundShells: backgroundShells.live(),
|
|
3936
|
+
backgroundTasks: sessionRpc.tasks ? registryCards() : shellTrackerCards(),
|
|
2174
3937
|
continuationDueSince,
|
|
3938
|
+
compacting: isCompacting(),
|
|
3939
|
+
pendingHumanRequests,
|
|
3940
|
+
sessionRpc: { ...sessionRpc },
|
|
3941
|
+
activeEntries: activeTurn
|
|
3942
|
+
? activeTurn.entries.map((entry) => ({
|
|
3943
|
+
id: entry.message?.id || null,
|
|
3944
|
+
role: entry.role,
|
|
3945
|
+
runtimeId: entry.runtimeId,
|
|
3946
|
+
consumed: entry.consumed,
|
|
3947
|
+
delivery: entry.delivery,
|
|
3948
|
+
settled: entry.settled,
|
|
3949
|
+
foldedInto: entry.foldedInto?.message?.id || null,
|
|
3950
|
+
}))
|
|
3951
|
+
: [],
|
|
2175
3952
|
}),
|
|
2176
3953
|
_evaluateLifecycle: evaluateLifecycle,
|
|
2177
3954
|
};
|