@oar-sh/oar 0.9.21 → 0.9.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +113 -0
- package/README.md +9 -9
- package/package.json +1 -1
- package/server/claude-cloud-worker/claude-cloud-session-worker.mjs +9 -1
- package/server/claude-worker/claude-sdk-adapter.mjs +149 -4
- package/server/claude-worker/claude-session-process.mjs +9 -4
- package/server/claude-worker/claude-session-worker-main.mjs +10 -1
- package/server/copilot-worker/copilot-sdk-session-process.mjs +1 -0
- package/server/copilot-worker/copilot-sdk-session-worker.mjs +10 -1
- package/server/cursor-worker/cursor-session-worker.mjs +9 -1
- package/server/db-schema.mjs +19 -0
- package/server/grok-worker/grok-background-tasks.mjs +1 -0
- package/server/grok-worker/grok-session-worker.mjs +10 -1
- package/server/grok-worker/grok-turn-runner.mjs +1 -0
- package/server/migrations/0037-relay-asks-queue-id.mjs +66 -0
- package/server/migrations/0038-turn-seq.mjs +92 -0
- package/server/public/app/activity-replay-state.mjs +160 -42
- package/server/public/app/api-client.js +7 -0
- package/server/public/app/ask-user-view.js +4 -3
- package/server/public/app/bootstrap.js +62 -4
- package/server/public/app/chat-actions-menu-state.mjs +2 -0
- package/server/public/app/context-usage-view.mjs +68 -0
- package/server/public/app/conversation-view.js +306 -49
- package/server/public/app/header-priority-actions.mjs +5 -1
- package/server/public/app/held-details.mjs +202 -0
- package/server/public/app/inbox-ui.js +70 -79
- package/server/public/app/inbox-view.mjs +35 -19
- package/server/public/app/journal-view.js +17 -0
- package/server/public/app/live-activity-cap.mjs +103 -0
- package/server/public/app/questions-panel-ui.js +634 -0
- package/server/public/app/questions-stub-view.mjs +40 -0
- package/server/public/app/questions-view.mjs +89 -0
- package/server/public/app/relay-board-view.js +12 -2
- package/server/public/app/relay-thoughts.mjs +15 -1
- package/server/public/app/session-history-ui.js +1 -0
- package/server/public/app/settings-modal.js +6 -0
- package/server/public/app/socket-handlers.js +42 -11
- package/server/public/app/stream-view-preference.mjs +60 -0
- package/server/public/app/subagent-fold.mjs +6 -3
- package/server/public/app/subagent-timeline-view.mjs +12 -5
- package/server/public/app/subagent-viewer.mjs +8 -6
- package/server/public/app/thoughts-preference.mjs +79 -0
- package/server/public/app/turn-stream-view.mjs +124 -0
- package/server/public/index.html +391 -214
- package/server/repositories/message-repository.mjs +10 -0
- package/server/repositories/question-repository.mjs +51 -7
- package/server/repositories/relay-turn-rows.mjs +56 -0
- package/server/routes/ask-user-routes.mjs +34 -5
- package/server/routes/messages-routes.mjs +35 -14
- package/server/routes/relay-ask-routes.mjs +15 -5
- package/server/routes/sessions-routes.mjs +76 -5
- package/server/server-runtime.mjs +178 -131
- package/server/services/cloudflared-tunnel-service.mjs +7 -0
- package/server/services/idle-timers.mjs +184 -0
- package/server/services/queue-wake-hook.mjs +145 -0
- package/server/services/queue-waker.mjs +127 -0
- package/server/services/relay-ask-service.mjs +64 -5
- package/server/services/relay-event-service.mjs +4 -1
- package/server/services/remote-relay-dispatcher.mjs +23 -4
- package/server/services/session-history-refresh-service.mjs +4 -0
- package/server/services/session-worker-supervisor-service.mjs +36 -0
- package/server/services/session-worker-websocket-service.mjs +143 -15
- package/server/services/ssh-tunnel-manager-service.mjs +7 -0
- package/shared/ask-user-bridge.mjs +9 -6
- package/shared/relay-asks.mjs +10 -0
- package/shared/relay-events.mjs +1 -0
- package/shared/worker-runtime/heartbeat.mjs +26 -0
- package/shared/worker-runtime/worker-keepalive.mjs +108 -0
- package/shared/worker-runtime/worker-websocket-link.mjs +21 -10
|
@@ -367,6 +367,9 @@ export function createSessionHistoryRefreshService({
|
|
|
367
367
|
timestamp,
|
|
368
368
|
activity.subagentRunId,
|
|
369
369
|
activity.metadata ? JSON.stringify(activity.metadata) : null,
|
|
370
|
+
// turn_seq: an imported turn has no order of its own; it keeps the
|
|
371
|
+
// grouped layout.
|
|
372
|
+
null,
|
|
370
373
|
);
|
|
371
374
|
}
|
|
372
375
|
const thoughts = ensureArray(message?.thoughts)
|
|
@@ -386,6 +389,7 @@ export function createSessionHistoryRefreshService({
|
|
|
386
389
|
thought.done ? 1 : 0,
|
|
387
390
|
timestamp,
|
|
388
391
|
thought.subagentRunId,
|
|
392
|
+
null, // turn_seq, as for the activity rows above
|
|
389
393
|
);
|
|
390
394
|
}
|
|
391
395
|
}
|
|
@@ -88,6 +88,12 @@ export function createSessionWorkerSupervisor({
|
|
|
88
88
|
// short enough that stopping a turn and immediately sending a new message
|
|
89
89
|
// does not add 30s in front of the worker's boot.
|
|
90
90
|
killBlockGraceMs = 3_000,
|
|
91
|
+
// A kill's block on new launches and deliveries for the session ended (its
|
|
92
|
+
// grace ran out, or the marker was cleared): (sessionId). The relay hands
|
|
93
|
+
// out what waited for it.
|
|
94
|
+
onKillBlockLifted = null,
|
|
95
|
+
setTimeoutImpl = setTimeout,
|
|
96
|
+
clearTimeoutImpl = clearTimeout,
|
|
91
97
|
isPidAlive = defaultIsPidAlive,
|
|
92
98
|
diagnosticPlanReference = null,
|
|
93
99
|
log = null,
|
|
@@ -98,6 +104,8 @@ export function createSessionWorkerSupervisor({
|
|
|
98
104
|
|
|
99
105
|
const pendingStarts = new Map();
|
|
100
106
|
const lifecycleBySession = new Map();
|
|
107
|
+
// sessionId → the timer that reports the end of its kill block.
|
|
108
|
+
const killBlockTimers = new Map();
|
|
101
109
|
const retryLimit = Math.max(0, clampInt(maxRestartRetries, 5));
|
|
102
110
|
const backoffBaseMs = Math.max(1, clampInt(restartBackoffBaseMs, 1_000));
|
|
103
111
|
const backoffMaxMs = Math.max(backoffBaseMs, clampInt(restartBackoffMaxMs, 30_000));
|
|
@@ -301,9 +309,35 @@ export function createSessionWorkerSupervisor({
|
|
|
301
309
|
});
|
|
302
310
|
}
|
|
303
311
|
|
|
312
|
+
function reportKillBlockLifted(sessionId) {
|
|
313
|
+
if (typeof onKillBlockLifted !== 'function') return;
|
|
314
|
+
try { onKillBlockLifted(sessionId); } catch {}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
function clearKillBlockTimer(sessionId) {
|
|
318
|
+
const timer = killBlockTimers.get(sessionId);
|
|
319
|
+
if (!timer) return false;
|
|
320
|
+
clearTimeoutImpl(timer);
|
|
321
|
+
killBlockTimers.delete(sessionId);
|
|
322
|
+
return true;
|
|
323
|
+
}
|
|
324
|
+
|
|
304
325
|
function markKilled(sessionId) {
|
|
305
326
|
if (!normalizeSessionId(sessionId)) return null;
|
|
306
327
|
const killed = setLifecycle(sessionId, { killedAtMs: nowMs() });
|
|
328
|
+
// The block itself is read off the clock (isKillBlocked); this timer only
|
|
329
|
+
// says when it ends, so the rows that waited go out then.
|
|
330
|
+
if (killGraceMs > 0 && typeof onKillBlockLifted === 'function') {
|
|
331
|
+
clearKillBlockTimer(sessionId);
|
|
332
|
+
const timer = setTimeoutImpl(() => {
|
|
333
|
+
killBlockTimers.delete(sessionId);
|
|
334
|
+
if (!isKillBlocked(sessionId)) reportKillBlockLifted(sessionId);
|
|
335
|
+
// A few ms past the grace: the timer's clock and nowMs() may differ
|
|
336
|
+
// by a tick.
|
|
337
|
+
}, killGraceMs + 5);
|
|
338
|
+
timer?.unref?.();
|
|
339
|
+
killBlockTimers.set(sessionId, timer);
|
|
340
|
+
}
|
|
307
341
|
return toLifecycleSnapshot(sessionId, killed);
|
|
308
342
|
}
|
|
309
343
|
|
|
@@ -489,10 +523,12 @@ export function createSessionWorkerSupervisor({
|
|
|
489
523
|
patch.degradedAtMs = null;
|
|
490
524
|
patch.uiState = 'white';
|
|
491
525
|
}
|
|
526
|
+
const liftsKillBlock = resetKilledMarker && clearKillBlockTimer(sessionId);
|
|
492
527
|
if (resetKilledMarker) {
|
|
493
528
|
patch.killedAtMs = null;
|
|
494
529
|
}
|
|
495
530
|
const cleared = setLifecycle(sessionId, patch);
|
|
531
|
+
if (liftsKillBlock) reportKillBlockLifted(sessionId);
|
|
496
532
|
return toLifecycleSnapshot(sessionId, cleared);
|
|
497
533
|
}
|
|
498
534
|
|
|
@@ -86,18 +86,21 @@ function queueSnapshotChanged(left, right) {
|
|
|
86
86
|
|| left.parkedCount !== right.parkedCount;
|
|
87
87
|
}
|
|
88
88
|
|
|
89
|
-
// The worker
|
|
90
|
-
// worker-
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
|
|
89
|
+
// The worker keepalive beats every 30 s while the worker is idle and every
|
|
90
|
+
// 10 s while it works (shared/worker-runtime/worker-keepalive.mjs; an older
|
|
91
|
+
// worker always every 10 s), and the link itself abandons a connection after
|
|
92
|
+
// 90 s of server silence (staleConnectionMs). Mirror that on this side: three
|
|
93
|
+
// missed idle beats and a retained socket no longer proves the worker is
|
|
94
|
+
// alive.
|
|
95
|
+
export const WORKER_SOCKET_LIVENESS_MAX_SILENCE_MS = 90_000;
|
|
94
96
|
|
|
95
97
|
// Protocol features this relay understands, advertised in server.hello. A
|
|
96
98
|
// worker that updated on disk before the relay restarted must not rely on
|
|
97
99
|
// them otherwise: an older relay ignores worker.unready / hello ready:false
|
|
98
100
|
// and treats a `steering-held` requeue as a failure (retry, backoff,
|
|
99
|
-
// worker marked errored).
|
|
100
|
-
|
|
101
|
+
// worker marked errored). `idle-keepalive`: an idle worker may beat every
|
|
102
|
+
// 30 s; an older relay's liveness and CLI windows expect a beat every 10 s.
|
|
103
|
+
export const WORKER_PROTOCOL_CAPABILITIES = Object.freeze(['worker-unready', 'steering-held', 'idle-keepalive']);
|
|
101
104
|
|
|
102
105
|
export function createSessionWorkerWebSocketService({
|
|
103
106
|
WebSocketServerImpl,
|
|
@@ -108,6 +111,19 @@ export function createSessionWorkerWebSocketService({
|
|
|
108
111
|
noteWorkerHeartbeat = () => {},
|
|
109
112
|
onDeliverySendFailed = async () => {},
|
|
110
113
|
onWorkerSocketClosed = async () => {},
|
|
114
|
+
// The number of open worker sockets changed: (count). The CLI presence
|
|
115
|
+
// window is wider while a worker socket is open.
|
|
116
|
+
onSocketCountChange = () => {},
|
|
117
|
+
// Every queue snapshot the poll takes: ({pendingCount, processingCount,
|
|
118
|
+
// parkedCount}). The relay arms its worker prime while rows are pending.
|
|
119
|
+
onQueueSnapshot = () => {},
|
|
120
|
+
// The earliest future retry time of a pending row, in ms (null: none). A
|
|
121
|
+
// one-shot timer fires then, so a backoff or a usage-limit hold ends on
|
|
122
|
+
// time rather than at the next poll.
|
|
123
|
+
nextDueAtMs = () => null,
|
|
124
|
+
// That timer fired with a row due: the relay wakes its queue (prime, hand
|
|
125
|
+
// out, arm), since a row whose hold ended may have no worker yet.
|
|
126
|
+
onDue = () => {},
|
|
111
127
|
requestWork = async () => null,
|
|
112
128
|
// A worker stated its kind ("claude", "claude-api", ...) in a hello or a
|
|
113
129
|
// ready: (sessionId, kind, pid). The registry keeps it.
|
|
@@ -147,6 +163,10 @@ export function createSessionWorkerWebSocketService({
|
|
|
147
163
|
const relayBuildId = normalizeBuildId(buildId);
|
|
148
164
|
|
|
149
165
|
let interval = null;
|
|
166
|
+
let dueTimer = null;
|
|
167
|
+
// The retry time the armed due timer stands for.
|
|
168
|
+
let dueTimerForMs = null;
|
|
169
|
+
let stopped = false;
|
|
150
170
|
let lastSnapshot = null;
|
|
151
171
|
let attached = false;
|
|
152
172
|
|
|
@@ -155,10 +175,15 @@ export function createSessionWorkerWebSocketService({
|
|
|
155
175
|
return Number.isFinite(value) ? value : Date.now();
|
|
156
176
|
}
|
|
157
177
|
|
|
178
|
+
function noteSocketCount() {
|
|
179
|
+
try { onSocketCountChange(clients.size); } catch {}
|
|
180
|
+
}
|
|
181
|
+
|
|
158
182
|
function removeSocket(socket, reason) {
|
|
159
183
|
const meta = clientState.get(socket);
|
|
160
|
-
clients.delete(socket);
|
|
184
|
+
const removed = clients.delete(socket);
|
|
161
185
|
clientState.delete(socket);
|
|
186
|
+
if (removed) noteSocketCount();
|
|
162
187
|
// Death-detection hook: a worker socket closing while its session owns
|
|
163
188
|
// an in-flight processing row used to be silently forgotten, leaving
|
|
164
189
|
// the row to the 600s stale sweep. The handler PID-probes and recovers.
|
|
@@ -222,11 +247,12 @@ export function createSessionWorkerWebSocketService({
|
|
|
222
247
|
};
|
|
223
248
|
}
|
|
224
249
|
|
|
225
|
-
function emitQueueChanged(reason = 'queue-update') {
|
|
250
|
+
function emitQueueChanged(reason = 'queue-update', { skip = null } = {}) {
|
|
226
251
|
const event = toQueueChangedEvent(reason);
|
|
227
252
|
const payload = JSON.stringify(event);
|
|
228
253
|
for (const socket of clients) {
|
|
229
254
|
if (socket.readyState !== socket.OPEN) continue;
|
|
255
|
+
if (skip?.has(socket)) continue;
|
|
230
256
|
try {
|
|
231
257
|
socket.send(payload);
|
|
232
258
|
} catch {
|
|
@@ -364,25 +390,115 @@ export function createSessionWorkerWebSocketService({
|
|
|
364
390
|
}
|
|
365
391
|
}
|
|
366
392
|
|
|
393
|
+
/** Hands a row to every ready socket; resolves to the sockets that got one. */
|
|
367
394
|
async function maybeDeliverToReadyClients(reason = 'queue-update') {
|
|
395
|
+
const sockets = [];
|
|
368
396
|
const deliveries = [];
|
|
369
397
|
for (const socket of clients) {
|
|
370
398
|
const meta = clientState.get(socket);
|
|
371
399
|
if (!meta?.ready || meta.delivering) continue;
|
|
400
|
+
sockets.push(socket);
|
|
372
401
|
deliveries.push(maybeDeliverToSocket(socket, reason));
|
|
373
402
|
}
|
|
374
|
-
|
|
375
|
-
|
|
403
|
+
const handed = new Set();
|
|
404
|
+
if (!deliveries.length) return handed;
|
|
405
|
+
const results = await Promise.allSettled(deliveries);
|
|
406
|
+
results.forEach((result, index) => {
|
|
407
|
+
if (result.status === 'fulfilled' && result.value === true) handed.add(sockets[index]);
|
|
408
|
+
});
|
|
409
|
+
return handed;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
// Ready sockets get their rows first; then every other socket hears that
|
|
413
|
+
// the queue changed (a worker the relay does not count as ready, a steering
|
|
414
|
+
// one mid-turn, answers with a ready of its own). A socket just handed a
|
|
415
|
+
// row is busy with it and is not told.
|
|
416
|
+
async function deliverThenAnnounce(reason, { announce = true } = {}) {
|
|
417
|
+
const handed = await maybeDeliverToReadyClients(reason);
|
|
418
|
+
if (announce) emitQueueChanged(reason, { skip: handed });
|
|
419
|
+
return handed;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* The queue may have work: a write, a gate that lifted, a retry come due.
|
|
424
|
+
* Hands rows out now instead of at the next poll.
|
|
425
|
+
*/
|
|
426
|
+
async function notifyQueueChanged(reason = 'queue-update') {
|
|
427
|
+
if (stopped) return new Set();
|
|
428
|
+
lastSnapshot = normalizeQueueSnapshot(queueCounts());
|
|
429
|
+
// A turn that ended with nothing behind it: no row to hand out, so no
|
|
430
|
+
// dequeue attempt per ready socket and no announcement.
|
|
431
|
+
const handed = lastSnapshot.pendingCount > 0 ? await deliverThenAnnounce(reason) : new Set();
|
|
432
|
+
armDueTimer();
|
|
433
|
+
return handed;
|
|
376
434
|
}
|
|
377
435
|
|
|
436
|
+
// The backstop. With nothing pending nothing can be handed out: no dequeue
|
|
437
|
+
// attempt, only the count change announced. With rows pending (a retry in
|
|
438
|
+
// backoff, a gate, a worker not yet up) every tick tries the ready sockets,
|
|
439
|
+
// as a net under the event-driven wakes.
|
|
378
440
|
function monitorQueue() {
|
|
379
441
|
const snapshot = normalizeQueueSnapshot(queueCounts());
|
|
442
|
+
try { onQueueSnapshot(snapshot); } catch {}
|
|
380
443
|
const changed = queueSnapshotChanged(snapshot, lastSnapshot);
|
|
381
444
|
lastSnapshot = snapshot;
|
|
382
|
-
if (
|
|
445
|
+
if (snapshot.pendingCount > 0) {
|
|
446
|
+
void deliverThenAnnounce(changed ? 'queue-count-changed' : 'ready-check', { announce: changed });
|
|
447
|
+
} else if (changed) {
|
|
383
448
|
emitQueueChanged('queue-count-changed');
|
|
384
449
|
}
|
|
385
|
-
|
|
450
|
+
armDueTimer();
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
const DUE_TIMER_MIN_MS = 50;
|
|
454
|
+
const DUE_TIMER_MAX_MS = 60_000;
|
|
455
|
+
|
|
456
|
+
function clearDueTimer() {
|
|
457
|
+
dueTimerForMs = null;
|
|
458
|
+
if (!dueTimer) return;
|
|
459
|
+
clearTimeout(dueTimer);
|
|
460
|
+
dueTimer = null;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* One timer for the earliest future retry time. An armed timer for the same
|
|
465
|
+
* time is kept; any other arm clears the one before. A time further than a
|
|
466
|
+
* minute away is re-read after a minute.
|
|
467
|
+
*/
|
|
468
|
+
function armDueTimer() {
|
|
469
|
+
if (stopped) {
|
|
470
|
+
clearDueTimer();
|
|
471
|
+
return;
|
|
472
|
+
}
|
|
473
|
+
// A timer whose time has come fires and re-arms itself: a poll in the
|
|
474
|
+
// same instant (its row no longer "future") must not clear it unfired.
|
|
475
|
+
if (dueTimer && dueTimerForMs !== null && dueTimerForMs <= nowMsSafe()) return;
|
|
476
|
+
let dueAt = null;
|
|
477
|
+
try { dueAt = nextDueAtMs(); } catch { dueAt = null; }
|
|
478
|
+
const dueMs = Number(dueAt);
|
|
479
|
+
if (dueAt === null || dueAt === undefined || !Number.isFinite(dueMs)) {
|
|
480
|
+
clearDueTimer();
|
|
481
|
+
return;
|
|
482
|
+
}
|
|
483
|
+
if (dueTimer && dueTimerForMs === dueMs) return;
|
|
484
|
+
clearDueTimer();
|
|
485
|
+
const untilDueMs = dueMs - nowMsSafe();
|
|
486
|
+
const delayMs = Math.min(DUE_TIMER_MAX_MS, Math.max(DUE_TIMER_MIN_MS, untilDueMs));
|
|
487
|
+
// A time further than the cap: this firing only re-reads.
|
|
488
|
+
const reachesDue = untilDueMs <= DUE_TIMER_MAX_MS;
|
|
489
|
+
dueTimerForMs = dueMs;
|
|
490
|
+
dueTimer = setTimeout(() => {
|
|
491
|
+
dueTimer = null;
|
|
492
|
+
dueTimerForMs = null;
|
|
493
|
+
if (stopped) return;
|
|
494
|
+
if (reachesDue) {
|
|
495
|
+
// Whatever came due may go now; a row that was claimed or cancelled
|
|
496
|
+
// meanwhile makes this a wake with nothing to do.
|
|
497
|
+
try { onDue('retry-due'); } catch {}
|
|
498
|
+
}
|
|
499
|
+
armDueTimer();
|
|
500
|
+
}, delayMs);
|
|
501
|
+
if (typeof dueTimer.unref === 'function') dueTimer.unref();
|
|
386
502
|
}
|
|
387
503
|
|
|
388
504
|
function rejectUpgrade(socket) {
|
|
@@ -432,6 +548,7 @@ export function createSessionWorkerWebSocketService({
|
|
|
432
548
|
deliveryFailures: 0,
|
|
433
549
|
});
|
|
434
550
|
if (identity.sessionId) supersedePriorSockets(ws, identity.sessionId);
|
|
551
|
+
noteSocketCount();
|
|
435
552
|
touchCli();
|
|
436
553
|
emitEvent(ws, serverHello('connected', identity.sessionId));
|
|
437
554
|
|
|
@@ -466,8 +583,13 @@ export function createSessionWorkerWebSocketService({
|
|
|
466
583
|
noteWorkerBuildId(meta, payload);
|
|
467
584
|
meta.ready = true;
|
|
468
585
|
meta.lastReadyAt = nowIso();
|
|
469
|
-
|
|
470
|
-
|
|
586
|
+
const readyReason = String(payload.reason || 'worker-ready');
|
|
587
|
+
noteHeartbeat(meta, readyReason);
|
|
588
|
+
// A keepalive's ready with nothing pending has nothing to draw: one
|
|
589
|
+
// count instead of a dequeue attempt. Any row that appears wakes the
|
|
590
|
+
// queue, and this socket stays ready for it.
|
|
591
|
+
if (readyReason === 'readiness-refresh' && normalizeQueueSnapshot(queueCounts()).pendingCount === 0) return;
|
|
592
|
+
void maybeDeliverToSocket(ws, readyReason);
|
|
471
593
|
return;
|
|
472
594
|
}
|
|
473
595
|
if (payload.type === 'worker.unready') {
|
|
@@ -526,6 +648,7 @@ export function createSessionWorkerWebSocketService({
|
|
|
526
648
|
}
|
|
527
649
|
|
|
528
650
|
function start() {
|
|
651
|
+
stopped = false;
|
|
529
652
|
if (!attached) {
|
|
530
653
|
httpServer.on('upgrade', onUpgrade);
|
|
531
654
|
wss.on('connection', onConnection);
|
|
@@ -537,10 +660,13 @@ export function createSessionWorkerWebSocketService({
|
|
|
537
660
|
}
|
|
538
661
|
lastSnapshot = normalizeQueueSnapshot(queueCounts());
|
|
539
662
|
emitQueueChanged('started');
|
|
663
|
+
armDueTimer();
|
|
540
664
|
logDebug(`[worker-ws] listening on ${wsPath}`);
|
|
541
665
|
}
|
|
542
666
|
|
|
543
667
|
function stop() {
|
|
668
|
+
stopped = true;
|
|
669
|
+
clearDueTimer();
|
|
544
670
|
if (interval) {
|
|
545
671
|
clearInterval(interval);
|
|
546
672
|
interval = null;
|
|
@@ -554,6 +680,7 @@ export function createSessionWorkerWebSocketService({
|
|
|
554
680
|
}
|
|
555
681
|
clients.clear();
|
|
556
682
|
clientState.clear();
|
|
683
|
+
noteSocketCount();
|
|
557
684
|
try { wss.close(); } catch (error) {
|
|
558
685
|
logWarn(`[worker-ws] close failed: ${error?.message || error}`);
|
|
559
686
|
}
|
|
@@ -610,6 +737,7 @@ export function createSessionWorkerWebSocketService({
|
|
|
610
737
|
stop,
|
|
611
738
|
status,
|
|
612
739
|
emitQueueChanged,
|
|
740
|
+
notifyQueueChanged,
|
|
613
741
|
emitDraining,
|
|
614
742
|
handleUpgrade,
|
|
615
743
|
sendControlToSession,
|
|
@@ -171,6 +171,9 @@ export function createSshTunnelManager({
|
|
|
171
171
|
io = null,
|
|
172
172
|
logger = console,
|
|
173
173
|
runtimeShutdownRef = () => false,
|
|
174
|
+
// The tunnel stopped blocking queue delivery (it connected): the relay
|
|
175
|
+
// hands out what waited for it.
|
|
176
|
+
onBlockingLifted = () => {},
|
|
174
177
|
platform = process.platform,
|
|
175
178
|
spawnImpl = spawn,
|
|
176
179
|
nowIso = () => new Date().toISOString(),
|
|
@@ -225,7 +228,11 @@ export function createSshTunnelManager({
|
|
|
225
228
|
};
|
|
226
229
|
|
|
227
230
|
const updateBlockingState = () => {
|
|
231
|
+
const wasBlocking = state.blocking;
|
|
228
232
|
state.blocking = state.required && state.mode === 'managed' && !state.connected;
|
|
233
|
+
if (wasBlocking && !state.blocking) {
|
|
234
|
+
try { onBlockingLifted(); } catch {}
|
|
235
|
+
}
|
|
229
236
|
};
|
|
230
237
|
|
|
231
238
|
const scheduleReconnect = (spawnTunnel, delayOverrideMs = null) => {
|
|
@@ -67,7 +67,10 @@ export function createAskUserBridge({
|
|
|
67
67
|
});
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
-
|
|
70
|
+
// `source` / `rationale` override the bridge-wide card identity for one ask
|
|
71
|
+
// (the Claude worker's safety-check approvals are not clarifications), and
|
|
72
|
+
// `allowFreeform: false` makes a card answered by its choices alone.
|
|
73
|
+
async function askSingleQuestion(entry, { signal, source, rationale, allowFreeform } = {}) {
|
|
71
74
|
const activeMsg = typeof getActiveMessage === 'function' ? getActiveMessage() : null;
|
|
72
75
|
const choices = entry.options.map((option) => option.label);
|
|
73
76
|
const promptParts = [entry.question];
|
|
@@ -82,7 +85,7 @@ export function createAskUserBridge({
|
|
|
82
85
|
mode: activeMsg?.relayMode || 'agent',
|
|
83
86
|
prompt: promptParts.join('\n\n'),
|
|
84
87
|
choices,
|
|
85
|
-
allowFreeform:
|
|
88
|
+
allowFreeform: allowFreeform !== false,
|
|
86
89
|
// Top-level, as the create route reads it (elicitation parity; absent
|
|
87
90
|
// for the flat question shape every existing caller sends).
|
|
88
91
|
...(entry.requestedSchema ? { requestedSchema: entry.requestedSchema } : {}),
|
|
@@ -92,8 +95,8 @@ export function createAskUserBridge({
|
|
|
92
95
|
attemptId: activeMsg?.attemptId || undefined,
|
|
93
96
|
timeout_ms: questionTimeoutMs,
|
|
94
97
|
context: {
|
|
95
|
-
source: questionSource,
|
|
96
|
-
rationale: questionRationale,
|
|
98
|
+
source: source || questionSource,
|
|
99
|
+
rationale: rationale || questionRationale,
|
|
97
100
|
queueMessageId: activeMsg?.id || null,
|
|
98
101
|
conversationId: activeMsg?.conversationId || null,
|
|
99
102
|
relayMode: activeMsg?.relayMode || 'agent',
|
|
@@ -106,7 +109,7 @@ export function createAskUserBridge({
|
|
|
106
109
|
return waitForRelayQuestionAnswer(questionId, { signal });
|
|
107
110
|
}
|
|
108
111
|
|
|
109
|
-
async function handleAskUserQuestion(input, { signal } = {}) {
|
|
112
|
+
async function handleAskUserQuestion(input, { signal, source, rationale, allowFreeform } = {}) {
|
|
110
113
|
const questions = normalizeQuestions(input);
|
|
111
114
|
if (!questions.length) {
|
|
112
115
|
return { answers: {}, structuredAnswers: {}, timedOut: false };
|
|
@@ -118,7 +121,7 @@ export function createAskUserBridge({
|
|
|
118
121
|
const structuredAnswers = {};
|
|
119
122
|
let timedOut = false;
|
|
120
123
|
for (const entry of questions) {
|
|
121
|
-
const result = await askSingleQuestion(entry, { signal });
|
|
124
|
+
const result = await askSingleQuestion(entry, { signal, source, rationale, allowFreeform });
|
|
122
125
|
answers[entry.question] = result.answer;
|
|
123
126
|
if (result.structuredAnswer) structuredAnswers[entry.question] = result.structuredAnswer;
|
|
124
127
|
if (entry.rawQuestion && entry.rawQuestion !== entry.question) {
|
package/shared/relay-asks.mjs
CHANGED
|
@@ -25,6 +25,11 @@ export const RELAY_QUESTION_KINDS = Object.freeze(['ask', 'ask_user', 'approval'
|
|
|
25
25
|
// Stored and sent between relays, so it kept the tool's name from before
|
|
26
26
|
// 0.9.18.
|
|
27
27
|
export const REMOTE_RELAY_APPROVAL_SOURCE = 'remote_relay';
|
|
28
|
+
// The `context.source` of a Claude session's Safety check card: an ask Claude
|
|
29
|
+
// Code marks as person-only (claude-sdk-adapter.mjs). Like the approval card,
|
|
30
|
+
// it is the user's to answer: never listed to an agent, never answered by
|
|
31
|
+
// one, from this relay or another.
|
|
32
|
+
export const PERSON_ONLY_QUESTION_SOURCE = 'ClaudeSafetyCheck';
|
|
28
33
|
export const RELAY_ASK_KIND = 'ask';
|
|
29
34
|
|
|
30
35
|
export const RELAY_ASK_URGENCIES = Object.freeze(['normal', 'now']);
|
|
@@ -145,6 +150,11 @@ export function normalizeRelayAskAnswer(ask, { answer = '', choice = '' } = {})
|
|
|
145
150
|
return { ok: true, value: { answer: text || picked, choice: picked } };
|
|
146
151
|
}
|
|
147
152
|
|
|
153
|
+
/** True for a Safety check card's context (a formatted question's or a stored request envelope's). */
|
|
154
|
+
export function isPersonOnlyQuestion(context) {
|
|
155
|
+
return !!context && typeof context === 'object' && toText(context.source) === PERSON_ONLY_QUESTION_SOURCE;
|
|
156
|
+
}
|
|
157
|
+
|
|
148
158
|
/** The kind of a formatted question card (a row of /api/relay-questions). */
|
|
149
159
|
export function relayQuestionKind(question) {
|
|
150
160
|
const kind = toText(question?.kind);
|
package/shared/relay-events.mjs
CHANGED
|
@@ -560,6 +560,7 @@ export function formatRelayEventLine(event) {
|
|
|
560
560
|
const parts = [`${head} ${quote(detail.prompt, RELAY_EVENT_LIMITS.questionChars)}`];
|
|
561
561
|
if (options.length) parts.push(`options: ${options.map((option) => `[${option}]`).join(' ')}`);
|
|
562
562
|
if (detail.approvalCard) parts.push('(an approval card: only the user can answer it)');
|
|
563
|
+
else if (detail.personOnly) parts.push('(a Safety check: the user\'s to answer)');
|
|
563
564
|
else if (oneLine(detail.questionId, 80)) parts.push(`(question_id ${oneLine(detail.questionId, 80)})`);
|
|
564
565
|
if (detail.kind === 'ask') parts.push('(asked with ask: it does not block that session)');
|
|
565
566
|
return parts.join(' ');
|
|
@@ -33,6 +33,10 @@ export function createHeartbeatController({
|
|
|
33
33
|
// Optional: called after every beat the relay answered, for work that
|
|
34
34
|
// waits for a reachable relay (posts kept after a failure).
|
|
35
35
|
onAnswered = () => {},
|
|
36
|
+
// Optional: how many posts are kept for a reachable relay (flushed by
|
|
37
|
+
// onAnswered). While any is, the worker counts as busy, so the next beat,
|
|
38
|
+
// which flushes them, is not an idle wait away.
|
|
39
|
+
getKeptPostCount = null,
|
|
36
40
|
// Optional: the kind of worker this is ("claude", "claude-api", ...). The
|
|
37
41
|
// relay hands a Claude chat's row only to a worker of the row's billing,
|
|
38
42
|
// and knows a worker's kind from what it says here and in its hello.
|
|
@@ -99,6 +103,27 @@ export function createHeartbeatController({
|
|
|
99
103
|
}
|
|
100
104
|
}
|
|
101
105
|
|
|
106
|
+
/**
|
|
107
|
+
* Whether the worker has work the relay follows by its beats: a row it
|
|
108
|
+
* owns, a live background task, a running turn, a settle failure to report,
|
|
109
|
+
* a kept post to flush.
|
|
110
|
+
* The keepalive beats at its busy cadence while this holds
|
|
111
|
+
* (worker-keepalive.mjs). A getter that throws counts as work.
|
|
112
|
+
*/
|
|
113
|
+
function hasWork() {
|
|
114
|
+
try {
|
|
115
|
+
if (typeof getActiveQueueMessageId === "function" && String(getActiveQueueMessageId() || "").trim()) return true;
|
|
116
|
+
if (typeof getActiveQueueMessageIds === "function" && (getActiveQueueMessageIds() || []).length) return true;
|
|
117
|
+
if (typeof getBackgroundTaskIds === "function" && (getBackgroundTaskIds() || []).length) return true;
|
|
118
|
+
if (typeof getSettleFailed === "function" && (getSettleFailed() || []).length) return true;
|
|
119
|
+
if (typeof getKeptPostCount === "function" && Number(getKeptPostCount()) > 0) return true;
|
|
120
|
+
if (typeof getSteeringState === "function" && getSteeringState()?.turnActive === true) return true;
|
|
121
|
+
return false;
|
|
122
|
+
} catch {
|
|
123
|
+
return true;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
102
127
|
/**
|
|
103
128
|
* An out-of-cycle pulse for state the relay should see before the next
|
|
104
129
|
* periodic one (the composer's steering snapshot, a relay that just came
|
|
@@ -135,6 +160,7 @@ export function createHeartbeatController({
|
|
|
135
160
|
}
|
|
136
161
|
|
|
137
162
|
return {
|
|
163
|
+
hasWork,
|
|
138
164
|
pulseHeartbeat,
|
|
139
165
|
requestPulse,
|
|
140
166
|
startHeartbeat,
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
// The worker's one keepalive timer. Each beat posts the heartbeat
|
|
2
|
+
// (heartbeat.mjs) and refreshes the websocket readiness
|
|
3
|
+
// (worker-websocket-link.mjs, built with readyRefreshMs: 0); before, each ran
|
|
4
|
+
// its own 10 s timer.
|
|
5
|
+
//
|
|
6
|
+
// Cadence: every 10 s while the worker has work (a row it owns, a delivery in
|
|
7
|
+
// flight, a hold, a background task, a running turn), and every 30 s while it
|
|
8
|
+
// is idle, but only on a relay that advertised `idle-keepalive` in its
|
|
9
|
+
// server.hello. An older relay expects a beat every 10 s (its liveness, CLI
|
|
10
|
+
// and stranded-session windows), and a worker whose socket is down knows no
|
|
11
|
+
// relay at all, so both get 10 s. Every server.hello and every delivery
|
|
12
|
+
// (nudge) keeps the busy cadence for the next 30 s and cuts an idle wait
|
|
13
|
+
// down to 10 s: the first beat naming a new turn's row is never an idle wait
|
|
14
|
+
// away, and a delivery lost on the wire or a relay that just came up is
|
|
15
|
+
// noticed as fast as before.
|
|
16
|
+
|
|
17
|
+
export const IDLE_KEEPALIVE_MS = 30_000;
|
|
18
|
+
export const BUSY_KEEPALIVE_MS = 10_000;
|
|
19
|
+
export const IDLE_KEEPALIVE_CAPABILITY = "idle-keepalive";
|
|
20
|
+
export const BUSY_HOLD_MS = 30_000;
|
|
21
|
+
|
|
22
|
+
export function createWorkerKeepalive({
|
|
23
|
+
heartbeat,
|
|
24
|
+
link,
|
|
25
|
+
idleMs = IDLE_KEEPALIVE_MS,
|
|
26
|
+
busyMs = BUSY_KEEPALIVE_MS,
|
|
27
|
+
busyHoldMs = BUSY_HOLD_MS,
|
|
28
|
+
setTimeoutImpl = setTimeout,
|
|
29
|
+
clearTimeoutImpl = clearTimeout,
|
|
30
|
+
now = () => Date.now(),
|
|
31
|
+
} = {}) {
|
|
32
|
+
let timer = null;
|
|
33
|
+
let dueAtMs = 0;
|
|
34
|
+
let running = false;
|
|
35
|
+
let busyUntilMs = 0;
|
|
36
|
+
|
|
37
|
+
function busy() {
|
|
38
|
+
if (now() < busyUntilMs) return true;
|
|
39
|
+
try {
|
|
40
|
+
if (heartbeat?.hasWork?.() === true) return true;
|
|
41
|
+
if (link?.isBusy?.() === true) return true;
|
|
42
|
+
return false;
|
|
43
|
+
} catch {
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function relayAllowsIdleCadence() {
|
|
49
|
+
try {
|
|
50
|
+
return link?.serverSupports?.(IDLE_KEEPALIVE_CAPABILITY) === true;
|
|
51
|
+
} catch {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** The wait before the next beat, decided after each beat. */
|
|
57
|
+
function nextDelayMs() {
|
|
58
|
+
if (busy() || !relayAllowsIdleCadence()) return busyMs;
|
|
59
|
+
return idleMs;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function schedule(delayMs) {
|
|
63
|
+
if (timer) clearTimeoutImpl(timer);
|
|
64
|
+
dueAtMs = now() + delayMs;
|
|
65
|
+
timer = setTimeoutImpl(beat, delayMs);
|
|
66
|
+
timer?.unref?.();
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function beat() {
|
|
70
|
+
timer = null;
|
|
71
|
+
if (!running) return;
|
|
72
|
+
void heartbeat?.pulseHeartbeat?.();
|
|
73
|
+
try { link?.refreshReadiness?.("readiness-refresh"); } catch {}
|
|
74
|
+
schedule(nextDelayMs());
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function start() {
|
|
78
|
+
if (running) return;
|
|
79
|
+
running = true;
|
|
80
|
+
// The first heartbeat goes out at once, as the heartbeat's own start did;
|
|
81
|
+
// the socket is not open yet, so its refresh waits for the next beat.
|
|
82
|
+
void heartbeat?.pulseHeartbeat?.();
|
|
83
|
+
schedule(nextDelayMs());
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The relay said hello or handed over work: the busy cadence holds for a
|
|
88
|
+
* while, and a beat further away than it comes sooner.
|
|
89
|
+
*/
|
|
90
|
+
function nudge() {
|
|
91
|
+
if (!running) return;
|
|
92
|
+
busyUntilMs = Math.max(busyUntilMs, now() + busyHoldMs);
|
|
93
|
+
if (timer && dueAtMs - now() > busyMs) schedule(busyMs);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function stop() {
|
|
97
|
+
running = false;
|
|
98
|
+
if (timer) clearTimeoutImpl(timer);
|
|
99
|
+
timer = null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return {
|
|
103
|
+
start,
|
|
104
|
+
stop,
|
|
105
|
+
nudge,
|
|
106
|
+
status: () => ({ running, dueAtMs: timer ? dueAtMs : null }),
|
|
107
|
+
};
|
|
108
|
+
}
|