@oar-sh/oar 0.9.21 → 0.9.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +206 -0
- package/README.md +9 -9
- package/bin/oar.js +28 -0
- package/package.json +13 -14
- package/server/claude-cloud-worker/claude-cloud-session-worker.mjs +9 -1
- package/server/claude-worker/claude-cli-pinned-env.mjs +4 -1
- package/server/claude-worker/claude-sdk-adapter.mjs +149 -4
- package/server/claude-worker/claude-session-process.mjs +9 -4
- package/server/claude-worker/claude-session-worker-main.mjs +10 -1
- package/server/copilot-worker/copilot-sdk-session-process.mjs +1 -0
- package/server/copilot-worker/copilot-sdk-session-worker.mjs +10 -1
- package/server/cursor-worker/cursor-session-worker.mjs +9 -1
- package/server/db-schema.mjs +19 -0
- package/server/grok-worker/grok-background-tasks.mjs +1 -0
- package/server/grok-worker/grok-session-worker.mjs +10 -1
- package/server/grok-worker/grok-turn-runner.mjs +1 -0
- package/server/migrations/0037-relay-asks-queue-id.mjs +66 -0
- package/server/migrations/0038-turn-seq.mjs +92 -0
- package/server/public/app/activity-replay-state.mjs +160 -42
- package/server/public/app/api-client.js +7 -0
- package/server/public/app/ask-user-view.js +4 -3
- package/server/public/app/bootstrap.js +62 -4
- package/server/public/app/chat-actions-menu-state.mjs +2 -0
- package/server/public/app/context-usage-view.mjs +68 -0
- package/server/public/app/conversation-view.js +306 -49
- package/server/public/app/header-priority-actions.mjs +5 -1
- package/server/public/app/held-details.mjs +202 -0
- package/server/public/app/inbox-ui.js +70 -79
- package/server/public/app/inbox-view.mjs +35 -19
- package/server/public/app/journal-view.js +17 -0
- package/server/public/app/live-activity-cap.mjs +103 -0
- package/server/public/app/questions-panel-ui.js +634 -0
- package/server/public/app/questions-stub-view.mjs +40 -0
- package/server/public/app/questions-view.mjs +89 -0
- package/server/public/app/relay-board-view.js +12 -2
- package/server/public/app/relay-thoughts.mjs +15 -1
- package/server/public/app/session-history-ui.js +1 -0
- package/server/public/app/settings-modal.js +6 -0
- package/server/public/app/socket-handlers.js +42 -11
- package/server/public/app/stream-view-preference.mjs +60 -0
- package/server/public/app/subagent-fold.mjs +6 -3
- package/server/public/app/subagent-timeline-view.mjs +12 -5
- package/server/public/app/subagent-viewer.mjs +8 -6
- package/server/public/app/thoughts-preference.mjs +79 -0
- package/server/public/app/turn-stream-view.mjs +124 -0
- package/server/public/index.html +391 -214
- package/server/repositories/message-repository.mjs +10 -0
- package/server/repositories/question-repository.mjs +51 -7
- package/server/repositories/relay-turn-rows.mjs +56 -0
- package/server/routes/ask-user-routes.mjs +34 -5
- package/server/routes/messages-routes.mjs +46 -34
- package/server/routes/relay-ask-routes.mjs +15 -5
- package/server/routes/sessions-routes.mjs +77 -6
- package/server/server-runtime.mjs +199 -171
- package/server/services/cloudflared-tunnel-service.mjs +7 -0
- package/server/services/earlier-database-service.mjs +1 -1
- package/server/services/file-serving.mjs +13 -0
- package/server/services/idle-timers.mjs +184 -0
- package/server/services/native-addon-build-service.mjs +224 -0
- package/server/services/oar-cli-helpers.mjs +7 -4
- package/server/services/queue-wake-hook.mjs +145 -0
- package/server/services/queue-waker.mjs +127 -0
- package/server/services/relay-ask-service.mjs +64 -5
- package/server/services/relay-console-log.mjs +2 -3
- package/server/services/relay-event-service.mjs +4 -1
- package/server/services/remote-relay-dispatcher.mjs +23 -4
- package/server/services/remote-relay-loopback.mjs +3 -3
- package/server/services/sdk-session-sync-service.mjs +1 -1
- package/server/services/session-history-refresh-service.mjs +4 -0
- package/server/services/session-worker-supervisor-service.mjs +36 -0
- package/server/services/session-worker-websocket-service.mjs +143 -15
- package/server/services/ssh-tunnel-manager-service.mjs +7 -0
- package/server/services/update-install-service.mjs +32 -1
- package/shared/ask-user-bridge.mjs +9 -6
- package/shared/relay-asks.mjs +10 -0
- package/shared/relay-events.mjs +1 -0
- package/shared/worker-runtime/heartbeat.mjs +26 -0
- package/shared/worker-runtime/worker-keepalive.mjs +108 -0
- package/shared/worker-runtime/worker-websocket-link.mjs +21 -10
- package/server/services/package-data-rescue-preinstall.mjs +0 -22
- package/server/services/package-data-rescue.mjs +0 -139
- package/server/tty-console-bootstrap.mjs +0 -75
|
@@ -86,18 +86,21 @@ function queueSnapshotChanged(left, right) {
|
|
|
86
86
|
|| left.parkedCount !== right.parkedCount;
|
|
87
87
|
}
|
|
88
88
|
|
|
89
|
-
// The worker
|
|
90
|
-
// worker-
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
|
|
89
|
+
// The worker keepalive beats every 30 s while the worker is idle and every
|
|
90
|
+
// 10 s while it works (shared/worker-runtime/worker-keepalive.mjs; an older
|
|
91
|
+
// worker always every 10 s), and the link itself abandons a connection after
|
|
92
|
+
// 90 s of server silence (staleConnectionMs). Mirror that on this side: three
|
|
93
|
+
// missed idle beats and a retained socket no longer proves the worker is
|
|
94
|
+
// alive.
|
|
95
|
+
export const WORKER_SOCKET_LIVENESS_MAX_SILENCE_MS = 90_000;
|
|
94
96
|
|
|
95
97
|
// Protocol features this relay understands, advertised in server.hello. A
|
|
96
98
|
// worker that updated on disk before the relay restarted must not rely on
|
|
97
99
|
// them otherwise: an older relay ignores worker.unready / hello ready:false
|
|
98
100
|
// and treats a `steering-held` requeue as a failure (retry, backoff,
|
|
99
|
-
// worker marked errored).
|
|
100
|
-
|
|
101
|
+
// worker marked errored). `idle-keepalive`: an idle worker may beat every
|
|
102
|
+
// 30 s; an older relay's liveness and CLI windows expect a beat every 10 s.
|
|
103
|
+
export const WORKER_PROTOCOL_CAPABILITIES = Object.freeze(['worker-unready', 'steering-held', 'idle-keepalive']);
|
|
101
104
|
|
|
102
105
|
export function createSessionWorkerWebSocketService({
|
|
103
106
|
WebSocketServerImpl,
|
|
@@ -108,6 +111,19 @@ export function createSessionWorkerWebSocketService({
|
|
|
108
111
|
noteWorkerHeartbeat = () => {},
|
|
109
112
|
onDeliverySendFailed = async () => {},
|
|
110
113
|
onWorkerSocketClosed = async () => {},
|
|
114
|
+
// The number of open worker sockets changed: (count). The CLI presence
|
|
115
|
+
// window is wider while a worker socket is open.
|
|
116
|
+
onSocketCountChange = () => {},
|
|
117
|
+
// Every queue snapshot the poll takes: ({pendingCount, processingCount,
|
|
118
|
+
// parkedCount}). The relay arms its worker prime while rows are pending.
|
|
119
|
+
onQueueSnapshot = () => {},
|
|
120
|
+
// The earliest future retry time of a pending row, in ms (null: none). A
|
|
121
|
+
// one-shot timer fires then, so a backoff or a usage-limit hold ends on
|
|
122
|
+
// time rather than at the next poll.
|
|
123
|
+
nextDueAtMs = () => null,
|
|
124
|
+
// That timer fired with a row due: the relay wakes its queue (prime, hand
|
|
125
|
+
// out, arm), since a row whose hold ended may have no worker yet.
|
|
126
|
+
onDue = () => {},
|
|
111
127
|
requestWork = async () => null,
|
|
112
128
|
// A worker stated its kind ("claude", "claude-api", ...) in a hello or a
|
|
113
129
|
// ready: (sessionId, kind, pid). The registry keeps it.
|
|
@@ -147,6 +163,10 @@ export function createSessionWorkerWebSocketService({
|
|
|
147
163
|
const relayBuildId = normalizeBuildId(buildId);
|
|
148
164
|
|
|
149
165
|
let interval = null;
|
|
166
|
+
let dueTimer = null;
|
|
167
|
+
// The retry time the armed due timer stands for.
|
|
168
|
+
let dueTimerForMs = null;
|
|
169
|
+
let stopped = false;
|
|
150
170
|
let lastSnapshot = null;
|
|
151
171
|
let attached = false;
|
|
152
172
|
|
|
@@ -155,10 +175,15 @@ export function createSessionWorkerWebSocketService({
|
|
|
155
175
|
return Number.isFinite(value) ? value : Date.now();
|
|
156
176
|
}
|
|
157
177
|
|
|
178
|
+
function noteSocketCount() {
|
|
179
|
+
try { onSocketCountChange(clients.size); } catch {}
|
|
180
|
+
}
|
|
181
|
+
|
|
158
182
|
function removeSocket(socket, reason) {
|
|
159
183
|
const meta = clientState.get(socket);
|
|
160
|
-
clients.delete(socket);
|
|
184
|
+
const removed = clients.delete(socket);
|
|
161
185
|
clientState.delete(socket);
|
|
186
|
+
if (removed) noteSocketCount();
|
|
162
187
|
// Death-detection hook: a worker socket closing while its session owns
|
|
163
188
|
// an in-flight processing row used to be silently forgotten, leaving
|
|
164
189
|
// the row to the 600s stale sweep. The handler PID-probes and recovers.
|
|
@@ -222,11 +247,12 @@ export function createSessionWorkerWebSocketService({
|
|
|
222
247
|
};
|
|
223
248
|
}
|
|
224
249
|
|
|
225
|
-
function emitQueueChanged(reason = 'queue-update') {
|
|
250
|
+
function emitQueueChanged(reason = 'queue-update', { skip = null } = {}) {
|
|
226
251
|
const event = toQueueChangedEvent(reason);
|
|
227
252
|
const payload = JSON.stringify(event);
|
|
228
253
|
for (const socket of clients) {
|
|
229
254
|
if (socket.readyState !== socket.OPEN) continue;
|
|
255
|
+
if (skip?.has(socket)) continue;
|
|
230
256
|
try {
|
|
231
257
|
socket.send(payload);
|
|
232
258
|
} catch {
|
|
@@ -364,25 +390,115 @@ export function createSessionWorkerWebSocketService({
|
|
|
364
390
|
}
|
|
365
391
|
}
|
|
366
392
|
|
|
393
|
+
/** Hands a row to every ready socket; resolves to the sockets that got one. */
|
|
367
394
|
async function maybeDeliverToReadyClients(reason = 'queue-update') {
|
|
395
|
+
const sockets = [];
|
|
368
396
|
const deliveries = [];
|
|
369
397
|
for (const socket of clients) {
|
|
370
398
|
const meta = clientState.get(socket);
|
|
371
399
|
if (!meta?.ready || meta.delivering) continue;
|
|
400
|
+
sockets.push(socket);
|
|
372
401
|
deliveries.push(maybeDeliverToSocket(socket, reason));
|
|
373
402
|
}
|
|
374
|
-
|
|
375
|
-
|
|
403
|
+
const handed = new Set();
|
|
404
|
+
if (!deliveries.length) return handed;
|
|
405
|
+
const results = await Promise.allSettled(deliveries);
|
|
406
|
+
results.forEach((result, index) => {
|
|
407
|
+
if (result.status === 'fulfilled' && result.value === true) handed.add(sockets[index]);
|
|
408
|
+
});
|
|
409
|
+
return handed;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
// Ready sockets get their rows first; then every other socket hears that
|
|
413
|
+
// the queue changed (a worker the relay does not count as ready, a steering
|
|
414
|
+
// one mid-turn, answers with a ready of its own). A socket just handed a
|
|
415
|
+
// row is busy with it and is not told.
|
|
416
|
+
async function deliverThenAnnounce(reason, { announce = true } = {}) {
|
|
417
|
+
const handed = await maybeDeliverToReadyClients(reason);
|
|
418
|
+
if (announce) emitQueueChanged(reason, { skip: handed });
|
|
419
|
+
return handed;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* The queue may have work: a write, a gate that lifted, a retry come due.
|
|
424
|
+
* Hands rows out now instead of at the next poll.
|
|
425
|
+
*/
|
|
426
|
+
async function notifyQueueChanged(reason = 'queue-update') {
|
|
427
|
+
if (stopped) return new Set();
|
|
428
|
+
lastSnapshot = normalizeQueueSnapshot(queueCounts());
|
|
429
|
+
// A turn that ended with nothing behind it: no row to hand out, so no
|
|
430
|
+
// dequeue attempt per ready socket and no announcement.
|
|
431
|
+
const handed = lastSnapshot.pendingCount > 0 ? await deliverThenAnnounce(reason) : new Set();
|
|
432
|
+
armDueTimer();
|
|
433
|
+
return handed;
|
|
376
434
|
}
|
|
377
435
|
|
|
436
|
+
// The backstop. With nothing pending nothing can be handed out: no dequeue
|
|
437
|
+
// attempt, only the count change announced. With rows pending (a retry in
|
|
438
|
+
// backoff, a gate, a worker not yet up) every tick tries the ready sockets,
|
|
439
|
+
// as a net under the event-driven wakes.
|
|
378
440
|
function monitorQueue() {
|
|
379
441
|
const snapshot = normalizeQueueSnapshot(queueCounts());
|
|
442
|
+
try { onQueueSnapshot(snapshot); } catch {}
|
|
380
443
|
const changed = queueSnapshotChanged(snapshot, lastSnapshot);
|
|
381
444
|
lastSnapshot = snapshot;
|
|
382
|
-
if (
|
|
445
|
+
if (snapshot.pendingCount > 0) {
|
|
446
|
+
void deliverThenAnnounce(changed ? 'queue-count-changed' : 'ready-check', { announce: changed });
|
|
447
|
+
} else if (changed) {
|
|
383
448
|
emitQueueChanged('queue-count-changed');
|
|
384
449
|
}
|
|
385
|
-
|
|
450
|
+
armDueTimer();
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
const DUE_TIMER_MIN_MS = 50;
|
|
454
|
+
const DUE_TIMER_MAX_MS = 60_000;
|
|
455
|
+
|
|
456
|
+
function clearDueTimer() {
|
|
457
|
+
dueTimerForMs = null;
|
|
458
|
+
if (!dueTimer) return;
|
|
459
|
+
clearTimeout(dueTimer);
|
|
460
|
+
dueTimer = null;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* One timer for the earliest future retry time. An armed timer for the same
|
|
465
|
+
* time is kept; any other arm clears the one before. A time further than a
|
|
466
|
+
* minute away is re-read after a minute.
|
|
467
|
+
*/
|
|
468
|
+
function armDueTimer() {
|
|
469
|
+
if (stopped) {
|
|
470
|
+
clearDueTimer();
|
|
471
|
+
return;
|
|
472
|
+
}
|
|
473
|
+
// A timer whose time has come fires and re-arms itself: a poll in the
|
|
474
|
+
// same instant (its row no longer "future") must not clear it unfired.
|
|
475
|
+
if (dueTimer && dueTimerForMs !== null && dueTimerForMs <= nowMsSafe()) return;
|
|
476
|
+
let dueAt = null;
|
|
477
|
+
try { dueAt = nextDueAtMs(); } catch { dueAt = null; }
|
|
478
|
+
const dueMs = Number(dueAt);
|
|
479
|
+
if (dueAt === null || dueAt === undefined || !Number.isFinite(dueMs)) {
|
|
480
|
+
clearDueTimer();
|
|
481
|
+
return;
|
|
482
|
+
}
|
|
483
|
+
if (dueTimer && dueTimerForMs === dueMs) return;
|
|
484
|
+
clearDueTimer();
|
|
485
|
+
const untilDueMs = dueMs - nowMsSafe();
|
|
486
|
+
const delayMs = Math.min(DUE_TIMER_MAX_MS, Math.max(DUE_TIMER_MIN_MS, untilDueMs));
|
|
487
|
+
// A time further than the cap: this firing only re-reads.
|
|
488
|
+
const reachesDue = untilDueMs <= DUE_TIMER_MAX_MS;
|
|
489
|
+
dueTimerForMs = dueMs;
|
|
490
|
+
dueTimer = setTimeout(() => {
|
|
491
|
+
dueTimer = null;
|
|
492
|
+
dueTimerForMs = null;
|
|
493
|
+
if (stopped) return;
|
|
494
|
+
if (reachesDue) {
|
|
495
|
+
// Whatever came due may go now; a row that was claimed or cancelled
|
|
496
|
+
// meanwhile makes this a wake with nothing to do.
|
|
497
|
+
try { onDue('retry-due'); } catch {}
|
|
498
|
+
}
|
|
499
|
+
armDueTimer();
|
|
500
|
+
}, delayMs);
|
|
501
|
+
if (typeof dueTimer.unref === 'function') dueTimer.unref();
|
|
386
502
|
}
|
|
387
503
|
|
|
388
504
|
function rejectUpgrade(socket) {
|
|
@@ -432,6 +548,7 @@ export function createSessionWorkerWebSocketService({
|
|
|
432
548
|
deliveryFailures: 0,
|
|
433
549
|
});
|
|
434
550
|
if (identity.sessionId) supersedePriorSockets(ws, identity.sessionId);
|
|
551
|
+
noteSocketCount();
|
|
435
552
|
touchCli();
|
|
436
553
|
emitEvent(ws, serverHello('connected', identity.sessionId));
|
|
437
554
|
|
|
@@ -466,8 +583,13 @@ export function createSessionWorkerWebSocketService({
|
|
|
466
583
|
noteWorkerBuildId(meta, payload);
|
|
467
584
|
meta.ready = true;
|
|
468
585
|
meta.lastReadyAt = nowIso();
|
|
469
|
-
|
|
470
|
-
|
|
586
|
+
const readyReason = String(payload.reason || 'worker-ready');
|
|
587
|
+
noteHeartbeat(meta, readyReason);
|
|
588
|
+
// A keepalive's ready with nothing pending has nothing to draw: one
|
|
589
|
+
// count instead of a dequeue attempt. Any row that appears wakes the
|
|
590
|
+
// queue, and this socket stays ready for it.
|
|
591
|
+
if (readyReason === 'readiness-refresh' && normalizeQueueSnapshot(queueCounts()).pendingCount === 0) return;
|
|
592
|
+
void maybeDeliverToSocket(ws, readyReason);
|
|
471
593
|
return;
|
|
472
594
|
}
|
|
473
595
|
if (payload.type === 'worker.unready') {
|
|
@@ -526,6 +648,7 @@ export function createSessionWorkerWebSocketService({
|
|
|
526
648
|
}
|
|
527
649
|
|
|
528
650
|
function start() {
|
|
651
|
+
stopped = false;
|
|
529
652
|
if (!attached) {
|
|
530
653
|
httpServer.on('upgrade', onUpgrade);
|
|
531
654
|
wss.on('connection', onConnection);
|
|
@@ -537,10 +660,13 @@ export function createSessionWorkerWebSocketService({
|
|
|
537
660
|
}
|
|
538
661
|
lastSnapshot = normalizeQueueSnapshot(queueCounts());
|
|
539
662
|
emitQueueChanged('started');
|
|
663
|
+
armDueTimer();
|
|
540
664
|
logDebug(`[worker-ws] listening on ${wsPath}`);
|
|
541
665
|
}
|
|
542
666
|
|
|
543
667
|
function stop() {
|
|
668
|
+
stopped = true;
|
|
669
|
+
clearDueTimer();
|
|
544
670
|
if (interval) {
|
|
545
671
|
clearInterval(interval);
|
|
546
672
|
interval = null;
|
|
@@ -554,6 +680,7 @@ export function createSessionWorkerWebSocketService({
|
|
|
554
680
|
}
|
|
555
681
|
clients.clear();
|
|
556
682
|
clientState.clear();
|
|
683
|
+
noteSocketCount();
|
|
557
684
|
try { wss.close(); } catch (error) {
|
|
558
685
|
logWarn(`[worker-ws] close failed: ${error?.message || error}`);
|
|
559
686
|
}
|
|
@@ -610,6 +737,7 @@ export function createSessionWorkerWebSocketService({
|
|
|
610
737
|
stop,
|
|
611
738
|
status,
|
|
612
739
|
emitQueueChanged,
|
|
740
|
+
notifyQueueChanged,
|
|
613
741
|
emitDraining,
|
|
614
742
|
handleUpgrade,
|
|
615
743
|
sendControlToSession,
|
|
@@ -171,6 +171,9 @@ export function createSshTunnelManager({
|
|
|
171
171
|
io = null,
|
|
172
172
|
logger = console,
|
|
173
173
|
runtimeShutdownRef = () => false,
|
|
174
|
+
// The tunnel stopped blocking queue delivery (it connected): the relay
|
|
175
|
+
// hands out what waited for it.
|
|
176
|
+
onBlockingLifted = () => {},
|
|
174
177
|
platform = process.platform,
|
|
175
178
|
spawnImpl = spawn,
|
|
176
179
|
nowIso = () => new Date().toISOString(),
|
|
@@ -225,7 +228,11 @@ export function createSshTunnelManager({
|
|
|
225
228
|
};
|
|
226
229
|
|
|
227
230
|
const updateBlockingState = () => {
|
|
231
|
+
const wasBlocking = state.blocking;
|
|
228
232
|
state.blocking = state.required && state.mode === 'managed' && !state.connected;
|
|
233
|
+
if (wasBlocking && !state.blocking) {
|
|
234
|
+
try { onBlockingLifted(); } catch {}
|
|
235
|
+
}
|
|
229
236
|
};
|
|
230
237
|
|
|
231
238
|
const scheduleReconnect = (spawnTunnel, delayOverrideMs = null) => {
|
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
stripTerminalEscapes,
|
|
12
12
|
tailOf,
|
|
13
13
|
} from './cli-process-runner.mjs';
|
|
14
|
+
import { NATIVE_BUILD_COMMAND, buildBetterSqlite3, needsSourceBuild } from './native-addon-build-service.mjs';
|
|
14
15
|
import { NPM_ALLOW_SCRIPTS_ARG, resolveNpmInvocation } from './oar-cli-helpers.mjs';
|
|
15
16
|
import { compareSemverIsh, parseSemverIsh } from '../../shared/update-semver.mjs';
|
|
16
17
|
|
|
@@ -107,6 +108,8 @@ export function createUpdateInstallService({
|
|
|
107
108
|
spawnImpl = defaultSpawn,
|
|
108
109
|
env = process.env,
|
|
109
110
|
platform = process.platform,
|
|
111
|
+
arch = process.arch,
|
|
112
|
+
buildNativeAddons = buildBetterSqlite3,
|
|
110
113
|
setTimeoutImpl = setTimeout,
|
|
111
114
|
clearTimeoutImpl = clearTimeout,
|
|
112
115
|
nowImpl = Date.now,
|
|
@@ -242,15 +245,43 @@ export function createUpdateInstallService({
|
|
|
242
245
|
);
|
|
243
246
|
activeChild = null;
|
|
244
247
|
flushLogNow();
|
|
245
|
-
finishedAt = new Date(nowImpl()).toISOString();
|
|
246
248
|
|
|
247
249
|
if (!result.ok) {
|
|
250
|
+
finishedAt = new Date(nowImpl()).toISOString();
|
|
248
251
|
state = UPDATE_INSTALL_STATES.ERROR;
|
|
249
252
|
lastError = result.error || `npm exited with code ${result.code}`;
|
|
250
253
|
broadcast();
|
|
251
254
|
return { ok: false, statusCode: 500, error: lastError };
|
|
252
255
|
}
|
|
253
256
|
|
|
257
|
+
// No prebuilt SQLite binding here: build it before the restart, which
|
|
258
|
+
// would start on none (native-addon-build-service.mjs). A failed build
|
|
259
|
+
// keeps this relay running on the binding it has loaded.
|
|
260
|
+
if (needsSourceBuild({ platform, arch })) {
|
|
261
|
+
const toLog = (line) => appendLog(`${line}\n`);
|
|
262
|
+
const build = await buildNativeAddons({
|
|
263
|
+
packageRoot,
|
|
264
|
+
platform,
|
|
265
|
+
arch,
|
|
266
|
+
execPath,
|
|
267
|
+
env: npm.env,
|
|
268
|
+
spawnImpl,
|
|
269
|
+
onOutput: (text) => appendLog(text),
|
|
270
|
+
onChild: (child) => { activeChild = child; },
|
|
271
|
+
logger: { log: toLog, error: toLog },
|
|
272
|
+
});
|
|
273
|
+
activeChild = null;
|
|
274
|
+
flushLogNow();
|
|
275
|
+
if (!build.ok) {
|
|
276
|
+
finishedAt = new Date(nowImpl()).toISOString();
|
|
277
|
+
state = UPDATE_INSTALL_STATES.ERROR;
|
|
278
|
+
lastError = `${UPDATE_PACKAGE_NAME}@${target} is installed, but its SQLite binding did not build (${build.error}). Run ${NATIVE_BUILD_COMMAND}, then restart the relay.`;
|
|
279
|
+
broadcast();
|
|
280
|
+
return { ok: false, statusCode: 500, error: lastError };
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
finishedAt = new Date(nowImpl()).toISOString();
|
|
284
|
+
|
|
254
285
|
// Persist the attempt BEFORE the restart request: the next boot compares
|
|
255
286
|
// the running version against it and turns "nothing changed" into a
|
|
256
287
|
// visible failure carrying this log tail.
|
|
@@ -67,7 +67,10 @@ export function createAskUserBridge({
|
|
|
67
67
|
});
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
-
|
|
70
|
+
// `source` / `rationale` override the bridge-wide card identity for one ask
|
|
71
|
+
// (the Claude worker's safety-check approvals are not clarifications), and
|
|
72
|
+
// `allowFreeform: false` makes a card answered by its choices alone.
|
|
73
|
+
async function askSingleQuestion(entry, { signal, source, rationale, allowFreeform } = {}) {
|
|
71
74
|
const activeMsg = typeof getActiveMessage === 'function' ? getActiveMessage() : null;
|
|
72
75
|
const choices = entry.options.map((option) => option.label);
|
|
73
76
|
const promptParts = [entry.question];
|
|
@@ -82,7 +85,7 @@ export function createAskUserBridge({
|
|
|
82
85
|
mode: activeMsg?.relayMode || 'agent',
|
|
83
86
|
prompt: promptParts.join('\n\n'),
|
|
84
87
|
choices,
|
|
85
|
-
allowFreeform:
|
|
88
|
+
allowFreeform: allowFreeform !== false,
|
|
86
89
|
// Top-level, as the create route reads it (elicitation parity; absent
|
|
87
90
|
// for the flat question shape every existing caller sends).
|
|
88
91
|
...(entry.requestedSchema ? { requestedSchema: entry.requestedSchema } : {}),
|
|
@@ -92,8 +95,8 @@ export function createAskUserBridge({
|
|
|
92
95
|
attemptId: activeMsg?.attemptId || undefined,
|
|
93
96
|
timeout_ms: questionTimeoutMs,
|
|
94
97
|
context: {
|
|
95
|
-
source: questionSource,
|
|
96
|
-
rationale: questionRationale,
|
|
98
|
+
source: source || questionSource,
|
|
99
|
+
rationale: rationale || questionRationale,
|
|
97
100
|
queueMessageId: activeMsg?.id || null,
|
|
98
101
|
conversationId: activeMsg?.conversationId || null,
|
|
99
102
|
relayMode: activeMsg?.relayMode || 'agent',
|
|
@@ -106,7 +109,7 @@ export function createAskUserBridge({
|
|
|
106
109
|
return waitForRelayQuestionAnswer(questionId, { signal });
|
|
107
110
|
}
|
|
108
111
|
|
|
109
|
-
async function handleAskUserQuestion(input, { signal } = {}) {
|
|
112
|
+
async function handleAskUserQuestion(input, { signal, source, rationale, allowFreeform } = {}) {
|
|
110
113
|
const questions = normalizeQuestions(input);
|
|
111
114
|
if (!questions.length) {
|
|
112
115
|
return { answers: {}, structuredAnswers: {}, timedOut: false };
|
|
@@ -118,7 +121,7 @@ export function createAskUserBridge({
|
|
|
118
121
|
const structuredAnswers = {};
|
|
119
122
|
let timedOut = false;
|
|
120
123
|
for (const entry of questions) {
|
|
121
|
-
const result = await askSingleQuestion(entry, { signal });
|
|
124
|
+
const result = await askSingleQuestion(entry, { signal, source, rationale, allowFreeform });
|
|
122
125
|
answers[entry.question] = result.answer;
|
|
123
126
|
if (result.structuredAnswer) structuredAnswers[entry.question] = result.structuredAnswer;
|
|
124
127
|
if (entry.rawQuestion && entry.rawQuestion !== entry.question) {
|
package/shared/relay-asks.mjs
CHANGED
|
@@ -25,6 +25,11 @@ export const RELAY_QUESTION_KINDS = Object.freeze(['ask', 'ask_user', 'approval'
|
|
|
25
25
|
// Stored and sent between relays, so it kept the tool's name from before
|
|
26
26
|
// 0.9.18.
|
|
27
27
|
export const REMOTE_RELAY_APPROVAL_SOURCE = 'remote_relay';
|
|
28
|
+
// The `context.source` of a Claude session's Safety check card: an ask Claude
|
|
29
|
+
// Code marks as person-only (claude-sdk-adapter.mjs). Like the approval card,
|
|
30
|
+
// it is the user's to answer: never listed to an agent, never answered by
|
|
31
|
+
// one, from this relay or another.
|
|
32
|
+
export const PERSON_ONLY_QUESTION_SOURCE = 'ClaudeSafetyCheck';
|
|
28
33
|
export const RELAY_ASK_KIND = 'ask';
|
|
29
34
|
|
|
30
35
|
export const RELAY_ASK_URGENCIES = Object.freeze(['normal', 'now']);
|
|
@@ -145,6 +150,11 @@ export function normalizeRelayAskAnswer(ask, { answer = '', choice = '' } = {})
|
|
|
145
150
|
return { ok: true, value: { answer: text || picked, choice: picked } };
|
|
146
151
|
}
|
|
147
152
|
|
|
153
|
+
/** True for a Safety check card's context (a formatted question's or a stored request envelope's). */
|
|
154
|
+
export function isPersonOnlyQuestion(context) {
|
|
155
|
+
return !!context && typeof context === 'object' && toText(context.source) === PERSON_ONLY_QUESTION_SOURCE;
|
|
156
|
+
}
|
|
157
|
+
|
|
148
158
|
/** The kind of a formatted question card (a row of /api/relay-questions). */
|
|
149
159
|
export function relayQuestionKind(question) {
|
|
150
160
|
const kind = toText(question?.kind);
|
package/shared/relay-events.mjs
CHANGED
|
@@ -560,6 +560,7 @@ export function formatRelayEventLine(event) {
|
|
|
560
560
|
const parts = [`${head} ${quote(detail.prompt, RELAY_EVENT_LIMITS.questionChars)}`];
|
|
561
561
|
if (options.length) parts.push(`options: ${options.map((option) => `[${option}]`).join(' ')}`);
|
|
562
562
|
if (detail.approvalCard) parts.push('(an approval card: only the user can answer it)');
|
|
563
|
+
else if (detail.personOnly) parts.push('(a Safety check: the user\'s to answer)');
|
|
563
564
|
else if (oneLine(detail.questionId, 80)) parts.push(`(question_id ${oneLine(detail.questionId, 80)})`);
|
|
564
565
|
if (detail.kind === 'ask') parts.push('(asked with ask: it does not block that session)');
|
|
565
566
|
return parts.join(' ');
|
|
@@ -33,6 +33,10 @@ export function createHeartbeatController({
|
|
|
33
33
|
// Optional: called after every beat the relay answered, for work that
|
|
34
34
|
// waits for a reachable relay (posts kept after a failure).
|
|
35
35
|
onAnswered = () => {},
|
|
36
|
+
// Optional: how many posts are kept for a reachable relay (flushed by
|
|
37
|
+
// onAnswered). While any is, the worker counts as busy, so the next beat,
|
|
38
|
+
// which flushes them, is not an idle wait away.
|
|
39
|
+
getKeptPostCount = null,
|
|
36
40
|
// Optional: the kind of worker this is ("claude", "claude-api", ...). The
|
|
37
41
|
// relay hands a Claude chat's row only to a worker of the row's billing,
|
|
38
42
|
// and knows a worker's kind from what it says here and in its hello.
|
|
@@ -99,6 +103,27 @@ export function createHeartbeatController({
|
|
|
99
103
|
}
|
|
100
104
|
}
|
|
101
105
|
|
|
106
|
+
/**
|
|
107
|
+
* Whether the worker has work the relay follows by its beats: a row it
|
|
108
|
+
* owns, a live background task, a running turn, a settle failure to report,
|
|
109
|
+
* a kept post to flush.
|
|
110
|
+
* The keepalive beats at its busy cadence while this holds
|
|
111
|
+
* (worker-keepalive.mjs). A getter that throws counts as work.
|
|
112
|
+
*/
|
|
113
|
+
function hasWork() {
|
|
114
|
+
try {
|
|
115
|
+
if (typeof getActiveQueueMessageId === "function" && String(getActiveQueueMessageId() || "").trim()) return true;
|
|
116
|
+
if (typeof getActiveQueueMessageIds === "function" && (getActiveQueueMessageIds() || []).length) return true;
|
|
117
|
+
if (typeof getBackgroundTaskIds === "function" && (getBackgroundTaskIds() || []).length) return true;
|
|
118
|
+
if (typeof getSettleFailed === "function" && (getSettleFailed() || []).length) return true;
|
|
119
|
+
if (typeof getKeptPostCount === "function" && Number(getKeptPostCount()) > 0) return true;
|
|
120
|
+
if (typeof getSteeringState === "function" && getSteeringState()?.turnActive === true) return true;
|
|
121
|
+
return false;
|
|
122
|
+
} catch {
|
|
123
|
+
return true;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
102
127
|
/**
|
|
103
128
|
* An out-of-cycle pulse for state the relay should see before the next
|
|
104
129
|
* periodic one (the composer's steering snapshot, a relay that just came
|
|
@@ -135,6 +160,7 @@ export function createHeartbeatController({
|
|
|
135
160
|
}
|
|
136
161
|
|
|
137
162
|
return {
|
|
163
|
+
hasWork,
|
|
138
164
|
pulseHeartbeat,
|
|
139
165
|
requestPulse,
|
|
140
166
|
startHeartbeat,
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
// The worker's one keepalive timer. Each beat posts the heartbeat
|
|
2
|
+
// (heartbeat.mjs) and refreshes the websocket readiness
|
|
3
|
+
// (worker-websocket-link.mjs, built with readyRefreshMs: 0); before, each ran
|
|
4
|
+
// its own 10 s timer.
|
|
5
|
+
//
|
|
6
|
+
// Cadence: every 10 s while the worker has work (a row it owns, a delivery in
|
|
7
|
+
// flight, a hold, a background task, a running turn), and every 30 s while it
|
|
8
|
+
// is idle, but only on a relay that advertised `idle-keepalive` in its
|
|
9
|
+
// server.hello. An older relay expects a beat every 10 s (its liveness, CLI
|
|
10
|
+
// and stranded-session windows), and a worker whose socket is down knows no
|
|
11
|
+
// relay at all, so both get 10 s. Every server.hello and every delivery
|
|
12
|
+
// (nudge) keeps the busy cadence for the next 30 s and cuts an idle wait
|
|
13
|
+
// down to 10 s: the first beat naming a new turn's row is never an idle wait
|
|
14
|
+
// away, and a delivery lost on the wire or a relay that just came up is
|
|
15
|
+
// noticed as fast as before.
|
|
16
|
+
|
|
17
|
+
export const IDLE_KEEPALIVE_MS = 30_000;
|
|
18
|
+
export const BUSY_KEEPALIVE_MS = 10_000;
|
|
19
|
+
export const IDLE_KEEPALIVE_CAPABILITY = "idle-keepalive";
|
|
20
|
+
export const BUSY_HOLD_MS = 30_000;
|
|
21
|
+
|
|
22
|
+
export function createWorkerKeepalive({
|
|
23
|
+
heartbeat,
|
|
24
|
+
link,
|
|
25
|
+
idleMs = IDLE_KEEPALIVE_MS,
|
|
26
|
+
busyMs = BUSY_KEEPALIVE_MS,
|
|
27
|
+
busyHoldMs = BUSY_HOLD_MS,
|
|
28
|
+
setTimeoutImpl = setTimeout,
|
|
29
|
+
clearTimeoutImpl = clearTimeout,
|
|
30
|
+
now = () => Date.now(),
|
|
31
|
+
} = {}) {
|
|
32
|
+
let timer = null;
|
|
33
|
+
let dueAtMs = 0;
|
|
34
|
+
let running = false;
|
|
35
|
+
let busyUntilMs = 0;
|
|
36
|
+
|
|
37
|
+
function busy() {
|
|
38
|
+
if (now() < busyUntilMs) return true;
|
|
39
|
+
try {
|
|
40
|
+
if (heartbeat?.hasWork?.() === true) return true;
|
|
41
|
+
if (link?.isBusy?.() === true) return true;
|
|
42
|
+
return false;
|
|
43
|
+
} catch {
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function relayAllowsIdleCadence() {
|
|
49
|
+
try {
|
|
50
|
+
return link?.serverSupports?.(IDLE_KEEPALIVE_CAPABILITY) === true;
|
|
51
|
+
} catch {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** The wait before the next beat, decided after each beat. */
|
|
57
|
+
function nextDelayMs() {
|
|
58
|
+
if (busy() || !relayAllowsIdleCadence()) return busyMs;
|
|
59
|
+
return idleMs;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function schedule(delayMs) {
|
|
63
|
+
if (timer) clearTimeoutImpl(timer);
|
|
64
|
+
dueAtMs = now() + delayMs;
|
|
65
|
+
timer = setTimeoutImpl(beat, delayMs);
|
|
66
|
+
timer?.unref?.();
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function beat() {
|
|
70
|
+
timer = null;
|
|
71
|
+
if (!running) return;
|
|
72
|
+
void heartbeat?.pulseHeartbeat?.();
|
|
73
|
+
try { link?.refreshReadiness?.("readiness-refresh"); } catch {}
|
|
74
|
+
schedule(nextDelayMs());
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function start() {
|
|
78
|
+
if (running) return;
|
|
79
|
+
running = true;
|
|
80
|
+
// The first heartbeat goes out at once, as the heartbeat's own start did;
|
|
81
|
+
// the socket is not open yet, so its refresh waits for the next beat.
|
|
82
|
+
void heartbeat?.pulseHeartbeat?.();
|
|
83
|
+
schedule(nextDelayMs());
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The relay said hello or handed over work: the busy cadence holds for a
|
|
88
|
+
* while, and a beat further away than it comes sooner.
|
|
89
|
+
*/
|
|
90
|
+
function nudge() {
|
|
91
|
+
if (!running) return;
|
|
92
|
+
busyUntilMs = Math.max(busyUntilMs, now() + busyHoldMs);
|
|
93
|
+
if (timer && dueAtMs - now() > busyMs) schedule(busyMs);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function stop() {
|
|
97
|
+
running = false;
|
|
98
|
+
if (timer) clearTimeoutImpl(timer);
|
|
99
|
+
timer = null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return {
|
|
103
|
+
start,
|
|
104
|
+
stop,
|
|
105
|
+
nudge,
|
|
106
|
+
status: () => ({ running, dueAtMs: timer ? dueAtMs : null }),
|
|
107
|
+
};
|
|
108
|
+
}
|
|
@@ -63,9 +63,13 @@ export function createWorkerWebSocketLink({
|
|
|
63
63
|
getDeliveryHeld = null,
|
|
64
64
|
// Called with the hello's reason once per connection, on the relay's first
|
|
65
65
|
// server.hello: from then on the relay (possibly a restarted one that knows
|
|
66
|
-
// nothing of this worker) has the socket. Later hellos only ack
|
|
67
|
-
// worker.hello
|
|
66
|
+
// nothing of this worker) has the socket. Later hellos only ack a later
|
|
67
|
+
// worker.hello, and are not reported.
|
|
68
68
|
onServerHello = () => {},
|
|
69
|
+
// Called with "server-hello" on every server.hello and with "deliver" on
|
|
70
|
+
// every queue.deliver: the keepalive keeps its busy cadence for a while
|
|
71
|
+
// after either (worker-keepalive.mjs).
|
|
72
|
+
onRelayFrame = () => {},
|
|
69
73
|
getSessionId = () => null,
|
|
70
74
|
getPid = () => null,
|
|
71
75
|
// The kind of worker this is ("claude", "claude-api", ...), stated in
|
|
@@ -88,8 +92,12 @@ export function createWorkerWebSocketLink({
|
|
|
88
92
|
// Failed attempts after the first few are logged at most once per this
|
|
89
93
|
// interval, with a count of the ones in between.
|
|
90
94
|
failureLogIntervalMs = 60_000,
|
|
95
|
+
// The link's own readiness refresh; 0 leaves it to the caller, which then
|
|
96
|
+
// calls refreshReadiness itself (worker-keepalive.mjs does, on its beat).
|
|
91
97
|
readyRefreshMs = 10_000,
|
|
92
|
-
|
|
98
|
+
// Three missed idle beats (30 s each) without any word from the relay: the
|
|
99
|
+
// connection is given up and made again.
|
|
100
|
+
staleConnectionMs = 90_000,
|
|
93
101
|
WebSocketImpl = globalThis.WebSocket,
|
|
94
102
|
setTimeoutImpl = setTimeout,
|
|
95
103
|
clearTimeoutImpl = clearTimeout,
|
|
@@ -309,19 +317,17 @@ export function createWorkerWebSocketLink({
|
|
|
309
317
|
notifyUnready(reason);
|
|
310
318
|
return true;
|
|
311
319
|
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
return true;
|
|
317
|
-
}
|
|
318
|
-
notifyHello(reason);
|
|
320
|
+
// notifyReady gates itself: mid-delivery it only goes through when the
|
|
321
|
+
// worker is steering-ready. No hello: the socket's identity, kind and
|
|
322
|
+
// build were bound by the hello at open, and a ready carries them again;
|
|
323
|
+
// a hello would only draw a second server.hello and a second ready.
|
|
319
324
|
void notifyReady(reason);
|
|
320
325
|
return true;
|
|
321
326
|
}
|
|
322
327
|
|
|
323
328
|
function startReadyRefreshTimer() {
|
|
324
329
|
clearReadyRefreshTimer();
|
|
330
|
+
if (Number(readyRefreshMs) === 0) return;
|
|
325
331
|
const intervalMs = Math.max(1000, Number(readyRefreshMs) || 10_000);
|
|
326
332
|
readyRefreshTimer = setIntervalImpl(() => {
|
|
327
333
|
refreshReadiness("readiness-refresh");
|
|
@@ -449,11 +455,13 @@ export function createWorkerWebSocketLink({
|
|
|
449
455
|
return;
|
|
450
456
|
}
|
|
451
457
|
if (payload?.type === "queue.deliver") {
|
|
458
|
+
try { onRelayFrame("deliver"); } catch {}
|
|
452
459
|
void deliverPending(payload.pending || null, String(payload.reason || "queue-deliver"))
|
|
453
460
|
.finally(() => notifyReady("delivery-complete"));
|
|
454
461
|
return;
|
|
455
462
|
}
|
|
456
463
|
if (payload?.type === "server.hello") {
|
|
464
|
+
try { onRelayFrame("server-hello"); } catch {}
|
|
457
465
|
serverCapabilities = new Set(Array.isArray(payload.capabilities) ? payload.capabilities.map(String) : []);
|
|
458
466
|
relayBuildId = normalizeText(payload.buildId);
|
|
459
467
|
// Only a line in the log: the relay decides when to replace this
|
|
@@ -565,6 +573,9 @@ export function createWorkerWebSocketLink({
|
|
|
565
573
|
return {
|
|
566
574
|
notifyReady,
|
|
567
575
|
notifyUnready,
|
|
576
|
+
refreshReadiness,
|
|
577
|
+
// A delivery in flight or a hold: the keepalive beats at its busy cadence.
|
|
578
|
+
isBusy: () => deliveriesInFlight.size > 0 || deliveryHeld(),
|
|
568
579
|
serverSupports: (capability) => serverCapabilities.has(String(capability || "")),
|
|
569
580
|
start,
|
|
570
581
|
stop,
|