@oar-sh/oar 0.9.2 → 0.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/.github/extensions/web-relay/extension.mjs +6 -1
  2. package/.github/extensions/web-relay/skills/tool-activity.mjs +7 -0
  3. package/CHANGELOG.md +336 -0
  4. package/README.md +455 -423
  5. package/bin/oar.js +25 -10
  6. package/package.json +11 -5
  7. package/server/README.md +44 -4
  8. package/server/claude-worker/claude-remote-relay-tool.mjs +99 -0
  9. package/server/claude-worker/claude-sdk-adapter.mjs +26 -4
  10. package/server/claude-worker/claude-session-process.mjs +1073 -69
  11. package/server/claude-worker/claude-session-worker.mjs +45 -4
  12. package/server/claude-worker/claude-turn-publisher.mjs +81 -20
  13. package/server/claude-worker/claude-worker-link-wiring.mjs +9 -0
  14. package/server/claude-worker/sdk-message-normalizer.mjs +7 -0
  15. package/server/copilot-worker/copilot-ask-user-tool.mjs +89 -0
  16. package/server/copilot-worker/copilot-continuation-signals.mjs +55 -0
  17. package/server/copilot-worker/copilot-question-bridge.mjs +34 -12
  18. package/server/copilot-worker/copilot-remote-relay-tool.mjs +59 -0
  19. package/server/copilot-worker/copilot-sdk-event-normalizer.mjs +113 -14
  20. package/server/copilot-worker/copilot-sdk-session-process.mjs +2349 -498
  21. package/server/copilot-worker/copilot-sdk-session-worker.mjs +82 -20
  22. package/server/cursor-worker/cursor-remote-relay-tool.mjs +65 -0
  23. package/server/cursor-worker/cursor-turn-runner.mjs +45 -2
  24. package/server/cursor-worker/sdk-message-normalizer.mjs +25 -0
  25. package/server/db-schema.mjs +28 -0
  26. package/server/grok-worker/acp-client.mjs +92 -6
  27. package/server/grok-worker/grok-turn-runner.mjs +59 -8
  28. package/server/grok-worker/sdk-message-normalizer.mjs +13 -0
  29. package/server/mcp/oar-mcp-server.mjs +473 -0
  30. package/server/migrations/0004-steer-settle-markers.mjs +119 -0
  31. package/server/migrations/0005-remote-relays.mjs +90 -0
  32. package/server/public/app/action-confirmations.js +141 -13
  33. package/server/public/app/api-client.js +125 -2
  34. package/server/public/app/ask-user-view.js +102 -13
  35. package/server/public/app/attachments-view.js +37 -2
  36. package/server/public/app/background-task-timeout-options.mjs +9 -0
  37. package/server/public/app/background-tasks-view.mjs +33 -18
  38. package/server/public/app/bootstrap.js +55 -14
  39. package/server/public/app/composer-control-state.mjs +52 -34
  40. package/server/public/app/conversation-deep-link.mjs +42 -0
  41. package/server/public/app/conversation-draft-sync.mjs +187 -0
  42. package/server/public/app/conversation-list-filter.mjs +86 -0
  43. package/server/public/app/conversation-provider-indicator.mjs +8 -0
  44. package/server/public/app/conversation-view.js +1016 -200
  45. package/server/public/app/emoji-view.js +2 -0
  46. package/server/public/app/font-scaling.js +4 -37
  47. package/server/public/app/host-suspend-ui.js +272 -0
  48. package/server/public/app/journal-view.js +149 -6
  49. package/server/public/app/mention-autocomplete.mjs +298 -0
  50. package/server/public/app/model-selector-options.mjs +1 -1
  51. package/server/public/app/push-settings.js +1 -0
  52. package/server/public/app/pwa-install.js +8 -3
  53. package/server/public/app/question-multi-select.mjs +52 -0
  54. package/server/public/app/relay-prompt-sanitizer.mjs +61 -3
  55. package/server/public/app/remote-relay-origin-view.mjs +76 -0
  56. package/server/public/app/remote-relay-shared.mjs +85 -0
  57. package/server/public/app/remote-relays-settings.js +618 -0
  58. package/server/public/app/remote-relays-store.mjs +126 -0
  59. package/server/public/app/settings-modal.js +12 -8
  60. package/server/public/app/settings-tabs.js +1 -1
  61. package/server/public/app/socket-handlers.js +37 -1
  62. package/server/public/app/steered-turn-merge.mjs +100 -0
  63. package/server/public/app/store.js +126 -15
  64. package/server/public/app/sync-outbox.mjs +16 -8
  65. package/server/public/app/thinking-follow.js +260 -0
  66. package/server/public/app/tool-activity-emoji.mjs +40 -0
  67. package/server/public/app/transcript-viewport-anchor.mjs +210 -0
  68. package/server/public/index.html +463 -17
  69. package/server/public/sw.js +7 -0
  70. package/server/relay-tools.md +7 -1
  71. package/server/repositories/message-repository.mjs +68 -18
  72. package/server/repositories/question-repository.mjs +18 -0
  73. package/server/repositories/remote-relay-repository.mjs +75 -0
  74. package/server/repositories/session-repository.mjs +13 -32
  75. package/server/routes/ask-user-routes.mjs +42 -2
  76. package/server/routes/messages-routes.mjs +572 -56
  77. package/server/routes/remote-relay-routes.mjs +133 -0
  78. package/server/routes/sessions-routes.mjs +378 -84
  79. package/server/server-runtime.mjs +303 -25
  80. package/server/services/github-ci-activity-service.mjs +199 -0
  81. package/server/services/host-suspend-command.mjs +38 -0
  82. package/server/services/host-suspend-service.mjs +254 -0
  83. package/server/services/live-turn-picker.mjs +47 -0
  84. package/server/services/push-dispatch-service.mjs +37 -4
  85. package/server/services/relay-prompt-sanitizer.mjs +17 -2
  86. package/server/services/relay-question-context.mjs +46 -0
  87. package/server/services/remote-relay-caller-context.mjs +341 -0
  88. package/server/services/remote-relay-client.mjs +332 -0
  89. package/server/services/remote-relay-dispatcher.mjs +1500 -0
  90. package/server/services/remote-relay-inbound.mjs +187 -0
  91. package/server/services/remote-relay-pairing.mjs +189 -0
  92. package/server/services/remote-relay-registry-service.mjs +581 -0
  93. package/server/services/sdk-session-import-service.mjs +49 -7
  94. package/server/services/session-worker-launch-service.mjs +131 -12
  95. package/server/services/session-worker-registry-service.mjs +75 -0
  96. package/server/services/session-worker-supervisor-service.mjs +7 -1
  97. package/server/services/session-worker-websocket-service.mjs +23 -1
  98. package/server/services/steer-resend.mjs +45 -0
  99. package/shared/background-task-timeout.mjs +12 -4
  100. package/shared/control-poller.mjs +4 -1
  101. package/shared/question-multi-select.mjs +16 -0
  102. package/shared/relay-stop-failure.mjs +15 -0
  103. package/shared/remote-relay-contract.mjs +539 -0
  104. package/shared/remote-relay-mentions.mjs +129 -0
  105. package/shared/remote-relay-tool-core.mjs +314 -0
  106. package/shared/slash-command-guard.mjs +76 -0
  107. package/shared/steer-note.mjs +66 -0
  108. package/shared/steer-settle-failure.mjs +89 -0
  109. package/shared/steer-settle-markers.mjs +18 -0
  110. package/shared/worker-crash-guard.mjs +40 -25
  111. package/shared/worker-runtime/api-client.mjs +112 -22
  112. package/shared/worker-runtime/heartbeat.mjs +55 -1
  113. package/shared/worker-runtime/runner-link-wiring.mjs +44 -0
  114. package/shared/worker-runtime/worker-websocket-link.mjs +119 -8
  115. package/server/claude-worker/claude-session-test-harness.mjs +0 -250
  116. package/server/claude-worker/fixtures/workflow-run-record.json +0 -1
  117. package/server/copilot-worker/copilot-sdk-test-harness.mjs +0 -382
  118. package/server/copilot-worker/fixtures/abort-turn.json +0 -588
  119. package/server/copilot-worker/fixtures/ask-user-turn.json +0 -1154
  120. package/server/copilot-worker/fixtures/background-timer-continuation.json +0 -445
  121. package/server/copilot-worker/fixtures/background-timer-turn.json +0 -497
  122. package/server/copilot-worker/fixtures/happy-turn.json +0 -566
  123. package/server/copilot-worker/fixtures/quota-turn.json +0 -647
  124. package/server/copilot-worker/fixtures/reasoning-turn.json +0 -625
  125. package/server/copilot-worker/fixtures/resume-turn.json +0 -700
  126. package/server/copilot-worker/fixtures/subagent-turn.json +0 -1378
  127. package/server/copilot-worker/fixtures/tool-permission-turn.json +0 -1293
  128. package/server/routes/messages-routes-test-harness.mjs +0 -111
  129. package/server/services/fixtures/claude-auth-stub.cmd +0 -1
  130. package/server/services/fixtures/claude-auth-stub.mjs +0 -76
  131. package/server/services/fixtures/claude-auth-stub.sh +0 -4
  132. package/server/services/fixtures/cli-install-stub.cmd +0 -1
  133. package/server/services/fixtures/cli-install-stub.mjs +0 -98
  134. package/server/services/fixtures/cli-install-stub.sh +0 -4
  135. package/server/services/fixtures/grok-stub.cmd +0 -1
  136. package/server/services/fixtures/grok-stub.mjs +0 -89
  137. package/server/services/fixtures/grok-stub.sh +0 -4
  138. package/shared/fixtures/copilot-catalog-raw-2026-09-07.json +0 -2363
@@ -24,6 +24,9 @@ import {
24
24
  normalizeWorkspaceRootKey,
25
25
  } from './services/workspace-root-path-policy.mjs';
26
26
  import { stopSessionWorkerProcesses } from './services/session-worker-stop-service.mjs';
27
+ import { pickLiveTurnRowId } from './services/live-turn-picker.mjs';
28
+ import { buildSteerSettleFailure, steerAgentLabelForProvider } from '../shared/steer-settle-failure.mjs';
29
+ import { sanitizeRelayQuestionContext as sanitizeRelayQuestionContextWith } from './services/relay-question-context.mjs';
27
30
  import { isRealPathWithinRoot } from './services/workspace-symlink-guard.mjs';
28
31
  import { applySchema } from './db-schema.mjs';
29
32
  import { createSessionRepository } from './repositories/session-repository.mjs';
@@ -37,6 +40,13 @@ import { registerRelayBoardRoutes } from './routes/relay-board-routes.mjs';
37
40
  import { registerCacheRoutes } from './routes/cache-routes.mjs';
38
41
  import { registerGitRoutes } from './routes/git-routes.mjs';
39
42
  import { registerPreviewRoutes } from './routes/preview-routes.mjs';
43
+ import { registerRemoteRelayRoutes } from './routes/remote-relay-routes.mjs';
44
+ import { createRemoteRelayRepository } from './repositories/remote-relay-repository.mjs';
45
+ import { createRemoteRelayClient } from './services/remote-relay-client.mjs';
46
+ import { createRemoteRelayRegistry } from './services/remote-relay-registry-service.mjs';
47
+ import { createRemoteRelayPairing } from './services/remote-relay-pairing.mjs';
48
+ import { createRemoteRelayCallerContext } from './services/remote-relay-caller-context.mjs';
49
+ import { createRemoteRelayDispatcher } from './services/remote-relay-dispatcher.mjs';
40
50
  import { createDeleteArchiveService } from './services/delete-archive-service.mjs';
41
51
  import { createStatusEventService } from './services/status-event-service.mjs';
42
52
  import { createClaudeAuthService } from './services/claude-auth-service.mjs';
@@ -45,6 +55,9 @@ import { createGrokAuthService } from './services/grok-auth-service.mjs';
45
55
  import { sweepUnreferencedUploads, UNREFERENCED_UPLOADS_QUERY } from './services/upload-sweep.mjs';
46
56
  import webpush from 'web-push';
47
57
  import { createPushDispatchService, ensurePushVapidKeys } from './services/push-dispatch-service.mjs';
58
+ import { createHostSuspendService } from './services/host-suspend-service.mjs';
59
+ import { createGitHubCiActivityService } from './services/github-ci-activity-service.mjs';
60
+ import { runHostSuspendToRam } from './services/host-suspend-command.mjs';
48
61
  import { createActiveDeviceTracker } from './services/push-active-device-service.mjs';
49
62
  import { registerPushRoutes } from './routes/push-routes.mjs';
50
63
  import { createImageOperationService } from './services/image-operation-service.mjs';
@@ -3133,6 +3146,36 @@ function getRelayShutdownState() {
3133
3146
  };
3134
3147
  }
3135
3148
 
3149
+ // Every client shows a banner while a shutdown/restart is queued, so the
3150
+ // state goes out on each transition (request, cancel, drain) and on connect.
3151
+ function relayShutdownStatePayload() {
3152
+ return { ...getRelayShutdownState(), queue: queueCounts() };
3153
+ }
3154
+ function emitRelayShutdownState() {
3155
+ try { io.emit('relay_shutdown_state', relayShutdownStatePayload()); } catch {}
3156
+ }
3157
+
3158
+ function cancelRelayShutdown({ requestedBy = 'localhost-api' } = {}) {
3159
+ if (runtimeShutdownStarted) {
3160
+ return { cancelled: false, status: 'shutting_down', ...relayShutdownStatePayload() };
3161
+ }
3162
+ if (!pendingRelayShutdownRequest) {
3163
+ return { cancelled: false, ...relayShutdownStatePayload() };
3164
+ }
3165
+ const dropped = pendingRelayShutdownRequest;
3166
+ pendingRelayShutdownRequest = null;
3167
+ if (runtimeTimers.shutdownDrain) {
3168
+ try { clearInterval(runtimeTimers.shutdownDrain); } catch {}
3169
+ runtimeTimers.shutdownDrain = null;
3170
+ }
3171
+ console.log(
3172
+ `${runtimeLogPrefix()}Relay ${dropped.action || 'shutdown'} request cancelled by `
3173
+ + `${String(requestedBy || 'localhost-api').trim().slice(0, 80) || 'localhost-api'}`,
3174
+ );
3175
+ emitRelayShutdownState();
3176
+ return { cancelled: true, dropped, ...relayShutdownStatePayload() };
3177
+ }
3178
+
3136
3179
  function requestRelayShutdown({ reason = 'manual-request', requestedBy = 'localhost-api', restart = false } = {}) {
3137
3180
  const safeReason = String(reason || 'manual-request').trim().slice(0, 140) || 'manual-request';
3138
3181
  const safeRequestedBy = String(requestedBy || 'localhost-api').trim().slice(0, 80) || 'localhost-api';
@@ -3185,12 +3228,14 @@ function requestRelayShutdown({ reason = 'manual-request', requestedBy = 'localh
3185
3228
  );
3186
3229
  sessionWorkerWebSocketService.emitDraining(`api-${action}:${requestInfo?.reason || 'manual-request'}`);
3187
3230
  void shutdownRuntime(`api-${action}:${requestInfo?.reason || 'manual-request'}`, { exitCode });
3231
+ emitRelayShutdownState();
3188
3232
  };
3189
3233
 
3190
3234
  if (!runtimeTimers.shutdownDrain) {
3191
3235
  runtimeTimers.shutdownDrain = setInterval(tryShutdownWhenIdle, 1000);
3192
3236
  }
3193
3237
  tryShutdownWhenIdle();
3238
+ if (pendingRelayShutdownRequest) emitRelayShutdownState();
3194
3239
  const counts = queueCounts();
3195
3240
  return {
3196
3241
  accepted: true,
@@ -4880,8 +4925,127 @@ const backgroundTaskStore = {
4880
4925
  },
4881
4926
  };
4882
4927
 
4928
+ // ─── Deferred host suspend ───────────────────────────────────────────────────
4929
+ // "💤 Suspend host" waits until nothing runs any more: no queued/processing
4930
+ // turn, no live background task of a live worker, no open CI run in a
4931
+ // workspace that was busy since the request. The relay-side signals are read
4932
+ // synchronously each tick; CI is refreshed in the background and read from
4933
+ // the last snapshot (a never-checked workspace counts as busy).
4934
+ const hostActivityWorkerStatuses = new Set(['starting', 'ready', 'processing']);
4935
+ function hostActivityPidAlive(pidValue) {
4936
+ const pid = Number(pidValue);
4937
+ if (!Number.isInteger(pid) || pid <= 0) return true;
4938
+ try {
4939
+ process.kill(pid, 0);
4940
+ return true;
4941
+ } catch (error) {
4942
+ return error?.code === 'EPERM';
4943
+ }
4944
+ }
4945
+ function hostActivityWorkerAlive(key) {
4946
+ const worker = sessionWorkerRegistry?.getWorker?.(key)
4947
+ || sessionWorkerRegistry?.getWorkerByConversationId?.(key)
4948
+ || null;
4949
+ if (!worker) return false;
4950
+ return hostActivityWorkerStatuses.has(String(worker.status || '').trim().toLowerCase())
4951
+ && hostActivityPidAlive(worker.pid);
4952
+ }
4953
+ const hostSuspendWatchedConversationIds = new Set();
4954
+ const githubCiActivityService = createGitHubCiActivityService({ logger: console });
4955
+ function hostActivityTitle(conversationId, fallbackTitle = null) {
4956
+ const title = String(fallbackTitle || stmts.getConv?.get?.(conversationId)?.title || '').trim();
4957
+ return title || String(conversationId || '').slice(0, 8);
4958
+ }
4959
+ function collectHostActivity({ request = null } = {}) {
4960
+ const blockers = [];
4961
+ const turnsByConversation = stmts.listActiveQueueCountsByConversation?.all?.() || [];
4962
+ for (const row of turnsByConversation) {
4963
+ const conversationId = String(row.conversation_id || '');
4964
+ if (!conversationId) continue;
4965
+ hostSuspendWatchedConversationIds.add(conversationId);
4966
+ const count = Number(row.cnt || 0);
4967
+ blockers.push({
4968
+ kind: 'turn',
4969
+ conversationId,
4970
+ title: hostActivityTitle(conversationId, row.title),
4971
+ count,
4972
+ detail: count > 1 ? `${count} turns running` : 'turn running',
4973
+ });
4974
+ }
4975
+ for (const [key, entry] of backgroundTaskStore.sets) {
4976
+ const tasks = Array.isArray(entry?.tasks) ? entry.tasks : [];
4977
+ if (!tasks.length || !hostActivityWorkerAlive(key)) continue;
4978
+ const worker = sessionWorkerRegistry?.getWorker?.(key) || sessionWorkerRegistry?.getWorkerByConversationId?.(key) || null;
4979
+ const conversationId = String(worker?.conversationId || key);
4980
+ hostSuspendWatchedConversationIds.add(conversationId);
4981
+ const what = tasks.slice(0, 3).map((task) => String(task?.description || task?.taskType || 'task').slice(0, 80)).join('; ');
4982
+ blockers.push({
4983
+ kind: 'background',
4984
+ conversationId,
4985
+ title: hostActivityTitle(conversationId),
4986
+ count: tasks.length,
4987
+ detail: `${tasks.length} background agent${tasks.length === 1 ? '' : 's'}: ${what}`,
4988
+ });
4989
+ }
4990
+ const runningSubagents = Number(stmts.countRunningSubagentRuns?.get?.()?.cnt || 0);
4991
+ // CI is only worth a network round-trip for workspaces that were busy since
4992
+ // the request (or, for the modal's peek, that are busy right now).
4993
+ const roots = [...hostSuspendWatchedConversationIds]
4994
+ .map((conversationId) => {
4995
+ try { return resolveConversationWorkspaceState({ conversationId })?.currentWorkspaceRootPath || ''; } catch { return ''; }
4996
+ })
4997
+ .filter(Boolean);
4998
+ if (roots.length) void githubCiActivityService.describeRuns(roots).catch(() => {});
4999
+ const ci = roots.length ? githubCiActivityService.snapshot(roots) : { repos: [], unknown: [], blockers: [] };
5000
+ blockers.push(...ci.blockers);
5001
+ if (!request && !hostSuspendService.getState().pending) hostSuspendWatchedConversationIds.clear();
5002
+ return { blockers, runningSubagents, ci };
5003
+ }
5004
+ const hostSuspendService = createHostSuspendService({
5005
+ collectActivity: collectHostActivity,
5006
+ runSuspend: () => runHostSuspendToRam({ logger: console }),
5007
+ logger: console,
5008
+ onStateChange: (state) => {
5009
+ if (state.status === 'idle') hostSuspendWatchedConversationIds.clear();
5010
+ try { io.emit('host_suspend_state', state); } catch {}
5011
+ },
5012
+ onSuspended: () => {
5013
+ void pushDispatchService.notifyHostSuspend({ body: 'All agents finished; the host is suspending now.' });
5014
+ },
5015
+ onDropped: (state, reason) => {
5016
+ void pushDispatchService.notifyHostSuspend({ body: `Queued host suspend dropped: relay ${reason || 'restarting'}.` });
5017
+ },
5018
+ });
5019
+
5020
+ // The live turn among a conversation's processing rows. Mid-turn steering can
5021
+ // leave several processing at once — the running turn, steered messages the
5022
+ // CLI folded into it (no output of their own), a handed-off row still settling
5023
+ // — so the live turn is the row that produced output most recently; with
5024
+ // nothing to separate them yet, the oldest processing row (the turn a steered
5025
+ // message joins) wins. See live-turn-picker.mjs for why recency, not per-row
5026
+ // stream seq. Returns the whole row so callers get relay_mode/timestamps too.
5027
+ function resolveLiveTurnQueueRow(conversationId) {
5028
+ const rows = stmts.listProcessingQueueByConversation?.all(conversationId)
5029
+ || (stmts.getLatestProcessingQueueByConversation.get(conversationId)
5030
+ ? [stmts.getLatestProcessingQueueByConversation.get(conversationId)]
5031
+ : []);
5032
+ if (!rows.length) return null;
5033
+ if (rows.length === 1) return rows[0];
5034
+ const byId = new Map(rows.map((row) => [String(row.id), row]));
5035
+ const liveId = pickLiveTurnRowId(rows.map((row) => {
5036
+ const output = stmts.getLastOutputAtByQueueMessage?.get({ id: row.id }) || null;
5037
+ return {
5038
+ id: String(row.id),
5039
+ lastOutputAtMs: Date.parse(String(output?.last_output_at || '')) || 0,
5040
+ streamDone: Number(output?.stream_done || 0) === 1,
5041
+ processingAtMs: Date.parse(String(row.processing_at || row.timestamp || '')) || 0,
5042
+ };
5043
+ }));
5044
+ return byId.get(liveId) || rows[0];
5045
+ }
5046
+
4883
5047
  function inFlightStateForConversation(conversationId) {
4884
- const row = stmts.getLatestProcessingQueueByConversation.get(conversationId);
5048
+ const row = resolveLiveTurnQueueRow(conversationId);
4885
5049
  if (!row) return null;
4886
5050
  const streamEvents = relayStreamEventsForQueueMessage(row.id);
4887
5051
  const lastStreamEvent = streamEvents.length ? streamEvents[streamEvents.length - 1] : null;
@@ -4906,17 +5070,17 @@ function inFlightStateForConversation(conversationId) {
4906
5070
  // retry_count like /api/requeue does, but bypasses that route's budget check,
4907
5071
  // so the sweep enforces it here). Minimal mirror of the route-side
4908
5072
  // failQueueMessage: mark failed, insert the assistant failure message, emit.
4909
- function failRecoveredRowTerminally(row, reason) {
5073
+ function failRecoveredRowTerminally(row, reason, { failureRecord = null, failureText: presetText = null } = {}) {
4910
5074
  const now = new Date().toISOString();
4911
5075
  const responseId = uuidv4();
4912
5076
  const retryCount = Number(row?.retry_count || 0) + 1;
4913
- const failureText = `Relay recovery limit reached after ${retryCount} attempts (${reason}). The worker kept dying before completing this turn, so the message was failed to keep the queue moving. Send it again to retry.`;
5077
+ const failureText = presetText || `Relay recovery limit reached after ${retryCount} attempts (${reason}). The worker kept dying before completing this turn, so the message was failed to keep the queue moving. Send it again to retry.`;
4914
5078
  const requestedModel = String(row?.model || '').trim() || null;
4915
5079
  const tx = db.transaction(() => {
4916
5080
  // setFailed is compare-and-set on status (processing/pending/parked): a
4917
5081
  // row that settled between the sweep's listing and this write must not
4918
5082
  // grow an orphan failure bubble on top of its real outcome.
4919
- const result = stmts.setFailed.run(JSON.stringify({
5083
+ const result = stmts.setFailed.run(JSON.stringify(failureRecord || {
4920
5084
  kind: 'recovery-limit',
4921
5085
  error: 'recovery-limit',
4922
5086
  code: 'recovery-limit',
@@ -4940,6 +5104,7 @@ function failRecoveredRowTerminally(row, reason) {
4940
5104
  null,
4941
5105
  requestedModel ? 'user' : 'auto',
4942
5106
  );
5107
+ stmts.setMessageSourceId?.run(row.id, responseId);
4943
5108
  stmts.linkActivityToResponse?.run(responseId, row.id);
4944
5109
  stmts.linkStreamEventsToResponse?.run(responseId, row.id);
4945
5110
  stmts.linkThoughtsToResponse?.run(responseId, row.id);
@@ -4954,7 +5119,20 @@ function failRecoveredRowTerminally(row, reason) {
4954
5119
  message: { role: 'assistant', text: failureText, model: requestedModel || 'unknown', mode: String(row?.relay_mode || '').trim() || null, timestamp: now },
4955
5120
  });
4956
5121
  io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'failed' });
4957
- console.warn(`${runtimeLogPrefix()}RECOVERY LIMIT ${String(row.id).slice(0, 8)} conv=${String(row.conversation_id).slice(0, 8)} retries=${retryCount} reason=${reason}`);
5122
+ console.warn(`${runtimeLogPrefix()}${failureRecord ? 'CONSUMED FAILED' : 'RECOVERY LIMIT'} ${String(row.id).slice(0, 8)} conv=${String(row.conversation_id).slice(0, 8)} retries=${retryCount} reason=${reason}`);
5123
+ }
5124
+
5125
+ // A row whose prompt the worker's runtime already consumed (queue.consumed_at)
5126
+ // is at-most-once: recovery must never requeue it — a re-delivery would run the
5127
+ // prompt twice — so it fails terminally with the steer-settle wording, naming
5128
+ // the provider's runtime the way the workers' own settle wording does.
5129
+ function failConsumedRowTerminally(row, reason) {
5130
+ const providerType = stmts.getRuntimeSessionByConversation?.get?.(row?.conversation_id)?.provider_type;
5131
+ const failureRecord = buildSteerSettleFailure(row, { agentLabel: steerAgentLabelForProvider(providerType) });
5132
+ failRecoveredRowTerminally(row, reason, {
5133
+ failureRecord: { ...failureRecord, error: 'terminal-error', reason: String(reason || 'recovery') },
5134
+ failureText: `${failureRecord.message} Error code: ${failureRecord.stableCode}. ${failureRecord.guidance}`,
5135
+ });
4958
5136
  }
4959
5137
 
4960
5138
  function recoverProcessingOlderThan(cutoffIso, requeueAtIso, { ceilingBeforeIso = null } = {}) {
@@ -4966,7 +5144,9 @@ function recoverProcessingOlderThan(cutoffIso, requeueAtIso, { ceilingBeforeIso
4966
5144
  stmts.dropStaleContinuation?.run?.(row.id);
4967
5145
  io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'failed' });
4968
5146
  }
4969
- const allRows = stmts.listRecoverableProcessing.all(params);
5147
+ const listedRows = stmts.listRecoverableProcessing.all(params);
5148
+ for (const row of listedRows.filter((entry) => entry?.consumed_at)) failConsumedRowTerminally(row, 'stale-recovery');
5149
+ const allRows = listedRows.filter((entry) => !entry?.consumed_at);
4970
5150
  if (!allRows.length) return [];
4971
5151
  // Enforce the retry budget before requeueing: failing these rows first
4972
5152
  // flips them out of 'processing' so the recovery UPDATE below skips them.
@@ -5345,6 +5525,13 @@ const app = express();
5345
5525
  // via config.trustProxy (e.g. a hop count or subnet).
5346
5526
  app.set('trust proxy', config.trustProxy ?? 'loopback');
5347
5527
  const httpServer = http.createServer(app);
5528
+ // Node closes an idle keep-alive connection after 5 s. A client that pools
5529
+ // connections (a fronting proxy such as Caddy or cloudflared, an API client)
5530
+ // can pick such a socket in the very moment the relay closes it; its request
5531
+ // is then reset before the relay reads a byte, and a POST is not replayed.
5532
+ // Holding idle connections longer than those pools do closes that window.
5533
+ httpServer.keepAliveTimeout = 65_000;
5534
+ httpServer.headersTimeout = 66_000;
5348
5535
  httpServer.prependListener('request', (req, _res) => {
5349
5536
  rewriteSocketIoRequestPath(req, remotePath);
5350
5537
  });
@@ -5772,6 +5959,10 @@ function recoverDeadWorkerProcessingRows(sessionId, reason) {
5772
5959
  io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'failed' });
5773
5960
  continue;
5774
5961
  }
5962
+ if (row.consumed_at) {
5963
+ failConsumedRowTerminally(row, `worker-died-${reason}`);
5964
+ continue;
5965
+ }
5775
5966
  if (Number(row.retry_count || 0) + 1 >= MAX_REQUEUE_RETRIES) {
5776
5967
  failRecoveredRowTerminally(row, `worker-died-${reason}`);
5777
5968
  continue;
@@ -5814,11 +6005,15 @@ function handleWorkerSocketClosed({ sessionId, pid = null } = {}) {
5814
6005
  const sid = String(sessionId || '').trim();
5815
6006
  if (!sid) return;
5816
6007
  const owned = stmts.listProcessingRowsForOwner?.all?.(sid) || [];
5817
- if (!owned.length) return;
6008
+ // A dead worker's steering snapshot must not keep holding the composer even
6009
+ // when it owned no rows (the hold can outlive them, e.g. a continuation's).
6010
+ const heldSteering = Boolean(sessionWorkerRegistry?.getWorker?.(sid)?.steering);
6011
+ if (!owned.length && !heldSteering) return;
5818
6012
  const timer = setTimeout(() => {
5819
6013
  try {
5820
6014
  if (runtimeShutdownStarted) return;
5821
6015
  if (isWorkerSessionProcessAlive(sid, pid)) return; // reconnect or socket flap
6016
+ sessionWorkerRegistry?.clearSteering?.(sid);
5822
6017
  const stillOwned = stmts.listProcessingRowsForOwner?.all?.(sid) || [];
5823
6018
  if (!stillOwned.length) return;
5824
6019
  console.warn(`${runtimeLogPrefix()}WORKER DIED session=${sid.slice(0, 8)} rows=${stillOwned.length} trigger=socket-close — recovering owned turns`);
@@ -5986,6 +6181,7 @@ const runtimeState = {
5986
6181
  set relayPaused(value) { relayPaused = value; },
5987
6182
  get ttyConsoleActive() { return Boolean(ttyConsoleRuntime?.tty); },
5988
6183
  get relayShutdown() { return getRelayShutdownState(); },
6184
+ get hostSuspend() { return hostSuspendService.getState(); },
5989
6185
  get activeBridgeOwner() { return relayBridgeOwnerService.getOwner(); },
5990
6186
  get workerWebSocketStatus() { return sessionWorkerWebSocketService.status(); },
5991
6187
  get tmuxInspectorStatus() { return tmuxInspectorSocketService.status(); },
@@ -6002,6 +6198,82 @@ const windowsBootAutostartService = createWindowsBootAutostartService({
6002
6198
  });
6003
6199
 
6004
6200
  const gitChangesService = createGitChangesService();
6201
+
6202
+ // ─── Remote Relays ────────────────────────────────────────────────────────────
6203
+ // Other OAR relays this one is paired with, so its agents can work there. Built
6204
+ // before the shared route deps: the message, session and question routes read
6205
+ // the registry for mentions, unlocks and the inbound switch.
6206
+ // Null when migration 0005 failed this boot (db-schema retries it next boot):
6207
+ // remotes still list and pair, nothing unlocks until then.
6208
+ const remoteRelayRepository = (() => {
6209
+ try {
6210
+ return createRemoteRelayRepository(db);
6211
+ } catch (error) {
6212
+ console.warn(`[remote-relays] unlock storage unavailable: ${error?.message || error}`);
6213
+ return null;
6214
+ }
6215
+ })();
6216
+ const remoteRelayClient = createRemoteRelayClient({
6217
+ // A closure: the registry owning the instance id is declared just below.
6218
+ getOwnRelayId: () => remoteRelayRegistry.instanceId(),
6219
+ getOwnToken: () => config.authToken,
6220
+ });
6221
+ const remoteRelayRegistry = createRemoteRelayRegistry({
6222
+ readSetting: readAppSettingValue,
6223
+ writeSetting: (key, value) => stmts.upsertAppSetting.run(key, value, new Date().toISOString()),
6224
+ client: remoteRelayClient,
6225
+ repository: remoteRelayRepository,
6226
+ getOwnToken: () => config.authToken,
6227
+ getSelfName: getPwaAppName,
6228
+ hostname: os.hostname(),
6229
+ platform: process.platform,
6230
+ version: RUNNING_VERSION,
6231
+ emit: (event, payload) => io.emit(event, payload),
6232
+ logger: console,
6233
+ });
6234
+ const remoteRelayPairing = createRemoteRelayPairing({
6235
+ registry: remoteRelayRegistry,
6236
+ client: remoteRelayClient,
6237
+ getOwnToken: () => config.authToken,
6238
+ });
6239
+ // What the inbound routes need: no tokens, only ids, names and state.
6240
+ const remoteRelayInbound = {
6241
+ listRelays: () => remoteRelayRegistry.list().map(({ id, name, url, lastStatus, version }) => ({ id, name, url, lastStatus, version })),
6242
+ selfNames: () => [remoteRelayRegistry.selfName()],
6243
+ inboundEnabled: () => remoteRelayRegistry.getSelfSettings().inboundEnabled,
6244
+ recordUnlock: (conversationId, relayId, messageId) => remoteRelayRepository?.recordUnlock(conversationId, relayId, messageId) ?? false,
6245
+ describeRelay: (id) => {
6246
+ const relay = remoteRelayRegistry.get(id);
6247
+ return relay ? { id: relay.id, name: relay.name, online: relay.lastStatus === 'online', version: relay.version } : null;
6248
+ },
6249
+ repository: remoteRelayRepository,
6250
+ };
6251
+ // The remote_relay tool: who is calling (live turn, provider, model, mode,
6252
+ // hops) and the Allow / Deny card write actions need in ask/plan mode.
6253
+ const remoteRelayCallerContext = createRemoteRelayCallerContext({
6254
+ db,
6255
+ repository: remoteRelayRepository,
6256
+ questions: stmts,
6257
+ resolveLiveTurnQueueRow,
6258
+ formatQuestionRow,
6259
+ normalizeRelayMode,
6260
+ defaultRelayMode: DEFAULT_RELAY_MODE,
6261
+ emit: (event, payload) => io.emit(event, payload),
6262
+ notifyQuestion: (question) => pushDispatchService.notifyQuestion?.(question),
6263
+ uuid: uuidv4,
6264
+ logger: console,
6265
+ });
6266
+ const remoteRelayDispatcher = createRemoteRelayDispatcher({
6267
+ registry: remoteRelayRegistry,
6268
+ client: remoteRelayClient,
6269
+ repository: remoteRelayRepository,
6270
+ getCallerContext: (conversationId) => remoteRelayCallerContext.getCallerContext(conversationId),
6271
+ requestApproval: (request) => remoteRelayCallerContext.requestApproval(request),
6272
+ emitStatusEvent: ({ type, ...details }) => statusEventService.recordEvent(type, details),
6273
+ resolveConversationId: (id) => resolveConversationRecord({ conversationId: id, sdkSessionId: id })?.id || id,
6274
+ logger: console,
6275
+ });
6276
+
6005
6277
  const sharedRouteDeps = {
6006
6278
  auth,
6007
6279
  gitChangesService,
@@ -6113,6 +6385,7 @@ const sharedRouteDeps = {
6113
6385
  previewCardsForResponse,
6114
6386
  sanitizeActivityText,
6115
6387
  inFlightStateForConversation,
6388
+ resolveLiveTurnQueueRow,
6116
6389
  backgroundTaskStore,
6117
6390
  // A closure, not the registry itself: this deps object is built before the
6118
6391
  // preview lane is constructed further down the file.
@@ -6184,6 +6457,10 @@ const sharedRouteDeps = {
6184
6457
  getPwaAppName,
6185
6458
  setPwaAppName,
6186
6459
  requestRelayShutdown,
6460
+ cancelRelayShutdown,
6461
+ relayShutdownStatePayload,
6462
+ hostSuspendService,
6463
+ collectHostActivity,
6187
6464
  markSharedViewerPresence,
6188
6465
  getSharedWatcherCount,
6189
6466
  statusEventService,
@@ -6198,6 +6475,7 @@ const sharedRouteDeps = {
6198
6475
  windowsAutostartService,
6199
6476
  windowsBootAutostartService,
6200
6477
  pushDispatchService,
6478
+ remoteRelayInbound,
6201
6479
  };
6202
6480
  registerMessagesRoutes(app, sharedRouteDeps);
6203
6481
  registerSessionsRoutes(app, sharedRouteDeps);
@@ -6210,6 +6488,12 @@ registerPushRoutes(app, {
6210
6488
  pushDispatchService,
6211
6489
  getPushVapidPublicKey: () => pushVapidKeys.publicKey,
6212
6490
  });
6491
+ registerRemoteRelayRoutes(app, {
6492
+ auth,
6493
+ registry: remoteRelayRegistry,
6494
+ pairing: remoteRelayPairing,
6495
+ dispatcher: remoteRelayDispatcher,
6496
+ });
6213
6497
  function markCliOffline(reason = 'offline', { clearOwner = true } = {}) {
6214
6498
  cliLastSeen = null;
6215
6499
  const wasOnline = cliOnline;
@@ -6523,25 +6807,10 @@ function sanitizeRelayQuestionRequest(requestBody) {
6523
6807
  }
6524
6808
  }
6525
6809
 
6810
+ // The allow-list lives in services/relay-question-context.mjs (unit-tested);
6811
+ // this binds it to the relay's own mode normalizer.
6526
6812
  function sanitizeRelayQuestionContext(rawContext) {
6527
- if (!rawContext || typeof rawContext !== 'object' || Array.isArray(rawContext)) return null;
6528
- const context = {};
6529
- if (typeof rawContext.source === 'string' && rawContext.source.trim()) {
6530
- context.source = rawContext.source.trim().slice(0, 64);
6531
- }
6532
- if (typeof rawContext.rationale === 'string' && rawContext.rationale.trim()) {
6533
- context.rationale = rawContext.rationale.trim().slice(0, 240);
6534
- }
6535
- if (typeof rawContext.queueMessageId === 'string' && rawContext.queueMessageId.trim()) {
6536
- context.queueMessageId = rawContext.queueMessageId.trim();
6537
- }
6538
- if (typeof rawContext.conversationId === 'string' && rawContext.conversationId.trim()) {
6539
- context.conversationId = rawContext.conversationId.trim();
6540
- }
6541
- if (typeof rawContext.relayMode === 'string' && rawContext.relayMode.trim()) {
6542
- context.relayMode = normalizeRelayMode(rawContext.relayMode) || DEFAULT_RELAY_MODE;
6543
- }
6544
- return Object.keys(context).length ? context : null;
6813
+ return sanitizeRelayQuestionContextWith(rawContext, { normalizeRelayMode, defaultRelayMode: DEFAULT_RELAY_MODE });
6545
6814
  }
6546
6815
 
6547
6816
 
@@ -6579,6 +6848,9 @@ io.on('connection', (socket) => {
6579
6848
  });
6580
6849
  // Send current CLI status immediately on connect
6581
6850
  socket.emit('cli_status', { online: cliOnline });
6851
+ // Pending host suspend / relay restart banners must survive a reconnect.
6852
+ try { socket.emit('host_suspend_state', hostSuspendService.getState()); } catch {}
6853
+ try { socket.emit('relay_shutdown_state', relayShutdownStatePayload()); } catch {}
6582
6854
  // Visibility heartbeats for push suppression; also clears a stale
6583
6855
  // deviceVisible flag restored by connectionStateRecovery.
6584
6856
  activeDeviceTracker.registerSocket(socket);
@@ -6713,6 +6985,9 @@ function shutdownRuntime(reason = 'unknown', { exitCode = 0 } = {}) {
6713
6985
  runtimeShutdownStarted = true;
6714
6986
  console.log(`${runtimeLogPrefix()}Runtime shutdown started (${reason}, exitCode=${runtimeShutdownExitCode})`);
6715
6987
  clearRuntimeTimers();
6988
+ // A queued host suspend is in-memory only; drop it loudly rather than let
6989
+ // the next runtime forget about it in silence.
6990
+ try { hostSuspendService.dispose({ reason }); } catch {}
6716
6991
  try { updateCheckService?.stop(); } catch {}
6717
6992
  // Started here (flips the importer's closing flag synchronously) but awaited
6718
6993
  // below before the transports close: fire-and-forget disposal let a
@@ -6741,6 +7016,7 @@ function shutdownRuntime(reason = 'unknown', { exitCode = 0 } = {}) {
6741
7016
  sshTunnelManager.stop();
6742
7017
  cloudflaredTunnelManager.stop();
6743
7018
  previewHealthProbe.stop();
7019
+ try { remoteRelayRegistry.stopHealthLoop(); } catch {}
6744
7020
  // Previews are in-memory by design: a restart is the only implicit cleanup,
6745
7021
  // so every public link dies with the process.
6746
7022
  previewRegistry.clear();
@@ -6873,6 +7149,8 @@ httpServer.listen(config.port, listenHost, () => {
6873
7149
  // Start SSH tunnel after server is listening
6874
7150
  sshTunnelManager.start();
6875
7151
  cloudflaredTunnelManager.start();
7152
+ // Status, version and renames of paired relays; a first pass right away.
7153
+ remoteRelayRegistry.startHealthLoop();
6876
7154
  // A preview-lane failure must never take the relay with it: the listener is
6877
7155
  // optional infrastructure, so a bind error is logged and the relay carries on
6878
7156
  // without previews.