@oar-sh/oar 0.9.12 → 0.9.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/.github/extensions/web-relay/extension.mjs +9 -0
  2. package/.github/extensions/web-relay/server-lifecycle/managed-server.mjs +15 -1
  3. package/.github/extensions/web-relay/skills/question-bridge.mjs +4 -1
  4. package/CHANGELOG.md +508 -0
  5. package/README.md +12 -12
  6. package/bin/oar.js +155 -94
  7. package/package.json +5 -5
  8. package/server/README.md +479 -157
  9. package/server/claude-cloud-worker/claude-cloud-session-process.mjs +10 -4
  10. package/server/claude-worker/claude-session-process.mjs +92 -15
  11. package/server/claude-worker/claude-turn-publisher.mjs +20 -1
  12. package/server/claude-worker/subagent-timeline-recorder.mjs +75 -8
  13. package/server/claude-worker/task-output-discovery.mjs +28 -2
  14. package/server/copilot-worker/copilot-ask-user-tool.mjs +1 -1
  15. package/server/copilot-worker/copilot-continuation-signals.mjs +11 -5
  16. package/server/copilot-worker/copilot-prompt-context.mjs +3 -1
  17. package/server/copilot-worker/copilot-question-bridge.mjs +11 -6
  18. package/server/copilot-worker/copilot-sdk-adapter.mjs +11 -0
  19. package/server/copilot-worker/copilot-sdk-session-process.mjs +24 -1
  20. package/server/cursor-worker/cursor-ask-user-tool.mjs +1 -1
  21. package/server/cursor-worker/cursor-sdk-adapter.mjs +22 -7
  22. package/server/cursor-worker/cursor-session-worker.mjs +9 -2
  23. package/server/cursor-worker/cursor-turn-runner.mjs +18 -3
  24. package/server/db-schema.mjs +10 -0
  25. package/server/grok-worker/acp-host-services.mjs +6 -3
  26. package/server/grok-worker/grok-background-tasks.mjs +7 -0
  27. package/server/grok-worker/grok-sdk-adapter.mjs +14 -0
  28. package/server/grok-worker/grok-turn-runner.mjs +13 -3
  29. package/server/migrations/0018-background-task-rows.mjs +101 -0
  30. package/server/public/app/action-confirmations.js +12 -15
  31. package/server/public/app/api-client.js +60 -4
  32. package/server/public/app/ask-user-view.js +29 -4
  33. package/server/public/app/background-tasks-view.mjs +56 -7
  34. package/server/public/app/bootstrap.js +7 -0
  35. package/server/public/app/conversation-preferences.mjs +10 -4
  36. package/server/public/app/conversation-provider-indicator.mjs +18 -8
  37. package/server/public/app/conversation-view.js +33 -9
  38. package/server/public/app/host-suspend-ui.js +45 -44
  39. package/server/public/app/journal-view.js +9 -3
  40. package/server/public/app/push-settings.js +1 -0
  41. package/server/public/app/question-answer-rule.mjs +15 -0
  42. package/server/public/app/question-multi-select.mjs +9 -2
  43. package/server/public/app/relay-event-view.mjs +5 -1
  44. package/server/public/app/relay-prompt-sanitizer.mjs +9 -2
  45. package/server/public/app/relay-restart-note-view.mjs +36 -0
  46. package/server/public/app/relay-restart-ui.js +568 -0
  47. package/server/public/app/settings-modal.js +52 -0
  48. package/server/public/app/socket-handlers.js +1 -1
  49. package/server/public/app/task-console-render.mjs +171 -7
  50. package/server/public/app/task-console.mjs +111 -11
  51. package/server/public/app/task-output-text.mjs +15 -6
  52. package/server/public/app/transcript-jump-bottom-view.mjs +5 -1
  53. package/server/public/app/transcript-jump-bottom.mjs +20 -0
  54. package/server/public/index.html +76 -5
  55. package/server/relay-exit-codes.mjs +6 -0
  56. package/server/relay-self-restart.mjs +6 -1
  57. package/server/relay-tools.md +17 -9
  58. package/server/repositories/message-repository.mjs +32 -28
  59. package/server/repositories/remote-relay-repository.mjs +4 -2
  60. package/server/repositories/subagent-timeline-statements.mjs +2 -0
  61. package/server/routes/ask-user-routes.mjs +41 -7
  62. package/server/routes/messages-routes.mjs +220 -37
  63. package/server/routes/preview-routes.mjs +73 -7
  64. package/server/routes/relay-board-routes.mjs +19 -2
  65. package/server/routes/relay-restart-routes.mjs +73 -0
  66. package/server/routes/sessions-routes.mjs +79 -5
  67. package/server/server-runtime.mjs +335 -218
  68. package/server/services/about-service.mjs +13 -12
  69. package/server/services/background-task-persistence.mjs +133 -0
  70. package/server/services/background-task-store.mjs +240 -13
  71. package/server/services/conversation-model-fallback.mjs +57 -0
  72. package/server/services/cursor-session-root-service.mjs +70 -6
  73. package/server/services/data-dir-resolution.mjs +208 -19
  74. package/server/services/earlier-database-service.mjs +631 -0
  75. package/server/services/host-suspend-command.mjs +2 -1
  76. package/server/services/oar-cli-helpers.mjs +6 -1
  77. package/server/services/oar-state-layout.mjs +107 -0
  78. package/server/services/oar-state-migration-service.mjs +387 -96
  79. package/server/services/package-data-rescue-preinstall.mjs +22 -0
  80. package/server/services/package-data-rescue.mjs +137 -0
  81. package/server/services/preview-proxy-server.mjs +14 -1
  82. package/server/services/preview-registry-service.mjs +3 -2
  83. package/server/services/process-list-service.mjs +2 -1
  84. package/server/services/provider-model-selection.mjs +195 -0
  85. package/server/services/push-dispatch-service.mjs +42 -8
  86. package/server/services/relay-event-service.mjs +210 -14
  87. package/server/services/relay-prompt-sanitizer.mjs +7 -0
  88. package/server/services/relay-restart-picture.mjs +209 -0
  89. package/server/services/relay-restart-queue.mjs +387 -0
  90. package/server/services/relay-shutdown-gate.mjs +4 -1
  91. package/server/services/relay-singleton-guard.mjs +300 -36
  92. package/server/services/remote-relay-caller-context.mjs +7 -2
  93. package/server/services/remote-relay-dispatcher.mjs +217 -28
  94. package/server/services/restart-resume-service.mjs +303 -0
  95. package/server/services/session-activity-tracker.mjs +52 -15
  96. package/server/services/session-worker-launch-service.mjs +31 -1
  97. package/server/services/session-worker-process-service.mjs +6 -4
  98. package/server/services/stale-turn-recovery.mjs +99 -0
  99. package/server/services/subagent-timeline-service.mjs +4 -1
  100. package/server/services/task-output-service.mjs +217 -18
  101. package/server/services/windows-autostart-service.mjs +329 -31
  102. package/server/services/windows-boot-autostart-service.mjs +188 -60
  103. package/shared/preview-instructions.mjs +17 -12
  104. package/shared/preview-tool-core.mjs +32 -14
  105. package/shared/question-answer-rule.mjs +63 -0
  106. package/shared/relay-events.mjs +106 -3
  107. package/shared/remote-relay-contract.mjs +32 -15
  108. package/shared/subagent-timeline.mjs +4 -0
  109. package/shared/task-output-text.mjs +196 -83
  110. package/shared/windows-system-tools.mjs +25 -0
  111. package/shared/worker-runtime/config-loader.mjs +54 -8
  112. package/shared/worker-runtime/prompt-context.mjs +3 -1
  113. package/shared/worker-runtime/worker-websocket-link.mjs +96 -20
  114. package/shared/worker-runtime/working-directory.mjs +65 -0
  115. package/server/uploads/image_input.md +0 -540
@@ -42,7 +42,7 @@ import { registerRelayBoardRoutes } from './routes/relay-board-routes.mjs';
42
42
  import { registerCacheRoutes } from './routes/cache-routes.mjs';
43
43
  import { registerGitRoutes } from './routes/git-routes.mjs';
44
44
  import { registerClaudeCloudRoutes } from './routes/claude-cloud-routes.mjs';
45
- import { registerPreviewRoutes } from './routes/preview-routes.mjs';
45
+ import { createPreviewScopeResolver, registerPreviewRoutes } from './routes/preview-routes.mjs';
46
46
  import { registerRemoteRelayRoutes } from './routes/remote-relay-routes.mjs';
47
47
  import { registerRelayEventRoutes } from './routes/relay-event-routes.mjs';
48
48
  import { createRemoteRelayRepository } from './repositories/remote-relay-repository.mjs';
@@ -75,6 +75,13 @@ import { createHostSuspendService } from './services/host-suspend-service.mjs';
75
75
  import { USAGE_LIMIT_ACCOUNT_SOCKET_EVENT, createUsageLimitPauseService } from './services/usage-limit-pause-service.mjs';
76
76
  import { createSessionActivityTracker, resolveBootReportingWindowMs } from './services/session-activity-tracker.mjs';
77
77
  import { createBackgroundTaskStore } from './services/background-task-store.mjs';
78
+ import { createSqliteBackgroundTaskPersistence } from './services/background-task-persistence.mjs';
79
+ import {
80
+ RESTART_RESUME_SETTING_KEY,
81
+ createRestartResumeService,
82
+ parseRestartResumeUpdate,
83
+ readRestartResumeSetting,
84
+ } from './services/restart-resume-service.mjs';
78
85
  import { createBackgroundWorkView } from './services/background-work-view.mjs';
79
86
  import { createRelayEventService } from './services/relay-event-service.mjs';
80
87
  import { createRelayEventLinkService } from './services/relay-event-link-service.mjs';
@@ -90,6 +97,7 @@ import { runHostSuspendToRam } from './services/host-suspend-command.mjs';
90
97
  import { createActiveDeviceTracker } from './services/push-active-device-service.mjs';
91
98
  import { registerPushRoutes } from './routes/push-routes.mjs';
92
99
  import { registerAboutRoutes } from './routes/about-routes.mjs';
100
+ import { registerRelayRestartRoutes } from './routes/relay-restart-routes.mjs';
93
101
  import { createAboutService, resolveBuildCommit } from './services/about-service.mjs';
94
102
  import { createImageOperationService } from './services/image-operation-service.mjs';
95
103
  import {
@@ -116,7 +124,9 @@ import { createClaudeSessionRootResolver } from './services/claude-session-root-
116
124
  import { createCursorSessionRootResolver } from './services/cursor-session-root-service.mjs';
117
125
  import { createGitChangesService } from './services/git-changes-service.mjs';
118
126
  import { createRelaySingletonGuard } from './services/relay-singleton-guard.mjs';
119
- import { resolveDataDir } from './services/data-dir-resolution.mjs';
127
+ import { adoptEarlierDatabase, lostPackageDatabaseWarning, packageDatabaseEvidence } from './services/earlier-database-service.mjs';
128
+ import { ensureDataDirs, resolveDataDir, resolveUploadDir, resolveUploadReadDirs, uploadPathInDirs } from './services/data-dir-resolution.mjs';
129
+ import { mergeStrayMigratedUploads } from './services/oar-state-migration-service.mjs';
120
130
  import { createRelayRestartOrchestrator } from './services/relay-restart-orchestrator-service.mjs';
121
131
  import { createRelayBridgeOwnerService } from './services/relay-bridge-owner-service.mjs';
122
132
  import { createRelayCliLauncherService } from './services/relay-cli-launcher-service.mjs';
@@ -165,7 +175,7 @@ import {
165
175
  migrateLegacyConfigFeatures,
166
176
  resolveBootFeatureFlags,
167
177
  } from './features.mjs';
168
- import { RELAY_RESTART_EXIT_CODE } from './relay-exit-codes.mjs';
178
+ import { RELAY_RESTART_EXIT_CODE, RELAY_STOP_EXIT_CODE } from './relay-exit-codes.mjs';
169
179
  import { DEFAULT_QUESTION_TIMEOUT_MS, questionExpiresAt } from '../shared/question-timeout.mjs';
170
180
  import {
171
181
  parseTurnCeilingUpdate,
@@ -183,8 +193,17 @@ import {
183
193
  parseSharePathPrivacyUpdate,
184
194
  readSharePathPrivacySetting,
185
195
  } from '../shared/share-path-privacy.mjs';
186
- import { isRelayShutdownQueued, queuedShutdownCanProceed, relayShutdownGateReply } from './services/relay-shutdown-gate.mjs';
196
+ import { isRelayShutdownQueued, relayShutdownGateReply } from './services/relay-shutdown-gate.mjs';
197
+ import {
198
+ createRelayRestartQueue,
199
+ parseRelayRestartMaxWaitUpdate,
200
+ readRelayRestartMaxWaitSetting,
201
+ resolveRelayRestartMaxWaitOverrideMs,
202
+ RELAY_RESTART_MAX_WAIT_SETTING_KEY,
203
+ } from './services/relay-restart-queue.mjs';
204
+ import { createRestartPictureCollector } from './services/relay-restart-picture.mjs';
187
205
  import { createStaleRecoveryClockGuard } from './services/stale-recovery-clock-guard.mjs';
206
+ import { createStaleTurnRecovery } from './services/stale-turn-recovery.mjs';
188
207
  import {
189
208
  STALE_SUBAGENT_RUN_BOOT_DELAY_MS,
190
209
  STALE_SUBAGENT_RUN_SWEEP_INTERVAL_MS,
@@ -232,7 +251,7 @@ import {
232
251
  import { createModelVariantCatalogService } from './services/model-variant-catalog-service.mjs';
233
252
  import { applyPwaManifestTheme, normalizePwaTheme } from './services/pwa-manifest-theme.mjs';
234
253
  import { mapSubagentRunHeader } from './services/subagent-timeline-service.mjs';
235
- import { describeExecError, describeSpawnEnvironment, windowsSystemToolPath } from '../shared/windows-system-tools.mjs';
254
+ import { describeExecError, describeSpawnEnvironment, normalizeTempEnv, windowsSystemToolPath } from '../shared/windows-system-tools.mjs';
236
255
 
237
256
  const __dirname = path.dirname(fileURLToPath(import.meta.url));
238
257
  const REPO_ROOT = path.resolve(__dirname, '..');
@@ -263,9 +282,8 @@ const APP_CONFIG_PLACEHOLDER = /window\.__COPILOT_APP_CONFIG = \{ basePath: '[^'
263
282
  const PWA_VERSION_PLACEHOLDER = /const __PWA_VERSION = '[^']*';/;
264
283
  // The console goes to a terminal, a tmux pane or nowhere: keep a copy to read
265
284
  // after a restart or an incident.
266
- const relayConsoleLog = installRelayConsoleLog({
267
- logPath: resolveRelayConsoleLogPath({ env: process.env, serverDir: __dirname }),
268
- });
285
+ const relayConsoleLogPath = resolveRelayConsoleLogPath({ env: process.env, serverDir: __dirname });
286
+ const relayConsoleLog = installRelayConsoleLog({ logPath: relayConsoleLogPath });
269
287
  const ttyConsoleRuntime = await maybeStartTtyConsole({
270
288
  serverDir: __dirname,
271
289
  logsDir: path.join(__dirname, 'logs'),
@@ -273,6 +291,8 @@ const ttyConsoleRuntime = await maybeStartTtyConsole({
273
291
  });
274
292
  // The terminal console takes the console's methods over when it starts.
275
293
  relayConsoleLog?.rewrap();
294
+ // TEMP and TMP for this relay and everything it starts (normalizeTempEnv).
295
+ for (const change of normalizeTempEnv()) console.log(`[startup] ${change} (quotes taken off, or the folder is not there)`);
276
296
 
277
297
  // ─── Config ────────────────────────────────────────────────────────────────────
278
298
  const CONFIG_PATH = process.env.COPILOT_WEB_RELAY_CONFIG
@@ -280,26 +300,32 @@ const CONFIG_PATH = process.env.COPILOT_WEB_RELAY_CONFIG
280
300
  : path.join(__dirname, 'config.json');
281
301
  // env var > config.json `dataDir` > <server>/data — see data-dir-resolution.mjs
282
302
  // for why the config can carry it (a dev checkout pointing at a release
283
- // install's data, and autostart launchers that only pin the config path).
303
+ // install's data).
284
304
  const DATA_DIR = resolveDataDir({ env: process.env, configPath: CONFIG_PATH, serverDir: __dirname });
285
305
  const DB_PATH = path.join(DATA_DIR, 'copilot.db');
286
306
  const RELAY_LOCK_PATH = path.join(DATA_DIR, 'relay-server.lock');
287
- const UPLOAD_DIR = process.env.COPILOT_WEB_RELAY_DATA_DIR
288
- ? path.join(DATA_DIR, 'uploads')
289
- : path.join(__dirname, 'uploads');
307
+ // Uploads follow the data dir whenever the env var or config.json chose it.
308
+ const UPLOAD_DIR = resolveUploadDir({ dataDir: DATA_DIR, env: process.env, serverDir: __dirname });
309
+ // Uploads earlier versions left in <server>/uploads or <state root>/uploads
310
+ // are read in place; nothing there is written, moved or deleted.
311
+ const UPLOAD_READ_DIRS = resolveUploadReadDirs({ uploadDir: UPLOAD_DIR, dataDir: DATA_DIR, serverDir: __dirname });
290
312
  const INITIAL_WORKSPACE_ROOT = resolveStartupWorkspaceRoot(__dirname);
291
313
  const WORKSPACE_ROOT_LOCKED = true;
292
314
  const SESSION_COOKIE = 'copilot_session';
293
315
  const AUTH_COOKIE = 'copilot_auth';
294
316
 
295
- if (!fs.existsSync(DATA_DIR)) {
296
- fs.mkdirSync(DATA_DIR, { recursive: true, mode: 0o700 });
297
- }
317
+ ensureDataDirs({ dataDir: DATA_DIR, uploadDir: UPLOAD_DIR });
298
318
  if (process.platform !== 'win32') {
299
319
  fs.chmodSync(DATA_DIR, 0o700);
300
320
  }
301
- if (!fs.existsSync(UPLOAD_DIR)) {
302
- fs.mkdirSync(UPLOAD_DIR, { recursive: true });
321
+ // Earlier migrations put a checkout's uploads in <state root>/uploads, which
322
+ // nothing read: merge them in (never overwriting, never deleting the source).
323
+ mergeStrayMigratedUploads({ dataDir: DATA_DIR, uploadDir: UPLOAD_DIR, logger: console });
324
+ for (const dir of UPLOAD_READ_DIRS.slice(1)) {
325
+ try {
326
+ const count = fs.readdirSync(dir).filter((name) => isSha256(name)).length;
327
+ if (count) console.log(`[uploads] New uploads go to ${UPLOAD_DIR}; ${count} earlier upload${count === 1 ? '' : 's'} in ${dir} are read from there.`);
328
+ } catch {}
303
329
  }
304
330
 
305
331
  async function refreshOpenAIProviderModels() {
@@ -1081,18 +1107,20 @@ function setCursorProviderSettings(update = {}) {
1081
1107
  const CURSOR_MODEL_DISCOVERY_TIMEOUT_MS = 20_000;
1082
1108
 
1083
1109
  // Reasoning tiers for one discovered Cursor model, in composer vocabulary.
1084
- // Cursor exposes effort as a per-model parameter named 'effort' or 'reasoning'
1085
- // whose value list varies by model; 'extra-high' is Cursor's spelling of the
1086
- // composer's 'xhigh'. Values with no composer equivalent (e.g. 'minimal') are
1087
- // dropped rather than remapped onto a tier the model does not actually have.
1110
+ // Cursor exposes effort as a per-model parameter named 'effort', 'reasoning'
1111
+ // or 'reasoning_effort' whose value list varies by model; 'extra-high' is
1112
+ // Cursor's spelling of the composer's 'xhigh'. Values with no composer
1113
+ // equivalent (e.g. 'minimal') are dropped rather than remapped onto a tier the
1114
+ // model does not actually have.
1088
1115
  // Mirrors resolveCursorReasoningParams exactly, including its parameter
1089
- // precedence: it consults 'effort' first and only falls back to 'reasoning',
1090
- // so scanning both would advertise an off switch the worker never sends.
1116
+ // precedence: it consults 'effort' first, then 'reasoning', then
1117
+ // 'reasoning_effort', so scanning several would advertise an off switch the
1118
+ // worker never sends.
1091
1119
  function cursorEntrySupportsReasoningOff(entry) {
1092
1120
  const parameters = Array.isArray(entry?.parameters) ? entry.parameters : [];
1093
1121
  const byId = new Map(parameters.map((param) => [String(param?.id || '').trim().toLowerCase(), param]));
1094
1122
  if (byId.has('thinking')) return true;
1095
- const effortParam = byId.get('effort') || byId.get('reasoning');
1123
+ const effortParam = byId.get('effort') || byId.get('reasoning') || byId.get('reasoning_effort');
1096
1124
  const values = Array.isArray(effortParam?.values) ? effortParam.values : [];
1097
1125
  return values.some((value) => (
1098
1126
  String((value && typeof value === 'object' ? value.value : value) || '').trim().toLowerCase() === 'none'
@@ -1105,7 +1133,7 @@ function cursorEntryEffortLevels(entry) {
1105
1133
  // listing both parameters must not have its tiers read from one and its off
1106
1134
  // switch from the other.
1107
1135
  const byId = new Map(parameters.map((param) => [String(param?.id || '').trim().toLowerCase(), param]));
1108
- const effortParam = byId.get('effort') || byId.get('reasoning');
1136
+ const effortParam = byId.get('effort') || byId.get('reasoning') || byId.get('reasoning_effort');
1109
1137
  if (!effortParam) return [];
1110
1138
  const levels = [];
1111
1139
  for (const value of (Array.isArray(effortParam.values) ? effortParam.values : [])) {
@@ -1808,18 +1836,43 @@ function isProcessAlive(pid) {
1808
1836
  }
1809
1837
  }
1810
1838
 
1839
+ // The guard's own stale-lock rules (lockHolderMayRun; an EPERM is alive).
1811
1840
  const relaySingletonGuard = createRelaySingletonGuard({
1812
1841
  lockPath: RELAY_LOCK_PATH,
1813
1842
  pid: process.pid,
1814
1843
  token: config.authToken,
1815
- isProcessAlive,
1816
1844
  logger: console,
1817
1845
  });
1818
1846
  try {
1819
1847
  relaySingletonGuard.acquire();
1820
1848
  } catch (error) {
1849
+ // A held lock is not a crash: the supervisors do not retry 78.
1821
1850
  console.error(`[server] ${error?.message || String(error)}`);
1822
- process.exit(1);
1851
+ process.exit(error?.code === 'ERELAYLOCKED' ? RELAY_STOP_EXIT_CODE : 1);
1852
+ }
1853
+ // Before the database opens, under the lock (adoptEarlierDatabase); the
1854
+ // result also feeds lostPackageDatabaseWarning.
1855
+ let earlierDatabase;
1856
+ {
1857
+ try {
1858
+ earlierDatabase = adoptEarlierDatabase({
1859
+ dataDir: DATA_DIR,
1860
+ uploadDir: UPLOAD_DIR,
1861
+ serverDir: __dirname,
1862
+ configPath: CONFIG_PATH,
1863
+ packageRoot: REPO_ROOT,
1864
+ Database,
1865
+ authToken: config.authToken,
1866
+ logger: console,
1867
+ });
1868
+ } catch (error) {
1869
+ earlierDatabase = { status: 'refused', error: `[data] Not starting: the database this relay ran on until now could not be looked at (${error?.message || error}).` };
1870
+ }
1871
+ if (earlierDatabase.status === 'refused') {
1872
+ console.error(earlierDatabase.error);
1873
+ try { relaySingletonGuard.release(); } catch {}
1874
+ process.exit(RELAY_STOP_EXIT_CODE);
1875
+ }
1823
1876
  }
1824
1877
 
1825
1878
  const MAX_REQUEUE_RETRIES = Number.isFinite(Number(config.maxRequeueRetries))
@@ -1877,7 +1930,7 @@ const runtimeTimers = {
1877
1930
  questionExpiry: null,
1878
1931
  pendingWorkerPrime: null,
1879
1932
  sharedViewerPrune: null,
1880
- shutdownDrain: null,
1933
+ restoredTaskSettle: null,
1881
1934
  };
1882
1935
 
1883
1936
  function normalizeRemotePath(value) {
@@ -2112,7 +2165,7 @@ function setSharePathPrivacyMode(value) {
2112
2165
  // checkout, data and upload dirs, the workspace roots it knows, and the OS
2113
2166
  // user name. The share routes add the conversation's own roots.
2114
2167
  function sharePathKnownRoots() {
2115
- const roots = [REPO_ROOT, DATA_DIR, UPLOAD_DIR, currentWorkspaceRootPath(), INITIAL_WORKSPACE_ROOT];
2168
+ const roots = [REPO_ROOT, DATA_DIR, ...UPLOAD_READ_DIRS, currentWorkspaceRootPath(), INITIAL_WORKSPACE_ROOT];
2116
2169
  const userNames = [];
2117
2170
  try { roots.push(os.homedir()); } catch {}
2118
2171
  try { userNames.push(os.userInfo().username); } catch {}
@@ -2202,6 +2255,21 @@ function setTaskStallMinutes(value) {
2202
2255
  return { ok: true, minutes: parsed.minutes };
2203
2256
  }
2204
2257
 
2258
+ // "After a relay restart, tell interrupted sessions to carry on" (off by default).
2259
+ function getRestartResumeEnabled() {
2260
+ return readRestartResumeSetting(readAppSettingValue(RESTART_RESUME_SETTING_KEY));
2261
+ }
2262
+
2263
+ function setRestartResumeEnabled(value) {
2264
+ if (typeof stmts?.upsertAppSetting?.run !== 'function') {
2265
+ return { ok: false, error: 'Restart settings are unavailable' };
2266
+ }
2267
+ const parsed = parseRestartResumeUpdate(value);
2268
+ if (!parsed.ok) return parsed;
2269
+ stmts.upsertAppSetting.run(RESTART_RESUME_SETTING_KEY, parsed.enabled ? '1' : '0', new Date().toISOString());
2270
+ return { ok: true, enabled: parsed.enabled };
2271
+ }
2272
+
2205
2273
  function resolveDefaultSessionWorkspaceRootState() {
2206
2274
  return resolveDefaultSessionWorkspaceRootStateFromService({
2207
2275
  storedPath: readAppSettingValue(DEFAULT_SESSION_WORKSPACE_ROOT_KEY),
@@ -2648,6 +2716,7 @@ const configuredConversationSessionMode =
2648
2716
  updateModelCatalog({ models: [DEFAULT_MODEL], currentModel: DEFAULT_MODEL, defaultModel: DEFAULT_MODEL, source: 'bootstrap' });
2649
2717
 
2650
2718
  // ─── SQLite Setup ─────────────────────────────────────────────────────────────
2719
+ const databaseCreatedAtStart = !fs.existsSync(DB_PATH);
2651
2720
  const db = new Database(DB_PATH);
2652
2721
  if (process.platform !== 'win32') {
2653
2722
  fs.chmodSync(DB_PATH, 0o600);
@@ -3011,6 +3080,7 @@ function buildSessionWorkerLaunchEnvForSession(targetSessionId) {
3011
3080
  enabled: true,
3012
3081
  model: cursorModel,
3013
3082
  apiKey: cursorSettings.apiKey,
3083
+ dataDir: DATA_DIR,
3014
3084
  });
3015
3085
  }
3016
3086
  if (providerType === 'grok') {
@@ -3451,140 +3521,98 @@ function attachTtyConsoleRuntimeTitle(runtime) {
3451
3521
  }
3452
3522
  attachTtyConsoleRuntimeTitle(ttyConsoleRuntime);
3453
3523
 
3454
- let pendingRelayShutdownRequest = null;
3455
- function normalizeRestartRequestFlag(value) {
3456
- if (value === true || value === 1) return true;
3457
- if (value === false || value === 0 || value == null) return false;
3458
- const normalized = String(value).trim().toLowerCase();
3459
- return normalized === 'true' || normalized === '1' || normalized === 'yes' || normalized === 'on';
3460
- }
3461
-
3462
- function getRelayShutdownState() {
3463
- const pending = pendingRelayShutdownRequest;
3464
- if (!pending) {
3465
- return {
3466
- status: runtimeShutdownStarted ? 'shutting_down' : 'idle',
3467
- action: runtimeShutdownStarted ? 'shutdown' : null,
3468
- restart: false,
3469
- reason: null,
3470
- requestedBy: null,
3471
- requestedAt: null,
3472
- };
3524
+ // ─── Queued relay restart / shutdown ─────────────────────────────────────────
3525
+ // A request waits until no turn is processing and holds new turns meanwhile
3526
+ // (services/relay-shutdown-gate.mjs); past the maximum wait it asks instead
3527
+ // of acting (services/relay-restart-queue.mjs). Every client shows a banner
3528
+ // with what it waits for, from `relay_shutdown_state` (sent on each change
3529
+ // and on connect).
3530
+ const relayRestartMaxWaitOverrideMs = resolveRelayRestartMaxWaitOverrideMs(process.env);
3531
+ if (relayRestartMaxWaitOverrideMs !== null) {
3532
+ console.log(`${runtimeLogPrefix()}Relay restart max wait pinned to ${relayRestartMaxWaitOverrideMs} ms (OAR_RELAY_RESTART_MAX_WAIT_MS)`);
3533
+ }
3534
+ function getRelayRestartMaxWaitMinutes() {
3535
+ return readRelayRestartMaxWaitSetting(readAppSettingValue(RELAY_RESTART_MAX_WAIT_SETTING_KEY));
3536
+ }
3537
+ function setRelayRestartMaxWaitMinutes(value) {
3538
+ if (!stmts.upsertAppSetting) return { ok: false, error: 'Relay restart settings are unavailable' };
3539
+ const parsed = parseRelayRestartMaxWaitUpdate(value);
3540
+ if (!parsed.ok) return parsed;
3541
+ stmts.upsertAppSetting.run(RELAY_RESTART_MAX_WAIT_SETTING_KEY, String(parsed.minutes), new Date().toISOString());
3542
+ // A queued request measures its current period against the new value.
3543
+ relayRestartQueue.refresh();
3544
+ return { ok: true, maxWaitMinutes: parsed.minutes };
3545
+ }
3546
+ let restartPictureCollector = null;
3547
+ // What a restart would wait for, hold and leave running (the confirmation
3548
+ // asks before anything is queued). Built on first use: the background-work
3549
+ // view is set up further down.
3550
+ function collectRestartPicture({ action = 'restart' } = {}) {
3551
+ if (!restartPictureCollector) {
3552
+ restartPictureCollector = createRestartPictureCollector({
3553
+ db,
3554
+ backgroundWorkView,
3555
+ listHeldTasks: () => backgroundTaskStore.listRestored().map(({ key, task }) => ({
3556
+ conversationId: resolveConversationRecord({ conversationId: key, sdkSessionId: key })?.id || key,
3557
+ description: task?.description || task?.taskType || '',
3558
+ })),
3559
+ providerOf: (conversationId) => stmts.getRuntimeSessionByConversation?.get?.(conversationId)?.provider_type,
3560
+ titleOf: (conversationId) => hostActivityTitle(conversationId),
3561
+ });
3473
3562
  }
3474
- return {
3475
- status: runtimeShutdownStarted ? 'shutting_down' : 'queued',
3476
- action: pending.action || 'shutdown',
3477
- restart: pending.action === 'restart',
3478
- reason: pending.reason || null,
3479
- requestedBy: pending.requestedBy || null,
3480
- requestedAt: pending.requestedAt || null,
3481
- };
3563
+ return restartPictureCollector({ action });
3482
3564
  }
3565
+ const relayRestartQueue = createRelayRestartQueue({
3566
+ getCounts: () => queueCounts(),
3567
+ collectPicture: collectRestartPicture,
3568
+ startShutdown: ({ action, reason, requestedBy, requestedAt }) => {
3569
+ const exitCode = action === 'restart' ? RELAY_RESTART_EXIT_CODE : 0;
3570
+ // The next start tells the conversations that asked for relay_restarted
3571
+ // how this one ended (services/relay-event-service.mjs).
3572
+ try { relayEventService?.noteRelayExit({ action, reason, requestedBy, requestedAt }); } catch {}
3573
+ sessionWorkerWebSocketService.emitDraining(`api-${action}:${reason}`);
3574
+ void shutdownRuntime(`api-${action}:${reason}`, { exitCode });
3575
+ },
3576
+ getMaxWaitMs: () => (relayRestartMaxWaitOverrideMs !== null
3577
+ ? relayRestartMaxWaitOverrideMs
3578
+ : getRelayRestartMaxWaitMinutes() * 60_000),
3579
+ isShuttingDown: () => runtimeShutdownStarted,
3580
+ onStateChange: (state) => {
3581
+ try { io.emit('relay_shutdown_state', state); } catch {}
3582
+ },
3583
+ onMaxWaitReached: (state) => {
3584
+ const sessions = Array.isArray(state.waitingFor) ? state.waitingFor.length : 0;
3585
+ const minutes = Math.max(1, Math.round(Number(state.maxWaitMs || 0) / 60_000));
3586
+ void pushDispatchService.notifyRelayRestart({
3587
+ body: `The queued relay ${state.action} has waited ${minutes} min for ${sessions} session${sessions === 1 ? '' : 's'}. Restart now, keep waiting or cancel it.`,
3588
+ });
3589
+ },
3590
+ logger: console,
3591
+ logPrefix: () => runtimeLogPrefix(),
3592
+ });
3483
3593
 
3484
- // Every client shows a banner while a shutdown/restart is queued, so the
3485
- // state goes out on each transition (request, cancel, drain) and on connect.
3486
3594
  function relayShutdownStatePayload() {
3487
- return { ...getRelayShutdownState(), queue: queueCounts() };
3488
- }
3489
- function emitRelayShutdownState() {
3490
- try { io.emit('relay_shutdown_state', relayShutdownStatePayload()); } catch {}
3595
+ return relayRestartQueue.getState();
3491
3596
  }
3492
3597
 
3493
3598
  function cancelRelayShutdown({ requestedBy = 'localhost-api' } = {}) {
3494
- if (runtimeShutdownStarted) {
3495
- return { cancelled: false, status: 'shutting_down', ...relayShutdownStatePayload() };
3496
- }
3497
- if (!pendingRelayShutdownRequest) {
3498
- return { cancelled: false, ...relayShutdownStatePayload() };
3499
- }
3500
- const dropped = pendingRelayShutdownRequest;
3501
- pendingRelayShutdownRequest = null;
3502
- if (runtimeTimers.shutdownDrain) {
3503
- try { clearInterval(runtimeTimers.shutdownDrain); } catch {}
3504
- runtimeTimers.shutdownDrain = null;
3505
- }
3506
- console.log(
3507
- `${runtimeLogPrefix()}Relay ${dropped.action || 'shutdown'} request cancelled by `
3508
- + `${String(requestedBy || 'localhost-api').trim().slice(0, 80) || 'localhost-api'}`,
3509
- );
3510
- emitRelayShutdownState();
3511
- return { cancelled: true, dropped, ...relayShutdownStatePayload() };
3599
+ const result = relayRestartQueue.cancel({ requestedBy });
3600
+ if (result?.cancelled) relayEventService?.noteRestartCancelled();
3601
+ return result;
3512
3602
  }
3513
3603
 
3514
- function requestRelayShutdown({ reason = 'manual-request', requestedBy = 'localhost-api', restart = false } = {}) {
3515
- const safeReason = String(reason || 'manual-request').trim().slice(0, 140) || 'manual-request';
3516
- const safeRequestedBy = String(requestedBy || 'localhost-api').trim().slice(0, 80) || 'localhost-api';
3517
- const restartRequested = normalizeRestartRequestFlag(restart);
3518
- const requestedAction = restartRequested ? 'restart' : 'shutdown';
3519
- if (runtimeShutdownStarted) {
3520
- return {
3521
- accepted: false,
3522
- status: 'shutting_down',
3523
- action: requestedAction,
3524
- restart: restartRequested,
3525
- reason: safeReason,
3526
- };
3527
- }
3528
-
3529
- // Shutdown is deferred until no turn is processing; it never interrupts an
3530
- // active turn. From here on no pending row is handed out (the dequeue paths
3531
- // read relayShutdownQueued): a turn started meanwhile would postpone the
3532
- // restart again. Pending and parked rows wait in the database and run
3533
- // after the restart.
3534
- if (!pendingRelayShutdownRequest) {
3535
- pendingRelayShutdownRequest = {
3536
- action: requestedAction,
3537
- reason: safeReason,
3538
- requestedBy: safeRequestedBy,
3539
- requestedAt: new Date().toISOString(),
3540
- };
3541
- } else if (restartRequested && pendingRelayShutdownRequest.action !== 'restart') {
3542
- pendingRelayShutdownRequest = {
3543
- ...pendingRelayShutdownRequest,
3544
- action: 'restart',
3545
- reason: safeReason,
3546
- requestedBy: safeRequestedBy,
3547
- requestedAt: new Date().toISOString(),
3548
- };
3549
- }
3550
-
3551
- const tryShutdownWhenIdle = () => {
3552
- if (runtimeShutdownStarted) return;
3553
- const counts = queueCounts();
3554
- if (!queuedShutdownCanProceed(counts)) return;
3555
- const requestInfo = pendingRelayShutdownRequest;
3556
- pendingRelayShutdownRequest = null;
3557
- if (runtimeTimers.shutdownDrain) {
3558
- try { clearInterval(runtimeTimers.shutdownDrain); } catch {}
3559
- runtimeTimers.shutdownDrain = null;
3560
- }
3561
- const action = requestInfo?.action === 'restart' ? 'restart' : 'shutdown';
3562
- const exitCode = action === 'restart' ? RELAY_RESTART_EXIT_CODE : 0;
3563
- console.log(
3564
- `${runtimeLogPrefix()}Relay ${action} request accepted by ${requestInfo?.requestedBy || 'unknown'}`
3565
- + `; no turn is processing (${counts.pendingCount} pending, ${counts.parkedCount} parked wait), exiting now (reason=${requestInfo?.reason || 'manual-request'})`,
3566
- );
3567
- sessionWorkerWebSocketService.emitDraining(`api-${action}:${requestInfo?.reason || 'manual-request'}`);
3568
- void shutdownRuntime(`api-${action}:${requestInfo?.reason || 'manual-request'}`, { exitCode });
3569
- emitRelayShutdownState();
3570
- };
3604
+ function keepWaitingForRelayShutdown({ requestedBy = 'localhost-api' } = {}) {
3605
+ return relayRestartQueue.keepWaiting({ requestedBy });
3606
+ }
3571
3607
 
3572
- if (!runtimeTimers.shutdownDrain) {
3573
- runtimeTimers.shutdownDrain = setInterval(tryShutdownWhenIdle, 1000);
3574
- }
3575
- tryShutdownWhenIdle();
3576
- if (pendingRelayShutdownRequest) emitRelayShutdownState();
3577
- const counts = queueCounts();
3578
- return {
3579
- accepted: true,
3580
- status: 'queued',
3581
- action: pendingRelayShutdownRequest?.action || requestedAction,
3582
- restart: (pendingRelayShutdownRequest?.action || requestedAction) === 'restart',
3583
- requestedAt: pendingRelayShutdownRequest?.requestedAt || new Date().toISOString(),
3584
- reason: pendingRelayShutdownRequest?.reason || safeReason,
3585
- requestedBy: pendingRelayShutdownRequest?.requestedBy || safeRequestedBy,
3586
- queue: counts,
3587
- };
3608
+ function requestRelayShutdown(options = {}) {
3609
+ // The loop guard of relay_restarted: a conversation whose turn answers the
3610
+ // last restart is not woken by this one. Noted first, since an immediate
3611
+ // request exits from inside the call.
3612
+ if (!runtimeShutdownStarted) relayEventService?.noteRestartRequested();
3613
+ const result = relayRestartQueue.request(options);
3614
+ if (!relayRestartQueue.isQueued() && !runtimeShutdownStarted) relayEventService?.noteRestartCancelled();
3615
+ return result;
3588
3616
  }
3589
3617
 
3590
3618
  function toNullableInt(value) {
@@ -4227,7 +4255,9 @@ function isSha256(value) {
4227
4255
  }
4228
4256
 
4229
4257
  function uploadPathForSha(sha256) {
4230
- return path.join(UPLOAD_DIR, sha256);
4258
+ let expectedSize = null;
4259
+ try { expectedSize = stmts.getUploadFile.get(String(sha256 || '').trim().toLowerCase())?.size_bytes ?? null; } catch {}
4260
+ return uploadPathInDirs(UPLOAD_READ_DIRS, sha256, { expectedSize });
4231
4261
  }
4232
4262
 
4233
4263
  function uploadContentUrlForSha(sha256) {
@@ -4908,7 +4938,10 @@ function persistUploadBuffer(buffer, options = {}) {
4908
4938
  if (!Buffer.isBuffer(buffer) || !buffer.length) throw new Error('Empty upload');
4909
4939
  if (buffer.length > MAX_UPLOAD_BYTES) throw new Error('Uploaded file too large');
4910
4940
  const sha256 = crypto.createHash('sha256').update(buffer).digest('hex');
4911
- const filePath = uploadPathForSha(sha256);
4941
+ // Always kept in the upload dir: the folders of earlier versions can go
4942
+ // (a package update, a clean checkout), so a new reference never relies on
4943
+ // them. An existing file is never overwritten.
4944
+ const filePath = path.join(UPLOAD_DIR, sha256);
4912
4945
  if (!fs.existsSync(filePath)) {
4913
4946
  const tmpPath = path.join(UPLOAD_DIR, `${sha256}.${process.pid}.${Date.now()}.tmp`);
4914
4947
  fs.writeFileSync(tmpPath, buffer);
@@ -4946,13 +4979,14 @@ function buildStoredAttachment({ sha256, name, type, size }) {
4946
4979
  if (!isSha256(normalizedSha)) return null;
4947
4980
  const normalizedName = String(name || '').trim().slice(0, 255) || `upload-${normalizedSha.slice(0, 12)}`;
4948
4981
  const normalizedType = String(type || '').trim().toLowerCase().slice(0, 127) || 'application/octet-stream';
4982
+ const filePath = uploadPathForSha(normalizedSha);
4949
4983
  return {
4950
4984
  name: normalizedName,
4951
4985
  type: normalizedType,
4952
4986
  size: Number(size || 0),
4953
4987
  sha256: normalizedSha,
4954
- path: uploadPathForSha(normalizedSha),
4955
- reference: `@${uploadPathForSha(normalizedSha)}`,
4988
+ path: filePath,
4989
+ reference: `@${filePath}`,
4956
4990
  contentUrl: uploadContentUrlForSha(normalizedSha),
4957
4991
  };
4958
4992
  }
@@ -5025,7 +5059,9 @@ function deleteOrphanedUploads(hashes) {
5025
5059
  const refs = Number(stmts.countUploadRefsBySha.get(sha256)?.cnt || 0);
5026
5060
  if (refs > 0) continue;
5027
5061
  stmts.deleteUploadFile.run(sha256);
5028
- const filePath = uploadPathForSha(sha256);
5062
+ // Only the upload dir is swept; the folders of earlier versions are
5063
+ // read-only to the relay.
5064
+ const filePath = path.join(UPLOAD_DIR, sha256);
5029
5065
  if (fs.existsSync(filePath)) {
5030
5066
  try { fs.unlinkSync(filePath); } catch {}
5031
5067
  }
@@ -5248,10 +5284,14 @@ function previewCardsForResponse(responseMessageId) {
5248
5284
  .slice(0, 5);
5249
5285
  }
5250
5286
 
5251
- // Live background-task sets, as the session workers publish them. Memory
5252
- // only: after a runtime start the heartbeat's digest brings each set back
5253
- // (services/background-task-store.mjs).
5254
- const backgroundTaskStore = createBackgroundTaskStore();
5287
+ // Live background-task sets, as the session workers publish them. The sets
5288
+ // are memory only: after a runtime start the heartbeat's digest brings each
5289
+ // set back. The console rows (the output file of a running shell, the ended
5290
+ // rows of the last hour) are stored (migration 0018) and restored here; a
5291
+ // running one is held until its worker reports (services/background-task-store.mjs).
5292
+ const backgroundTaskStore = createBackgroundTaskStore({
5293
+ persistence: createSqliteBackgroundTaskPersistence(db, { logger: console }),
5294
+ });
5255
5295
  // The output a worker posts for a task with no output file (the console's
5256
5296
  // fallback); memory only, an hour.
5257
5297
  const taskOutputSnapshots = createTaskOutputSnapshots();
@@ -5366,6 +5406,29 @@ const usageLimitPauseService = createUsageLimitPauseService({
5366
5406
  logger: console,
5367
5407
  });
5368
5408
 
5409
+ // Turns a restart cut off for good (their worker went with the relay, and a
5410
+ // second delivery would run the prompt twice): with the setting on, their
5411
+ // session gets one note to carry on. The rows processing right now are the
5412
+ // ones the runtime before left; the recovery paths below report what became
5413
+ // of each (services/restart-resume-service.mjs).
5414
+ const restartResumeService = createRestartResumeService({
5415
+ db,
5416
+ stmts,
5417
+ isEnabled: getRestartResumeEnabled,
5418
+ emit: (event, payload) => io.emit(event, payload),
5419
+ // The note is an ordinary pending message: the primer starts a stopped
5420
+ // worker, the socket service hands it to a running one.
5421
+ wake: () => {
5422
+ void primePendingSessionWorkers('restart-resume');
5423
+ sessionWorkerWebSocketService?.emitQueueChanged?.('restart-resume');
5424
+ },
5425
+ logger: console,
5426
+ });
5427
+ restartResumeService.captureBoot();
5428
+ function noteRestartTurnOutcome(row, outcome) {
5429
+ try { restartResumeService.noteOutcome(row, outcome); } catch {}
5430
+ }
5431
+
5369
5432
  // ─── Relay events ─────────────────────────────────────────────────────────────
5370
5433
  // What each session is doing (state, tasks, quiet time) and the subscriptions
5371
5434
  // through which a watching session is woken with the events of the sessions
@@ -5531,56 +5594,21 @@ function failConsumedRowTerminally(row, reason) {
5531
5594
  });
5532
5595
  }
5533
5596
 
5534
- function recoverProcessingOlderThan(cutoffIso, requeueAtIso, { ceilingBeforeIso = null } = {}) {
5535
- const params = { inactiveBefore: cutoffIso, ceilingBefore: ceilingBeforeIso || null };
5536
- // Stale background-continuation turns are torn down, never requeued —
5537
- // replaying one would deliver the CLI's own continuation as a user prompt.
5538
- const staleContinuations = stmts.listStaleProcessingContinuations?.all?.(params) || [];
5539
- for (const row of staleContinuations) {
5540
- stmts.dropStaleContinuation?.run?.(row.id);
5541
- io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'failed' });
5542
- // A teardown is a turn ending with no reply; it must be visible in the
5543
- // console log (the only trace of a silent one was `ignored=already_failed`
5544
- // hours later, when the worker's reply arrived for the dead row).
5545
- const lastSeen = row.owner_last_claimed_at || row.processing_at || row.timestamp || null;
5546
- const reason = lastSeen && lastSeen < cutoffIso ? `inactive since ${lastSeen}` : `ceiling (started ${row.processing_at || row.timestamp})`;
5547
- console.warn(`${runtimeLogPrefix()}STALE CONTINUATION ${String(row.id).slice(0, 8)} conv=${String(row.conversation_id).slice(0, 8)} dropped: ${reason}`);
5548
- }
5549
- const listedRows = stmts.listRecoverableProcessing.all(params);
5550
- for (const row of listedRows.filter((entry) => entry?.consumed_at)) failConsumedRowTerminally(row, 'stale-recovery');
5551
- const allRows = listedRows.filter((entry) => !entry?.consumed_at);
5552
- if (!allRows.length) return [];
5553
- // Enforce the retry budget before requeueing: failing these rows first
5554
- // flips them out of 'processing' so the recovery UPDATE below skips them.
5555
- const overBudget = allRows.filter((row) => Number(row?.retry_count || 0) + 1 >= MAX_REQUEUE_RETRIES);
5556
- for (const row of overBudget) failRecoveredRowTerminally(row, 'stale-recovery');
5557
- const rows = allRows.filter((row) => Number(row?.retry_count || 0) + 1 < MAX_REQUEUE_RETRIES);
5558
- if (!rows.length) return [];
5559
-
5560
- const tx = db.transaction(() => {
5561
- stmts.recoverProcessingBefore.run({ ...params, requeueAt: requeueAtIso });
5562
- const settleRecoveredAbortControls = db.prepare(`
5563
- UPDATE relay_control_requests
5564
- SET status = 'failed',
5565
- error = ?,
5566
- updated_at = ?,
5567
- completed_at = ?
5568
- WHERE queue_message_id = ?
5569
- AND type IN ('abort_turn', 'abort_subagent')
5570
- AND status IN ('pending', 'processing')
5571
- `);
5572
- const now = new Date().toISOString();
5573
- for (const row of rows) {
5574
- settleRecoveredAbortControls.run('queue-recovered', now, now, row.id);
5575
- }
5576
- });
5577
- tx();
5578
-
5579
- for (const row of rows) {
5580
- io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'pending' });
5581
- }
5582
- io.emit('queue_updated', { recovered: rows.length });
5583
- return rows;
5597
+ // The stale-turn sweep itself lives in services/stale-turn-recovery.mjs so it
5598
+ // can be tested against a real database; io is created further down, hence
5599
+ // the late-bound emit.
5600
+ const staleTurnRecovery = createStaleTurnRecovery({
5601
+ db,
5602
+ stmts,
5603
+ emit: (event, payload) => io.emit(event, payload),
5604
+ maxRequeueRetries: MAX_REQUEUE_RETRIES,
5605
+ failRecoveredRowTerminally,
5606
+ failConsumedRowTerminally,
5607
+ noteRestartTurnOutcome,
5608
+ warn: (line) => console.warn(`${runtimeLogPrefix()}${line}`),
5609
+ });
5610
+ function recoverProcessingOlderThan(cutoffIso, requeueAtIso, options) {
5611
+ return staleTurnRecovery.recoverProcessingOlderThan(cutoffIso, requeueAtIso, options);
5584
5612
  }
5585
5613
 
5586
5614
  // One `/api/usage` needs the same token for the quota read and the billing
@@ -5915,7 +5943,7 @@ const readContextFromSessionEvents = contextSnapshotService.readContextFromSessi
5915
5943
  // Claude-provider sessions live under the Agent SDK's project layout rather than
5916
5944
  // ~/.copilot/session-state, so their browsable session folder resolves separately.
5917
5945
  const claudeSessionRootResolver = createClaudeSessionRootResolver({ fs, path });
5918
- const cursorSessionRootResolver = createCursorSessionRootResolver({ fs, path, serverDir: __dirname });
5946
+ const cursorSessionRootResolver = createCursorSessionRootResolver({ fs, path, serverDir: __dirname, dataDir: DATA_DIR });
5919
5947
 
5920
5948
  // ─── Express + Socket.io Setup ────────────────────────────────────────────────
5921
5949
  const app = express();
@@ -6442,14 +6470,17 @@ function recoverDeadWorkerProcessingRows(sessionId, reason) {
6442
6470
  // No user prompt to replay: a dead worker's continuation is torn down.
6443
6471
  stmts.dropStaleContinuation?.run?.(row.id);
6444
6472
  io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'failed' });
6473
+ noteRestartTurnOutcome(row, 'cut');
6445
6474
  continue;
6446
6475
  }
6447
6476
  if (row.consumed_at) {
6448
6477
  failConsumedRowTerminally(row, `worker-died-${reason}`);
6478
+ noteRestartTurnOutcome(row, 'cut');
6449
6479
  continue;
6450
6480
  }
6451
6481
  if (Number(row.retry_count || 0) + 1 >= MAX_REQUEUE_RETRIES) {
6452
6482
  failRecoveredRowTerminally(row, `worker-died-${reason}`);
6483
+ noteRestartTurnOutcome(row, 'failed');
6453
6484
  continue;
6454
6485
  }
6455
6486
  // The dead worker's subagent runs can never be terminated by the replay
@@ -6457,6 +6488,7 @@ function recoverDeadWorkerProcessingRows(sessionId, reason) {
6457
6488
  closeStrandedSubagentRuns(stmts.listRunningSubagentRunsByQueueMessage?.all?.(row.id) || [], row.id);
6458
6489
  stmts.recoverProcessingRowKeepOwner?.run?.(requeueAt, row.id);
6459
6490
  io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'pending' });
6491
+ noteRestartTurnOutcome(row, 'requeued');
6460
6492
  recovered += 1;
6461
6493
  }
6462
6494
  if (recovered > 0) {
@@ -6569,7 +6601,7 @@ function sessionHasBackgroundWork(sid) {
6569
6601
  const detachedWorkerWatch = createDetachedWorkerWatch({
6570
6602
  isEnabled: () => featureFlags?.SESSION_WORKER_ROUTING_ENABLED === true
6571
6603
  && !runtimeShutdownStarted
6572
- && !pendingRelayShutdownRequest,
6604
+ && !relayRestartQueue.isQueued(),
6573
6605
  listWaitingSessionIds: () => (stmts.listPendingWorkerOwnerSessionIds?.all?.(25) || [])
6574
6606
  .map((row) => row?.sdk_session_id),
6575
6607
  hasWorkerSocket: (sid) => sessionWorkerWebSocketService.hasWorkerSocket(sid) === true,
@@ -6766,10 +6798,10 @@ const runtimeState = {
6766
6798
  get relayPaused() { return relayPaused; },
6767
6799
  set relayPaused(value) { relayPaused = value; },
6768
6800
  get ttyConsoleActive() { return Boolean(ttyConsoleRuntime?.tty); },
6769
- get relayShutdown() { return getRelayShutdownState(); },
6801
+ get relayShutdown() { return relayRestartQueue.getState(); },
6770
6802
  // A shutdown or restart waits for the processing turns: no pending row
6771
- // goes out meanwhile (see requestRelayShutdown).
6772
- get relayShutdownQueued() { return !!pendingRelayShutdownRequest && !runtimeShutdownStarted; },
6803
+ // goes out meanwhile (services/relay-restart-queue.mjs).
6804
+ get relayShutdownQueued() { return relayRestartQueue.isQueued(); },
6773
6805
  get hostSuspend() { return hostSuspendService.getState(); },
6774
6806
  get usageLimit() {
6775
6807
  return { account: usageLimitPauseService.getAccountState(), pauses: usageLimitPauseService.listPauses() };
@@ -6788,6 +6820,34 @@ const windowsBootAutostartService = createWindowsBootAutostartService({
6788
6820
  packageRoot: REPO_ROOT,
6789
6821
  configPath: CONFIG_PATH,
6790
6822
  });
6823
+ // Autostart entries written before they pinned the data and log folders start
6824
+ // the relay on a database inside the package directory, or before TEMP and
6825
+ // the data drive are there. Brought up to date once this relay holds the
6826
+ // lock and its port; edited ones stay.
6827
+ function refreshWindowsAutostartEntries() {
6828
+ // Entries that ran the relay on the database inside the package.
6829
+ const unpinnedLaunchers = [];
6830
+ const onEarlierForm = ({ path: launcher, pinnedData }) => { if (!pinnedData) unpinnedLaunchers.push(launcher); };
6831
+ for (const [name, refresh] of [
6832
+ ['sign-in entry', () => windowsAutostartService.refreshInstalledEntry({ onEarlierForm })],
6833
+ ['boot launcher', () => windowsBootAutostartService.refreshInstalledLauncher({ onEarlierForm })],
6834
+ ]) {
6835
+ try {
6836
+ if (refresh()) console.log(`[autostart] brought the ${name} up to date: it pins the data and log folders and waits for TEMP and the data drive`);
6837
+ } catch (error) {
6838
+ console.warn(`[autostart] could not bring the ${name} up to date: ${error?.message || error}`);
6839
+ }
6840
+ }
6841
+ const warning = lostPackageDatabaseWarning({
6842
+ unpinnedLaunchers,
6843
+ earlierDatabase,
6844
+ databaseCreated: databaseCreatedAtStart,
6845
+ evidence: unpinnedLaunchers.length > 0 && packageDatabaseEvidence({ serverDir: __dirname, ownDirs: [DATA_DIR, path.dirname(relayConsoleLogPath)] }),
6846
+ serverDir: __dirname,
6847
+ dbPath: DB_PATH,
6848
+ });
6849
+ if (warning) console.error(warning);
6850
+ }
6791
6851
 
6792
6852
  const gitChangesService = createGitChangesService();
6793
6853
  // GET /api/git/remote: where a folder lives on GitHub (New Chat, Claude Cloud).
@@ -7017,6 +7077,9 @@ const sharedRouteDeps = {
7017
7077
  pokeTaskHealth: () => taskHealthPoller.poke(),
7018
7078
  getTaskStallMinutes,
7019
7079
  setTaskStallMinutes,
7080
+ getRestartResumeEnabled,
7081
+ setRestartResumeEnabled,
7082
+ noteRestartTurnOutcome,
7020
7083
  // A closure, not the registry itself: this deps object is built before the
7021
7084
  // preview lane is constructed further down the file.
7022
7085
  listConversationPreviews: (conversationId) => previewRegistry.listForConversation(conversationId),
@@ -7049,6 +7112,7 @@ const sharedRouteDeps = {
7049
7112
  fs,
7050
7113
  path,
7051
7114
  uploadsDir: UPLOAD_DIR,
7115
+ uploadReadDirs: UPLOAD_READ_DIRS,
7052
7116
  fetchUsageSummary,
7053
7117
  fetchCopilotBillingUsage,
7054
7118
  fetchGrokBillingUsage: createGrokBillingUsageFetcher(),
@@ -7146,6 +7210,16 @@ registerRelayEventRoutes(app, {
7146
7210
  relayEventService,
7147
7211
  remoteRelayInbound,
7148
7212
  });
7213
+ registerRelayRestartRoutes(app, {
7214
+ auth,
7215
+ // Read only: a GET neither fires the restart nor asks the question.
7216
+ getState: () => relayRestartQueue.getState(),
7217
+ collectRestartPicture,
7218
+ keepWaiting: keepWaitingForRelayShutdown,
7219
+ getMaxWaitMinutes: getRelayRestartMaxWaitMinutes,
7220
+ setMaxWaitMinutes: setRelayRestartMaxWaitMinutes,
7221
+ maxWaitOverrideMs: relayRestartMaxWaitOverrideMs,
7222
+ });
7149
7223
  registerAboutRoutes(app, {
7150
7224
  auth,
7151
7225
  aboutService: createAboutService({
@@ -7236,6 +7310,8 @@ function recoverStaleMessages() {
7236
7310
  const ceilingBeforeIso = ceilingMs > 0 ? new Date(Date.now() - ceilingMs).toISOString() : null;
7237
7311
  const requeueAt = addMsIso(cliOnline ? 30_000 : 2_000);
7238
7312
  const recoveredRows = recoverProcessingOlderThan(cutoff, requeueAt, { ceilingBeforeIso });
7313
+ // Turns of the start that ended on their own decide their session's note.
7314
+ try { restartResumeService.tick(); } catch {}
7239
7315
  if (recoveredRows.length > 0) {
7240
7316
  const ceilingNote = ceilingBeforeIso ? `, ceiling ${Math.round(ceilingMs / 60_000)}min` : '';
7241
7317
  console.log(
@@ -7561,6 +7637,27 @@ const taskOutputRooms = createTaskOutputRooms({
7561
7637
  // the live tasks with an output file.
7562
7638
  const taskHealthPoller = createTaskHealthPoller({ store: backgroundTaskStore, io });
7563
7639
  taskHealthPoller.start();
7640
+ // Console tasks held since the restart whose worker has not reported within
7641
+ // twice the boot window (at least 15 s: a test relay has none): that worker
7642
+ // did not survive the restart. They are marked ended (their output stays
7643
+ // readable for the hour), never running.
7644
+ runtimeTimers.restoredTaskSettle = setTimeout(() => {
7645
+ runtimeTimers.restoredTaskSettle = null;
7646
+ const keys = backgroundTaskStore.endUnconfirmed();
7647
+ if (!keys.length) return;
7648
+ console.log(`${runtimeLogPrefix()}BG-TASKS ${keys.length} session(s) did not report after the restart; their console tasks are marked ended`);
7649
+ for (const key of keys) {
7650
+ try {
7651
+ io.emit('background_tasks', {
7652
+ conversationId: key,
7653
+ tasks: backgroundTaskStore.withHealth(backgroundTaskStore.get(key), key),
7654
+ endedTasks: backgroundTaskStore.ended(key),
7655
+ });
7656
+ } catch {}
7657
+ }
7658
+ taskHealthPoller.poke();
7659
+ }, Math.max(15_000, 2 * resolveBootReportingWindowMs(process.env)));
7660
+ runtimeTimers.restoredTaskSettle?.unref?.();
7564
7661
  // The handshake presents the token as `auth.token` or the cookie; a query
7565
7662
  // `token` is not read (see services/relay-auth.mjs).
7566
7663
  io.use(relayAuth.socketMiddleware);
@@ -7723,12 +7820,21 @@ registerPreviewRoutes(app, {
7723
7820
  new Date().toISOString(),
7724
7821
  );
7725
7822
  },
7823
+ // A scope named by a conversation's id or its session id covers both.
7824
+ resolveConversationScope: createPreviewScopeResolver({
7825
+ getConversation: (id) => {
7826
+ const conv = stmts.getConvAnyStatus.get(id) || null;
7827
+ return conv && String(conv.status || '').trim().toLowerCase() !== 'deleted' ? conv : null;
7828
+ },
7829
+ getConversationBySdkSessionId: (id) => stmts.getConvBySdkSessionId?.get?.(id) || null,
7830
+ }),
7726
7831
  });
7727
7832
 
7728
7833
  function clearRuntimeTimers() {
7729
7834
  for (const timer of Object.values(runtimeTimers)) {
7730
7835
  if (!timer) continue;
7731
7836
  try { clearInterval(timer); } catch {}
7837
+ try { clearTimeout(timer); } catch {}
7732
7838
  }
7733
7839
  subagentRecordRetention?.stop();
7734
7840
  try { taskHealthPoller.stop(); } catch {}
@@ -7739,7 +7845,8 @@ function clearRuntimeTimers() {
7739
7845
  runtimeTimers.questionExpiry = null;
7740
7846
  runtimeTimers.pendingWorkerPrime = null;
7741
7847
  runtimeTimers.sharedViewerPrune = null;
7742
- runtimeTimers.shutdownDrain = null;
7848
+ runtimeTimers.restoredTaskSettle = null;
7849
+ try { relayRestartQueue.dispose(); } catch {}
7743
7850
  }
7744
7851
 
7745
7852
  let runtimeShutdownPromise = null;
@@ -7780,6 +7887,9 @@ function shutdownRuntime(reason = 'unknown', { exitCode = 0 } = {}) {
7780
7887
  console.warn(`${runtimeLogPrefix()}Grok auth service shutdown failed: ${error?.message || error}`);
7781
7888
  }
7782
7889
  sessionWorkerWebSocketService.stop();
7890
+ // A queued request wrote its own end already; this covers the rest
7891
+ // (signals, the owner gone).
7892
+ try { relayEventService.noteRelayExit({ action: runtimeShutdownExitCode === RELAY_RESTART_EXIT_CODE ? 'restart' : 'shutdown', reason }); } catch {}
7783
7893
  try { relayEventService.stop(); } catch {}
7784
7894
  try { relayEventLinkService.stop(); } catch {}
7785
7895
  tmuxInspectorSocketService.stop();
@@ -7910,12 +8020,19 @@ httpServer.on('error', (error) => {
7910
8020
  process.exit(1);
7911
8021
  });
7912
8022
  httpServer.listen(config.port, listenHost, () => {
8023
+ // Conversations that asked for relay_restarted hear that the relay is back:
8024
+ // once it listens, so a start that fails to bind does not take the record
8025
+ // of how the run before ended.
8026
+ try { relayEventService.noteRelayStarted(); } catch (error) {
8027
+ console.warn(`${runtimeLogPrefix()}relay_restarted not recorded: ${error?.message || error}`);
8028
+ }
7913
8029
  const readyBanner = buildRelayReadyBannerData();
7914
8030
  console.log('');
7915
8031
  for (const line of (readyBanner.lines || [])) {
7916
8032
  console.log(line);
7917
8033
  }
7918
8034
  console.log('\nCLI status: waiting for first heartbeat...\n');
8035
+ refreshWindowsAutostartEntries();
7919
8036
 
7920
8037
  // Start SSH tunnel after server is listening
7921
8038
  sshTunnelManager.start();